feat(studio): training history persistence and past runs viewer (#4501)
* feat(db): add SQLite storage layer for training history * feat(api): add training history endpoints and response models * feat(training): integrate DB persistence into training event loop * feat(ui): add training history views and card grid * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix(studio): address review issues in training history persistence - Strip hf_token/wandb_token from config before SQLite storage - Add UUID suffix to job_id for collision resistance - Use isfinite() for 0.0 metric handling throughout - Respect _should_stop in error event finalization - Run schema DDL once per process, not per connection - Close connection on schema init failure - Guard cleanup_orphaned_runs at startup - Cap _metric_buffer at 500 entries - Make FLUSH_THRESHOLD a class constant - Map 'running' to 'training' phase in historical view - Derive LR/GradNorm from history arrays in historical view - Fix nested button with div[role=button] in history cards - Guard String(value) against null/undefined in config popover - Clear selectedHistoryRunId on auto tab switch * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix(studio): address round-2 review findings across training backend and frontend Backend (training.py): - Move state mutation after proc.start() so a failed spawn does not wedge the backend with is_training=True - Create DB run row eagerly after proc.start() so runs appear in history during model loading, not after first metric event - Rewrite _flush_metrics_to_db() with snapshot-before-insert pattern to preserve metrics arriving during the write and retain buffer on failure - Guard eval_loss with float() coercion and math.isfinite(), matching the existing grad_norm guard - Increase pump thread join timeout from 3s to 8s to cover SQLite's default 5s lock timeout Frontend (studio-page.tsx): - Fix history navigation: check isTrainingRunning instead of showTrainingView in onSelectRun so completed runs are not misrouted - Replace activeTab state + auto-switch useEffect with derived tab to eliminate react-hooks/set-state-in-effect lint violation Frontend (historical-training-view.tsx): - Add explicit "running" branch to message ternary so running runs no longer fall through to "Training errored" - Derive loading from detail/error state and move cleanup to effect return to eliminate react-hooks/set-state-in-effect lint violation Frontend (progress-section.tsx): - Derive stopRequested from isTrainingRunning && stopRequestedLocal to eliminate react-hooks/set-state-in-effect lint violation and remove unused useEffect import * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix(studio): resolve 3 remaining bugs from round-2 review 1. Stuck on Current Run tab [12/20]: Only force "current-run" tab when isTrainingRunning is true, not when stale completed-run data exists. After training ends, users can freely navigate to Configure. 2. Incomplete metric sanitization [7/20]: Apply float() coercion and isfinite() guards to loss and learning_rate, matching the existing pattern used by grad_norm and eval_loss. Prevents TypeError from string values and NaN leaks into history arrays. 3. Stop button state leak across runs [10/20]: Add key={runtime.jobId} to ProgressSection so React remounts it when a new run starts, resetting stopRequestedLocal state. * fix(studio): deduplicate loss/lr sanitization in training event handler Reuse _safe_loss/_safe_lr from the progress update block instead of re-sanitizing the same raw event values for metric history. * fix(studio): restore loss > 0 guard to prevent eval steps injecting 0.0 into metric histories Round-2/3 fixes relaxed the history append guard from `loss > 0` to `loss is not None`, which let eval-only log events (where loss defaults to 0.0) append fake zeros into loss_history and lr_history. Restore the `loss > 0` check to match the worker's own has_train_loss gate. The float() coercion and isfinite() sanitization from round-3 remain intact. * fix(studio): resolve training history bugs — nullable loss/lr, tab nav, sparkline * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Daniel Han <danielhanchen@gmail.com>
This commit is contained in:
parent
3108750bb0
commit
208862218d
25 changed files with 2156 additions and 413 deletions
|
|
@ -81,8 +81,8 @@ class TrainingProgress:
|
|||
epoch: float = 0
|
||||
step: int = 0
|
||||
total_steps: int = 0
|
||||
loss: float = 0.0
|
||||
learning_rate: float = 0.0
|
||||
loss: Optional[float] = None
|
||||
learning_rate: Optional[float] = None
|
||||
is_training: bool = False
|
||||
is_completed: bool = False
|
||||
error: Optional[str] = None
|
||||
|
|
@ -244,7 +244,7 @@ class UnslothTrainer:
|
|||
def on_log(self, args, state, control, logs = None, **kwargs):
|
||||
if not logs:
|
||||
return
|
||||
loss_value = logs.get("loss", logs.get("train_loss", 0.0))
|
||||
loss_value = logs.get("loss", logs.get("train_loss", None))
|
||||
current_step = state.global_step
|
||||
grad_norm = logs.get("grad_norm", None)
|
||||
|
||||
|
|
@ -268,7 +268,7 @@ class UnslothTrainer:
|
|||
step = current_step,
|
||||
epoch = round(state.epoch, 2) if state.epoch else 0,
|
||||
loss = loss_value,
|
||||
learning_rate = logs.get("learning_rate", 0.0),
|
||||
learning_rate = logs.get("learning_rate", None),
|
||||
elapsed_seconds = elapsed_seconds,
|
||||
eta_seconds = eta_seconds,
|
||||
grad_norm = grad_norm,
|
||||
|
|
|
|||
|
|
@ -14,12 +14,14 @@ worker's mp.Queue, and exposes the same API surface to routes/training.py.
|
|||
Pattern follows core/data_recipe/jobs/manager.py.
|
||||
"""
|
||||
|
||||
import json as _json
|
||||
import math
|
||||
import multiprocessing as mp
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
import structlog
|
||||
from datetime import datetime, timezone
|
||||
from loggers import get_logger
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
|
@ -44,8 +46,8 @@ class TrainingProgress:
|
|||
epoch: float = 0
|
||||
step: int = 0
|
||||
total_steps: int = 0
|
||||
loss: float = 0.0
|
||||
learning_rate: float = 0.0
|
||||
loss: Optional[float] = None
|
||||
learning_rate: Optional[float] = None
|
||||
is_training: bool = False
|
||||
is_completed: bool = False
|
||||
error: Optional[str] = None
|
||||
|
|
@ -63,6 +65,8 @@ class TrainingBackend:
|
|||
Launches a fresh subprocess per training job, communicates via mp.Queue.
|
||||
"""
|
||||
|
||||
FLUSH_THRESHOLD: int = 10
|
||||
|
||||
def __init__(self):
|
||||
# Subprocess state
|
||||
self._proc: Optional[mp.Process] = None
|
||||
|
|
@ -91,13 +95,21 @@ class TrainingBackend:
|
|||
self.current_job_id: Optional[str] = None
|
||||
self._output_dir: Optional[str] = None
|
||||
|
||||
# DB persistence
|
||||
self._metric_buffer: list[dict] = []
|
||||
self._run_finalized: bool = False
|
||||
self._db_run_created: bool = False
|
||||
self._db_total_steps_set: bool = False
|
||||
self._db_config: Optional[dict] = None
|
||||
self._db_started_at: Optional[str] = None
|
||||
|
||||
logger.info("TrainingBackend initialized (subprocess mode)")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API (called by routes/training.py)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def start_training(self, **kwargs) -> bool:
|
||||
def start_training(self, job_id: str, **kwargs) -> bool:
|
||||
"""Spawn a subprocess to run the full training pipeline.
|
||||
|
||||
All kwargs are serialized into a config dict and sent to the worker.
|
||||
|
|
@ -108,30 +120,16 @@ class TrainingBackend:
|
|||
logger.warning("Training subprocess already running")
|
||||
return False
|
||||
|
||||
# Join prior pump thread to prevent it from consuming events
|
||||
# from the new job's queue (it reads self._event_queue dynamically).
|
||||
# Join prior pump thread — refuse to start if it won't die
|
||||
if self._pump_thread is not None and self._pump_thread.is_alive():
|
||||
self._pump_thread.join(timeout = 5.0)
|
||||
if self._pump_thread.is_alive():
|
||||
logger.warning("Previous pump thread did not exit within 5s")
|
||||
logger.warning(
|
||||
"Previous pump thread did not exit within 5s — refusing to start"
|
||||
)
|
||||
return False
|
||||
self._pump_thread = None
|
||||
|
||||
# Reset state
|
||||
self._should_stop = False
|
||||
self._cancel_requested = False
|
||||
self._progress = TrainingProgress(
|
||||
is_training = True, status_message = "Initializing training..."
|
||||
)
|
||||
self.loss_history.clear()
|
||||
self.lr_history.clear()
|
||||
self.step_history.clear()
|
||||
self.grad_norm_history.clear()
|
||||
self.grad_norm_step_history.clear()
|
||||
self.eval_loss_history.clear()
|
||||
self.eval_step_history.clear()
|
||||
self.eval_enabled = False
|
||||
self._output_dir = None
|
||||
|
||||
# Build config dict for the subprocess
|
||||
config = {
|
||||
"model_name": kwargs["model_name"],
|
||||
|
|
@ -193,23 +191,62 @@ class TrainingBackend:
|
|||
if config["training_type"] != "LoRA/QLoRA":
|
||||
config["load_in_4bit"] = False
|
||||
|
||||
# Spawn subprocess
|
||||
# Spawn subprocess — use locals so state is untouched on failure
|
||||
from .worker import run_training_process
|
||||
|
||||
self._event_queue = _CTX.Queue()
|
||||
self._stop_queue = _CTX.Queue()
|
||||
event_queue = _CTX.Queue()
|
||||
stop_queue = _CTX.Queue()
|
||||
|
||||
self._proc = _CTX.Process(
|
||||
proc = _CTX.Process(
|
||||
target = run_training_process,
|
||||
kwargs = {
|
||||
"event_queue": self._event_queue,
|
||||
"stop_queue": self._stop_queue,
|
||||
"event_queue": event_queue,
|
||||
"stop_queue": stop_queue,
|
||||
"config": config,
|
||||
},
|
||||
daemon = True,
|
||||
)
|
||||
self._proc.start()
|
||||
logger.info("Training subprocess started (pid=%s)", self._proc.pid)
|
||||
try:
|
||||
proc.start()
|
||||
except Exception:
|
||||
logger.error("Failed to start training subprocess", exc_info = True)
|
||||
return False
|
||||
|
||||
logger.info("Training subprocess started (pid=%s)", proc.pid)
|
||||
|
||||
# Reset state — safe because old pump thread is confirmed dead
|
||||
# and proc.start() succeeded
|
||||
self.current_job_id = job_id
|
||||
self._should_stop = False
|
||||
self._cancel_requested = False
|
||||
self._progress = TrainingProgress(
|
||||
is_training = True, status_message = "Initializing training..."
|
||||
)
|
||||
self.loss_history.clear()
|
||||
self.lr_history.clear()
|
||||
self.step_history.clear()
|
||||
self.grad_norm_history.clear()
|
||||
self.grad_norm_step_history.clear()
|
||||
self.eval_loss_history.clear()
|
||||
self.eval_step_history.clear()
|
||||
self.eval_enabled = False
|
||||
self._output_dir = None
|
||||
self._metric_buffer.clear()
|
||||
self._run_finalized = False
|
||||
self._db_run_created = False
|
||||
self._db_total_steps_set = False
|
||||
self._db_config = {
|
||||
k: v for k, v in config.items() if k not in {"hf_token", "wandb_token"}
|
||||
}
|
||||
self._db_started_at = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
# Assign subprocess handles after state reset
|
||||
self._event_queue = event_queue
|
||||
self._stop_queue = stop_queue
|
||||
self._proc = proc
|
||||
|
||||
# Eagerly create DB run row so the run appears in history during model loading
|
||||
self._ensure_db_run_created()
|
||||
|
||||
# Start event pump thread
|
||||
self._pump_thread = threading.Thread(target = self._pump_loop, daemon = True)
|
||||
|
|
@ -252,6 +289,11 @@ class TrainingBackend:
|
|||
proc.kill()
|
||||
proc.join(timeout = 2.0)
|
||||
|
||||
# Wait for pump thread to finish DB finalization before returning
|
||||
# (8s covers SQLite's default 5s lock timeout plus execution overhead)
|
||||
if self._pump_thread is not None and self._pump_thread.is_alive():
|
||||
self._pump_thread.join(timeout = 8.0)
|
||||
|
||||
def is_training_active(self) -> bool:
|
||||
"""Check if training is currently active."""
|
||||
with self._lock:
|
||||
|
|
@ -389,20 +431,54 @@ class TrainingBackend:
|
|||
self._progress.error
|
||||
or "Training process exited unexpectedly"
|
||||
)
|
||||
|
||||
self._ensure_db_run_created()
|
||||
self._finalize_run_in_db(
|
||||
status = "stopped" if self._should_stop else "error",
|
||||
error_message = None
|
||||
if self._should_stop
|
||||
else "Training process terminated unexpectedly",
|
||||
)
|
||||
return
|
||||
|
||||
def _handle_event(self, event: dict) -> None:
|
||||
"""Apply a subprocess event to local state."""
|
||||
"""Apply a subprocess event to local state.
|
||||
|
||||
State updates happen inside self._lock; DB I/O happens after
|
||||
releasing it so status-polling API endpoints are never blocked
|
||||
by slow SQLite writes.
|
||||
"""
|
||||
etype = event.get("type")
|
||||
db_action: Optional[str] = None
|
||||
db_action_kwargs: dict = {}
|
||||
|
||||
with self._lock:
|
||||
if etype == "progress":
|
||||
self._progress.step = event.get("step", self._progress.step)
|
||||
self._progress.epoch = event.get("epoch", self._progress.epoch)
|
||||
self._progress.loss = event.get("loss", self._progress.loss)
|
||||
self._progress.learning_rate = event.get(
|
||||
"learning_rate", self._progress.learning_rate
|
||||
)
|
||||
# loss/lr are sanitized below; update progress after coercion
|
||||
_raw_loss = event.get("loss")
|
||||
_raw_lr = event.get("learning_rate")
|
||||
try:
|
||||
_safe_loss = float(_raw_loss) if _raw_loss is not None else None
|
||||
except (TypeError, ValueError):
|
||||
logger.debug("Could not convert loss to float: %s", _raw_loss)
|
||||
_safe_loss = None
|
||||
if _safe_loss is not None and not math.isfinite(_safe_loss):
|
||||
_safe_loss = None
|
||||
try:
|
||||
_safe_lr = float(_raw_lr) if _raw_lr is not None else None
|
||||
except (TypeError, ValueError):
|
||||
logger.debug(
|
||||
"Could not convert learning_rate to float: %s", _raw_lr
|
||||
)
|
||||
_safe_lr = None
|
||||
if _safe_lr is not None and not math.isfinite(_safe_lr):
|
||||
_safe_lr = None
|
||||
if _safe_loss is not None:
|
||||
self._progress.loss = _safe_loss
|
||||
if _safe_lr is not None:
|
||||
self._progress.learning_rate = _safe_lr
|
||||
self._progress.total_steps = event.get(
|
||||
"total_steps", self._progress.total_steps
|
||||
)
|
||||
|
|
@ -416,30 +492,85 @@ class TrainingBackend:
|
|||
if status:
|
||||
self._progress.status_message = status
|
||||
|
||||
# Update metric histories
|
||||
# Update metric histories — reuse sanitized values from above
|
||||
step = event.get("step", 0)
|
||||
loss = event.get("loss", 0.0)
|
||||
lr = event.get("learning_rate", 0.0)
|
||||
if step >= 0 and loss > 0:
|
||||
loss = _safe_loss
|
||||
lr = _safe_lr
|
||||
if step > 0 and loss is not None:
|
||||
self.loss_history.append(loss)
|
||||
self.lr_history.append(lr)
|
||||
self.lr_history.append(lr if lr is not None else 0.0)
|
||||
self.step_history.append(step)
|
||||
|
||||
grad_norm = event.get("grad_norm")
|
||||
gn = None
|
||||
if grad_norm is not None:
|
||||
try:
|
||||
gn = float(grad_norm)
|
||||
except (TypeError, ValueError):
|
||||
gn = None
|
||||
if gn is not None and math.isfinite(gn):
|
||||
if step > 0 and gn is not None and math.isfinite(gn):
|
||||
self.grad_norm_history.append(gn)
|
||||
self.grad_norm_step_history.append(step)
|
||||
else:
|
||||
gn = None
|
||||
|
||||
eval_loss = event.get("eval_loss")
|
||||
if eval_loss is not None:
|
||||
self.eval_loss_history.append(eval_loss)
|
||||
self.eval_step_history.append(step)
|
||||
self.eval_enabled = True
|
||||
try:
|
||||
eval_loss = float(eval_loss)
|
||||
except (TypeError, ValueError):
|
||||
logger.debug(
|
||||
"Could not convert eval_loss to float: %s", eval_loss
|
||||
)
|
||||
eval_loss = None
|
||||
if step > 0 and eval_loss is not None and math.isfinite(eval_loss):
|
||||
self.eval_loss_history.append(eval_loss)
|
||||
self.eval_step_history.append(step)
|
||||
self.eval_enabled = True
|
||||
else:
|
||||
eval_loss = None
|
||||
|
||||
# Buffer metric for DB flush (loss/lr already sanitized above)
|
||||
self._metric_buffer.append(
|
||||
{
|
||||
"step": step,
|
||||
"loss": loss,
|
||||
"learning_rate": lr,
|
||||
"grad_norm": gn,
|
||||
"eval_loss": eval_loss,
|
||||
"epoch": event.get("epoch"),
|
||||
"num_tokens": event.get("num_tokens"),
|
||||
"elapsed_seconds": event.get("elapsed_seconds"),
|
||||
}
|
||||
)
|
||||
|
||||
# Decide which DB action to take after releasing the lock
|
||||
if not self._db_run_created and self.current_job_id and self._db_config:
|
||||
db_action = "create_run"
|
||||
db_action_kwargs = {
|
||||
"job_id": self.current_job_id,
|
||||
"model_name": self._db_config["model_name"],
|
||||
"dataset_name": self._db_config.get("hf_dataset")
|
||||
or next(
|
||||
iter(self._db_config.get("local_datasets") or []), "unknown"
|
||||
),
|
||||
"config_json": _json.dumps(self._db_config),
|
||||
"started_at": self._db_started_at
|
||||
or datetime.now(timezone.utc).isoformat(),
|
||||
"total_steps": event.get("total_steps"),
|
||||
}
|
||||
elif (
|
||||
event.get("total_steps")
|
||||
and self._db_run_created
|
||||
and not self._db_total_steps_set
|
||||
):
|
||||
db_action = "update_total_steps"
|
||||
db_action_kwargs = {
|
||||
"job_id": self.current_job_id,
|
||||
"total_steps": event["total_steps"],
|
||||
}
|
||||
elif len(self._metric_buffer) >= self.FLUSH_THRESHOLD:
|
||||
db_action = "flush"
|
||||
|
||||
elif etype == "eval_configured":
|
||||
self.eval_enabled = True
|
||||
|
|
@ -454,6 +585,14 @@ class TrainingBackend:
|
|||
self._output_dir = event.get("output_dir")
|
||||
msg = event.get("status_message", "Training completed")
|
||||
self._progress.status_message = msg
|
||||
if not self._db_run_created and self.current_job_id and self._db_config:
|
||||
db_action = "create_and_finalize"
|
||||
else:
|
||||
db_action = "finalize"
|
||||
db_action_kwargs = {
|
||||
"status": "stopped" if self._should_stop else "completed",
|
||||
"output_dir": self._output_dir,
|
||||
}
|
||||
|
||||
elif etype == "error":
|
||||
self._progress.is_training = False
|
||||
|
|
@ -462,6 +601,149 @@ class TrainingBackend:
|
|||
stack = event.get("stack", "")
|
||||
if stack:
|
||||
logger.error("Stack trace:\n%s", stack)
|
||||
if not self._db_run_created and self.current_job_id and self._db_config:
|
||||
db_action = "create_and_finalize"
|
||||
else:
|
||||
db_action = "finalize"
|
||||
db_action_kwargs = {
|
||||
"status": "stopped" if self._should_stop else "error",
|
||||
"error_message": event.get("error", "Unknown error"),
|
||||
}
|
||||
|
||||
# --- DB I/O outside the lock ---
|
||||
if db_action == "create_run":
|
||||
try:
|
||||
from storage.studio_db import create_run
|
||||
|
||||
create_run(
|
||||
id = db_action_kwargs["job_id"],
|
||||
model_name = db_action_kwargs["model_name"],
|
||||
dataset_name = db_action_kwargs["dataset_name"],
|
||||
config_json = db_action_kwargs["config_json"],
|
||||
started_at = db_action_kwargs["started_at"],
|
||||
total_steps = db_action_kwargs["total_steps"],
|
||||
)
|
||||
self._db_run_created = True
|
||||
if db_action_kwargs["total_steps"]:
|
||||
self._db_total_steps_set = True
|
||||
except Exception:
|
||||
logger.warning("Failed to create DB run record", exc_info = True)
|
||||
elif db_action == "create_and_finalize":
|
||||
self._ensure_db_run_created()
|
||||
self._finalize_run_in_db(**db_action_kwargs)
|
||||
elif db_action == "update_total_steps":
|
||||
try:
|
||||
from storage.studio_db import update_run_total_steps
|
||||
|
||||
update_run_total_steps(
|
||||
db_action_kwargs["job_id"], db_action_kwargs["total_steps"]
|
||||
)
|
||||
self._db_total_steps_set = True
|
||||
except Exception:
|
||||
logger.warning("Failed to update total_steps in DB", exc_info = True)
|
||||
elif db_action == "flush":
|
||||
self._flush_metrics_to_db()
|
||||
elif db_action == "finalize":
|
||||
self._finalize_run_in_db(**db_action_kwargs)
|
||||
|
||||
def _ensure_db_run_created(self) -> None:
|
||||
"""Create the DB row if it doesn't exist yet. Called outside the lock."""
|
||||
if self._db_run_created or not self.current_job_id or not self._db_config:
|
||||
return
|
||||
try:
|
||||
from storage.studio_db import create_run
|
||||
|
||||
dataset_name = self._db_config.get("hf_dataset") or next(
|
||||
iter(self._db_config.get("local_datasets") or []), "unknown"
|
||||
)
|
||||
create_run(
|
||||
id = self.current_job_id,
|
||||
model_name = self._db_config["model_name"],
|
||||
dataset_name = dataset_name,
|
||||
config_json = _json.dumps(self._db_config),
|
||||
started_at = self._db_started_at
|
||||
or datetime.now(timezone.utc).isoformat(),
|
||||
total_steps = self._progress.total_steps or None,
|
||||
)
|
||||
self._db_run_created = True
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to create DB run record for early failure", exc_info = True
|
||||
)
|
||||
|
||||
def _finalize_run_in_db(
|
||||
self,
|
||||
status: str,
|
||||
error_message: Optional[str] = None,
|
||||
output_dir: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Flush remaining metrics and mark a run as finished in the DB."""
|
||||
if not self.current_job_id or not self._db_run_created or self._run_finalized:
|
||||
return
|
||||
self._flush_metrics_to_db()
|
||||
try:
|
||||
from storage.studio_db import finish_run
|
||||
from utils.downsample import downsample
|
||||
|
||||
sparkline = downsample(self.loss_history, 50)
|
||||
finish_run(
|
||||
id = self.current_job_id,
|
||||
status = status,
|
||||
ended_at = datetime.now(timezone.utc).isoformat(),
|
||||
final_step = self._progress.step,
|
||||
final_loss = self._progress.loss
|
||||
if (
|
||||
self._progress.loss is not None
|
||||
and math.isfinite(self._progress.loss)
|
||||
)
|
||||
else None,
|
||||
duration_seconds = self._progress.elapsed_seconds,
|
||||
loss_sparkline = _json.dumps(sparkline),
|
||||
output_dir = output_dir,
|
||||
error_message = error_message,
|
||||
)
|
||||
self._run_finalized = True
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to finalize run in DB (status=%s)", status, exc_info = True
|
||||
)
|
||||
|
||||
def _flush_metrics_to_db(self) -> None:
|
||||
"""Flush buffered metrics to the database and update live progress."""
|
||||
if (
|
||||
not self._metric_buffer
|
||||
or not self.current_job_id
|
||||
or not self._db_run_created
|
||||
):
|
||||
return
|
||||
# Cap buffer to prevent unbounded memory growth
|
||||
if len(self._metric_buffer) > 500:
|
||||
logger.warning(
|
||||
"Metric buffer exceeded 500 entries (%d) — trimming oldest",
|
||||
len(self._metric_buffer),
|
||||
)
|
||||
self._metric_buffer = self._metric_buffer[-500:]
|
||||
# Snapshot before insert so metrics arriving during the write are preserved
|
||||
batch = list(self._metric_buffer)
|
||||
try:
|
||||
from storage.studio_db import insert_metrics_batch, update_run_progress
|
||||
|
||||
insert_metrics_batch(self.current_job_id, batch)
|
||||
del self._metric_buffer[: len(batch)]
|
||||
update_run_progress(
|
||||
id = self.current_job_id,
|
||||
step = self._progress.step,
|
||||
loss = self._progress.loss
|
||||
if (
|
||||
self._progress.loss is not None
|
||||
and math.isfinite(self._progress.loss)
|
||||
)
|
||||
else None,
|
||||
duration_seconds = self._progress.elapsed_seconds,
|
||||
)
|
||||
except Exception:
|
||||
# Leave buffer intact for retry on next flush
|
||||
logger.warning("Failed to flush metrics to DB", exc_info = True)
|
||||
|
||||
@staticmethod
|
||||
def _read_queue(q: Any, timeout_sec: float) -> Optional[dict]:
|
||||
|
|
@ -561,11 +843,13 @@ class TrainingBackend:
|
|||
if progress.error:
|
||||
title = f"Error: {progress.error}"
|
||||
elif progress.is_completed:
|
||||
title = f"Training completed! Final loss: {progress.loss:.4f}"
|
||||
loss_str = f"{progress.loss:.4f}" if progress.loss is not None else "--"
|
||||
title = f"Training completed! Final loss: {loss_str}"
|
||||
elif progress.status_message:
|
||||
title = progress.status_message
|
||||
elif progress.step > 0:
|
||||
title = f"Epoch: {progress.epoch} | Step: {progress.step}/{progress.total_steps} | Loss: {progress.loss:.4f}"
|
||||
loss_str = f"{progress.loss:.4f}" if progress.loss is not None else "--"
|
||||
title = f"Epoch: {progress.epoch} | Step: {progress.step}/{progress.total_steps} | Loss: {loss_str}"
|
||||
else:
|
||||
title = "Training Loss"
|
||||
|
||||
|
|
|
|||
|
|
@ -242,7 +242,7 @@ def run_training_process(
|
|||
|
||||
# Wire up progress callback → event_queue
|
||||
def _on_progress(progress: TrainingProgress):
|
||||
has_train_loss = progress.step >= 0 and progress.loss > 0
|
||||
has_train_loss = progress.step > 0 and progress.loss is not None
|
||||
has_eval_loss = progress.eval_loss is not None
|
||||
if has_train_loss or has_eval_loss:
|
||||
event_queue.put(
|
||||
|
|
@ -918,7 +918,7 @@ def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) ->
|
|||
def on_log(self, args, state, control, logs = None, **kwargs):
|
||||
if not logs:
|
||||
return
|
||||
loss_value = logs.get("loss", logs.get("train_loss", 0.0))
|
||||
loss_value = logs.get("loss", logs.get("train_loss", None))
|
||||
current_step = state.global_step
|
||||
|
||||
elapsed = time.time() - training_start_time
|
||||
|
|
@ -934,7 +934,7 @@ def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) ->
|
|||
"step": current_step,
|
||||
"epoch": round(state.epoch, 2) if state.epoch else 0,
|
||||
"loss": loss_value,
|
||||
"learning_rate": logs.get("learning_rate", 0.0),
|
||||
"learning_rate": logs.get("learning_rate", None),
|
||||
"total_steps": total_steps,
|
||||
"elapsed_seconds": elapsed,
|
||||
"eta_seconds": eta,
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ from routes import (
|
|||
export_router,
|
||||
inference_router,
|
||||
models_router,
|
||||
training_history_router,
|
||||
training_router,
|
||||
)
|
||||
from auth import storage
|
||||
|
|
@ -73,6 +74,17 @@ async def lifespan(app: FastAPI):
|
|||
# Detect hardware first — sets DEVICE global used everywhere
|
||||
detect_hardware()
|
||||
|
||||
from storage.studio_db import cleanup_orphaned_runs
|
||||
|
||||
try:
|
||||
cleanup_orphaned_runs()
|
||||
except Exception as exc:
|
||||
import structlog
|
||||
|
||||
structlog.get_logger(__name__).warning(
|
||||
"cleanup_orphaned_runs failed at startup: %s", exc
|
||||
)
|
||||
|
||||
# Pre-cache the helper GGUF model for LLM-assisted dataset detection.
|
||||
# Runs in a background thread so it doesn't block server startup.
|
||||
import threading
|
||||
|
|
@ -149,6 +161,9 @@ app.include_router(inference_router, prefix = "/v1", tags = ["openai-compat"])
|
|||
app.include_router(datasets_router, prefix = "/api/datasets", tags = ["datasets"])
|
||||
app.include_router(data_recipe_router, prefix = "/api/data-recipe", tags = ["data-recipe"])
|
||||
app.include_router(export_router, prefix = "/api/export", tags = ["export"])
|
||||
app.include_router(
|
||||
training_history_router, prefix = "/api/train", tags = ["training-history"]
|
||||
)
|
||||
|
||||
|
||||
# ============ Health and System Endpoints ============
|
||||
|
|
|
|||
|
|
@ -10,6 +10,11 @@ from .training import (
|
|||
TrainingJobResponse,
|
||||
TrainingStatus,
|
||||
TrainingProgress,
|
||||
TrainingRunSummary,
|
||||
TrainingRunListResponse,
|
||||
TrainingRunMetrics,
|
||||
TrainingRunDetailResponse,
|
||||
TrainingRunDeleteResponse,
|
||||
)
|
||||
from .models import (
|
||||
CheckpointInfo,
|
||||
|
|
@ -71,6 +76,11 @@ __all__ = [
|
|||
"TrainingJobResponse",
|
||||
"TrainingStatus",
|
||||
"TrainingProgress",
|
||||
"TrainingRunSummary",
|
||||
"TrainingRunListResponse",
|
||||
"TrainingRunMetrics",
|
||||
"TrainingRunDetailResponse",
|
||||
"TrainingRunDeleteResponse",
|
||||
# Model management schemas
|
||||
"ModelDetails",
|
||||
"LocalModelInfo",
|
||||
|
|
|
|||
|
|
@ -177,8 +177,8 @@ class TrainingProgress(BaseModel):
|
|||
job_id: str = Field(..., description = "Training job identifier")
|
||||
step: int = Field(..., description = "Current training step")
|
||||
total_steps: int = Field(..., description = "Total training steps")
|
||||
loss: float = Field(..., description = "Current loss value")
|
||||
learning_rate: float = Field(..., description = "Current learning rate")
|
||||
loss: Optional[float] = Field(None, description = "Current loss value")
|
||||
learning_rate: Optional[float] = Field(None, description = "Current learning rate")
|
||||
progress_percent: float = Field(
|
||||
..., description = "Progress percentage (0.0 to 100.0)"
|
||||
)
|
||||
|
|
@ -196,3 +196,59 @@ class TrainingProgress(BaseModel):
|
|||
eval_loss: Optional[float] = Field(
|
||||
None, description = "Eval loss from the most recent evaluation step"
|
||||
)
|
||||
|
||||
|
||||
class TrainingRunSummary(BaseModel):
|
||||
"""Summary of a training run for list views."""
|
||||
|
||||
id: str
|
||||
status: Literal["running", "completed", "stopped", "error"]
|
||||
model_name: str
|
||||
dataset_name: str
|
||||
started_at: str
|
||||
ended_at: Optional[str] = None
|
||||
total_steps: Optional[int] = None
|
||||
final_step: Optional[int] = None
|
||||
final_loss: Optional[float] = None
|
||||
output_dir: Optional[str] = None
|
||||
duration_seconds: Optional[float] = None
|
||||
error_message: Optional[str] = None
|
||||
loss_sparkline: Optional[List[float]] = None
|
||||
|
||||
|
||||
class TrainingRunListResponse(BaseModel):
|
||||
"""Response for listing training runs."""
|
||||
|
||||
runs: List[TrainingRunSummary]
|
||||
total: int
|
||||
|
||||
|
||||
class TrainingRunMetrics(BaseModel):
|
||||
"""Metrics arrays for a training run, using paired step arrays per metric."""
|
||||
|
||||
step_history: List[int] = Field(default_factory = list)
|
||||
loss_history: List[float] = Field(default_factory = list)
|
||||
loss_step_history: List[int] = Field(default_factory = list)
|
||||
lr_history: List[float] = Field(default_factory = list)
|
||||
lr_step_history: List[int] = Field(default_factory = list)
|
||||
grad_norm_history: List[float] = Field(default_factory = list)
|
||||
grad_norm_step_history: List[int] = Field(default_factory = list)
|
||||
eval_loss_history: List[float] = Field(default_factory = list)
|
||||
eval_step_history: List[int] = Field(default_factory = list)
|
||||
final_epoch: Optional[float] = None
|
||||
final_num_tokens: Optional[int] = None
|
||||
|
||||
|
||||
class TrainingRunDetailResponse(BaseModel):
|
||||
"""Response for a single training run with config and metrics."""
|
||||
|
||||
run: TrainingRunSummary
|
||||
config: dict
|
||||
metrics: TrainingRunMetrics
|
||||
|
||||
|
||||
class TrainingRunDeleteResponse(BaseModel):
|
||||
"""Response for deleting a training run."""
|
||||
|
||||
status: str
|
||||
message: str
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from routes.datasets import router as datasets_router
|
|||
from routes.auth import router as auth_router
|
||||
from routes.data_recipe import router as data_recipe_router
|
||||
from routes.export import router as export_router
|
||||
from routes.training_history import router as training_history_router
|
||||
|
||||
__all__ = [
|
||||
"training_router",
|
||||
|
|
@ -21,4 +22,5 @@ __all__ = [
|
|||
"auth_router",
|
||||
"data_recipe_router",
|
||||
"export_router",
|
||||
"training_history_router",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import structlog
|
|||
from loggers import get_logger
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
import uuid as _uuid
|
||||
|
||||
# Add backend directory to path
|
||||
# The backend code should be in the same directory structure
|
||||
|
|
@ -115,15 +116,11 @@ async def start_training(
|
|||
|
||||
backend = get_training_backend()
|
||||
|
||||
# Generate job ID and attach to backend for later status/progress calls
|
||||
job_id = f"job_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
|
||||
backend.current_job_id = job_id
|
||||
|
||||
# Check if training is already active
|
||||
# Check if training is already active (before mutating any state)
|
||||
if backend.is_training_active():
|
||||
existing_job_id: Optional[str] = getattr(backend, "current_job_id", "")
|
||||
return TrainingJobResponse(
|
||||
job_id = existing_job_id or job_id,
|
||||
job_id = existing_job_id or "",
|
||||
status = "error",
|
||||
message = (
|
||||
"Training is already in progress. "
|
||||
|
|
@ -132,6 +129,12 @@ async def start_training(
|
|||
error = "Training already active",
|
||||
)
|
||||
|
||||
# Generate job ID — passed into start_training() which sets it on the
|
||||
# backend only after confirming the old pump thread is dead.
|
||||
job_id = (
|
||||
f"job_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{_uuid.uuid4().hex[:8]}"
|
||||
)
|
||||
|
||||
# Validate dataset paths if provided
|
||||
if request.local_datasets:
|
||||
request.local_datasets = _validate_local_dataset_paths(
|
||||
|
|
@ -248,12 +251,12 @@ async def start_training(
|
|||
logger.warning("Could not shut down export subprocess: %s", e)
|
||||
|
||||
# start_training now spawns a subprocess (non-blocking)
|
||||
success = backend.start_training(**training_kwargs)
|
||||
success = backend.start_training(job_id = job_id, **training_kwargs)
|
||||
|
||||
if not success:
|
||||
progress_error = backend.trainer.training_progress.error
|
||||
return TrainingJobResponse(
|
||||
job_id = job_id,
|
||||
job_id = backend.current_job_id or "",
|
||||
status = "error",
|
||||
message = progress_error or "Failed to start training subprocess",
|
||||
error = progress_error or "subprocess_start_failed",
|
||||
|
|
@ -345,7 +348,7 @@ async def reset_training(
|
|||
error = None,
|
||||
status_message = "Ready to train",
|
||||
step = 0,
|
||||
loss = 0.0,
|
||||
loss = None,
|
||||
epoch = 0,
|
||||
total_steps = 0,
|
||||
)
|
||||
|
|
@ -419,8 +422,8 @@ async def get_training_status(
|
|||
"epoch": getattr(progress, "epoch", 0),
|
||||
"step": getattr(progress, "step", 0),
|
||||
"total_steps": getattr(progress, "total_steps", 0),
|
||||
"loss": getattr(progress, "loss", 0.0),
|
||||
"learning_rate": getattr(progress, "learning_rate", 0.0),
|
||||
"loss": getattr(progress, "loss", None),
|
||||
"learning_rate": getattr(progress, "learning_rate", None),
|
||||
}
|
||||
|
||||
# Build metric history for chart recovery after SSE reconnection
|
||||
|
|
@ -526,8 +529,8 @@ async def stream_training_progress(
|
|||
# ── Helpers ──────────────────────────────────────────────
|
||||
def build_progress(
|
||||
step: int,
|
||||
loss: float,
|
||||
learning_rate: float,
|
||||
loss: Optional[float],
|
||||
learning_rate: Optional[float],
|
||||
total_steps: int,
|
||||
epoch: Optional[float] = None,
|
||||
progress: Optional[Any] = None,
|
||||
|
|
@ -604,10 +607,10 @@ async def stream_training_progress(
|
|||
loss_val = (
|
||||
backend.loss_history[i]
|
||||
if i < len(backend.loss_history)
|
||||
else 0.0
|
||||
else None
|
||||
)
|
||||
lr_val = (
|
||||
backend.lr_history[i] if i < len(backend.lr_history) else 0.0
|
||||
backend.lr_history[i] if i < len(backend.lr_history) else None
|
||||
)
|
||||
tp_replay = getattr(
|
||||
getattr(backend, "trainer", None), "training_progress", None
|
||||
|
|
@ -645,8 +648,8 @@ async def stream_training_progress(
|
|||
|
||||
initial_progress = build_progress(
|
||||
step = 0,
|
||||
loss = 0.0,
|
||||
learning_rate = 0.0,
|
||||
loss = None,
|
||||
learning_rate = None,
|
||||
total_steps = initial_total_steps,
|
||||
epoch = initial_epoch,
|
||||
progress = tp,
|
||||
|
|
@ -660,9 +663,9 @@ async def stream_training_progress(
|
|||
if backend.step_history:
|
||||
final_step = backend.step_history[-1]
|
||||
final_loss = (
|
||||
backend.loss_history[-1] if backend.loss_history else 0.0
|
||||
backend.loss_history[-1] if backend.loss_history else None
|
||||
)
|
||||
final_lr = backend.lr_history[-1] if backend.lr_history else 0.0
|
||||
final_lr = backend.lr_history[-1] if backend.lr_history else None
|
||||
final_total_steps = (
|
||||
getattr(tp, "total_steps", final_step) if tp else final_step
|
||||
)
|
||||
|
|
@ -680,7 +683,9 @@ async def stream_training_progress(
|
|||
)
|
||||
else:
|
||||
yield format_sse(
|
||||
build_progress(-1, 0.0, 0.0, 0, progress = tp).model_dump_json(),
|
||||
build_progress(
|
||||
-1, None, None, 0, progress = tp
|
||||
).model_dump_json(),
|
||||
event = "complete",
|
||||
event_id = 0,
|
||||
)
|
||||
|
|
@ -698,9 +703,9 @@ async def stream_training_progress(
|
|||
if backend.step_history:
|
||||
current_step = backend.step_history[-1]
|
||||
current_loss = (
|
||||
backend.loss_history[-1] if backend.loss_history else 0.0
|
||||
backend.loss_history[-1] if backend.loss_history else None
|
||||
)
|
||||
current_lr = backend.lr_history[-1] if backend.lr_history else 0.0
|
||||
current_lr = backend.lr_history[-1] if backend.lr_history else None
|
||||
tp_inner = getattr(
|
||||
getattr(backend, "trainer", None), "training_progress", None
|
||||
)
|
||||
|
|
@ -763,8 +768,8 @@ async def stream_training_progress(
|
|||
)
|
||||
preparing_payload = build_progress(
|
||||
0,
|
||||
0.0,
|
||||
0.0,
|
||||
None,
|
||||
None,
|
||||
prep_total,
|
||||
progress = tp_prep,
|
||||
)
|
||||
|
|
@ -781,7 +786,7 @@ async def stream_training_progress(
|
|||
getattr(backend, "trainer", None), "training_progress", None
|
||||
)
|
||||
timeout_payload = build_progress(
|
||||
last_step, 0.0, 0.0, 0, progress = tp_timeout
|
||||
last_step, None, None, 0, progress = tp_timeout
|
||||
)
|
||||
yield format_sse(
|
||||
timeout_payload.model_dump_json(),
|
||||
|
|
@ -797,7 +802,7 @@ async def stream_training_progress(
|
|||
tp_error = getattr(
|
||||
getattr(backend, "trainer", None), "training_progress", None
|
||||
)
|
||||
error_payload = build_progress(0, 0.0, 0.0, 0, progress = tp_error)
|
||||
error_payload = build_progress(0, None, None, 0, progress = tp_error)
|
||||
yield format_sse(
|
||||
error_payload.model_dump_json(),
|
||||
event = "error",
|
||||
|
|
@ -807,8 +812,8 @@ async def stream_training_progress(
|
|||
|
||||
# ── Final "complete" event ───────────────────────────────
|
||||
final_step = backend.step_history[-1] if backend.step_history else last_step
|
||||
final_loss = backend.loss_history[-1] if backend.loss_history else 0.0
|
||||
final_lr = backend.lr_history[-1] if backend.lr_history else 0.0
|
||||
final_loss = backend.loss_history[-1] if backend.loss_history else None
|
||||
final_lr = backend.lr_history[-1] if backend.lr_history else None
|
||||
final_tp = getattr(getattr(backend, "trainer", None), "training_progress", None)
|
||||
final_total_steps = (
|
||||
getattr(final_tp, "total_steps", final_step) if final_tp else final_step
|
||||
|
|
|
|||
85
studio/backend/routes/training_history.py
Normal file
85
studio/backend/routes/training_history.py
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
"""
|
||||
Training history API routes — browse, view, and delete past training runs.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from loggers import get_logger
|
||||
|
||||
from auth.authentication import get_current_subject
|
||||
from models import (
|
||||
TrainingRunDeleteResponse,
|
||||
TrainingRunDetailResponse,
|
||||
TrainingRunListResponse,
|
||||
TrainingRunMetrics,
|
||||
TrainingRunSummary,
|
||||
)
|
||||
from storage.studio_db import delete_run, get_run, get_run_metrics, list_runs
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/runs", response_model = TrainingRunListResponse)
|
||||
async def list_training_runs(
|
||||
limit: int = Query(50, ge = 1, le = 200),
|
||||
offset: int = Query(0, ge = 0),
|
||||
current_subject: str = Depends(get_current_subject),
|
||||
):
|
||||
"""List training runs, newest first."""
|
||||
result = list_runs(limit = limit, offset = offset)
|
||||
return TrainingRunListResponse(
|
||||
runs = [TrainingRunSummary(**r) for r in result["runs"]],
|
||||
total = result["total"],
|
||||
)
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}", response_model = TrainingRunDetailResponse)
|
||||
async def get_training_run_detail(
|
||||
run_id: str,
|
||||
current_subject: str = Depends(get_current_subject),
|
||||
):
|
||||
"""Get a single training run with full config and metrics."""
|
||||
run = get_run(run_id)
|
||||
if run is None:
|
||||
raise HTTPException(status_code = 404, detail = f"Run {run_id} not found")
|
||||
|
||||
try:
|
||||
config = json.loads(run.get("config_json", "{}"))
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
logger.debug("Failed to parse config_json for run %s", run_id)
|
||||
config = {}
|
||||
|
||||
metrics_data = get_run_metrics(run_id)
|
||||
|
||||
return TrainingRunDetailResponse(
|
||||
run = TrainingRunSummary(**{k: v for k, v in run.items() if k != "config_json"}),
|
||||
config = config,
|
||||
metrics = TrainingRunMetrics(**metrics_data),
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/runs/{run_id}", response_model = TrainingRunDeleteResponse)
|
||||
async def delete_training_run(
|
||||
run_id: str,
|
||||
current_subject: str = Depends(get_current_subject),
|
||||
):
|
||||
"""Delete a training run and its metrics (CASCADE)."""
|
||||
run = get_run(run_id)
|
||||
if run is None:
|
||||
raise HTTPException(status_code = 404, detail = f"Run {run_id} not found")
|
||||
if run["status"] == "running":
|
||||
raise HTTPException(
|
||||
status_code = 409, detail = "Cannot delete a running training run"
|
||||
)
|
||||
logger.info("Deleting training run %s", run_id)
|
||||
delete_run(run_id)
|
||||
return TrainingRunDeleteResponse(
|
||||
status = "deleted",
|
||||
message = f"Run {run_id} deleted",
|
||||
)
|
||||
2
studio/backend/storage/__init__.py
Normal file
2
studio/backend/storage/__init__.py
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
362
studio/backend/storage/studio_db.py
Normal file
362
studio/backend/storage/studio_db.py
Normal file
|
|
@ -0,0 +1,362 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
"""
|
||||
SQLite storage for training run history and metrics.
|
||||
|
||||
Follows the same pattern as auth/storage.py — module-level functions,
|
||||
raw sqlite3, per-function connections. Enhancements over auth:
|
||||
- WAL mode for concurrent read/write access
|
||||
- PRAGMA foreign_keys = ON for CASCADE deletes
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
import threading
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from typing import Optional
|
||||
|
||||
from utils.paths import studio_db_path, ensure_dir
|
||||
|
||||
_schema_lock = threading.Lock()
|
||||
_schema_ready = False
|
||||
|
||||
|
||||
def _ensure_schema(conn: sqlite3.Connection) -> None:
|
||||
"""Create tables and indexes if they don't exist. Called once per process."""
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS training_runs (
|
||||
id TEXT NOT NULL PRIMARY KEY,
|
||||
status TEXT NOT NULL DEFAULT 'running',
|
||||
model_name TEXT NOT NULL,
|
||||
dataset_name TEXT NOT NULL,
|
||||
config_json TEXT NOT NULL,
|
||||
started_at TEXT NOT NULL,
|
||||
ended_at TEXT,
|
||||
total_steps INTEGER,
|
||||
final_step INTEGER,
|
||||
final_loss REAL,
|
||||
output_dir TEXT,
|
||||
error_message TEXT,
|
||||
duration_seconds REAL,
|
||||
loss_sparkline TEXT
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS training_metrics (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
run_id TEXT NOT NULL REFERENCES training_runs(id) ON DELETE CASCADE,
|
||||
step INTEGER NOT NULL,
|
||||
loss REAL,
|
||||
learning_rate REAL,
|
||||
grad_norm REAL,
|
||||
eval_loss REAL,
|
||||
epoch REAL,
|
||||
num_tokens INTEGER,
|
||||
elapsed_seconds REAL,
|
||||
UNIQUE(run_id, step)
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_metrics_run_id ON training_metrics(run_id)"
|
||||
)
|
||||
|
||||
|
||||
def get_connection() -> sqlite3.Connection:
|
||||
"""Open studio.db with WAL mode, create tables once per process, enable foreign keys."""
|
||||
global _schema_ready
|
||||
db_path = studio_db_path()
|
||||
ensure_dir(db_path.parent)
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
# foreign_keys is session-scoped, must be set per connection
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
if not _schema_ready:
|
||||
with _schema_lock:
|
||||
if not _schema_ready:
|
||||
try:
|
||||
_ensure_schema(conn)
|
||||
_schema_ready = True
|
||||
except Exception:
|
||||
conn.close()
|
||||
raise
|
||||
return conn
|
||||
|
||||
|
||||
def create_run(
|
||||
id: str,
|
||||
model_name: str,
|
||||
dataset_name: str,
|
||||
config_json: str,
|
||||
started_at: str,
|
||||
total_steps: Optional[int],
|
||||
) -> None:
|
||||
conn = get_connection()
|
||||
try:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO training_runs (id, model_name, dataset_name, config_json, started_at, total_steps)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(id, model_name, dataset_name, config_json, started_at, total_steps),
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def update_run_total_steps(id: str, total_steps: int) -> None:
|
||||
conn = get_connection()
|
||||
try:
|
||||
conn.execute(
|
||||
"UPDATE training_runs SET total_steps = ? WHERE id = ?",
|
||||
(total_steps, id),
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def update_run_progress(
|
||||
id: str, step: int, loss: Optional[float], duration_seconds: Optional[float]
|
||||
) -> None:
|
||||
"""Update current progress on a running training run (called on each metric flush)."""
|
||||
conn = get_connection()
|
||||
try:
|
||||
conn.execute(
|
||||
"UPDATE training_runs SET final_step = ?, final_loss = ?, duration_seconds = ? WHERE id = ?",
|
||||
(step, loss, duration_seconds, id),
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def finish_run(
|
||||
id: str,
|
||||
status: str,
|
||||
ended_at: str,
|
||||
final_step: Optional[int],
|
||||
final_loss: Optional[float],
|
||||
duration_seconds: Optional[float],
|
||||
loss_sparkline: Optional[str] = None,
|
||||
output_dir: Optional[str] = None,
|
||||
error_message: Optional[str] = None,
|
||||
) -> None:
|
||||
conn = get_connection()
|
||||
try:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE training_runs
|
||||
SET status = ?, ended_at = ?, final_step = ?, final_loss = ?,
|
||||
duration_seconds = ?, loss_sparkline = ?, output_dir = ?,
|
||||
error_message = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(
|
||||
status,
|
||||
ended_at,
|
||||
final_step,
|
||||
final_loss,
|
||||
duration_seconds,
|
||||
loss_sparkline,
|
||||
output_dir,
|
||||
error_message,
|
||||
id,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def insert_metrics_batch(run_id: str, metrics: list[dict]) -> None:
|
||||
if not metrics:
|
||||
return
|
||||
conn = get_connection()
|
||||
try:
|
||||
conn.executemany(
|
||||
"""
|
||||
INSERT INTO training_metrics
|
||||
(run_id, step, loss, learning_rate, grad_norm, eval_loss, epoch, num_tokens, elapsed_seconds)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(run_id, step) DO UPDATE SET
|
||||
loss = COALESCE(excluded.loss, loss),
|
||||
learning_rate = COALESCE(excluded.learning_rate, learning_rate),
|
||||
grad_norm = COALESCE(excluded.grad_norm, grad_norm),
|
||||
eval_loss = COALESCE(excluded.eval_loss, eval_loss),
|
||||
epoch = COALESCE(excluded.epoch, epoch),
|
||||
num_tokens = COALESCE(excluded.num_tokens, num_tokens),
|
||||
elapsed_seconds = COALESCE(excluded.elapsed_seconds, elapsed_seconds)
|
||||
""",
|
||||
[
|
||||
(
|
||||
run_id,
|
||||
m.get("step"),
|
||||
m.get("loss"),
|
||||
m.get("learning_rate"),
|
||||
m.get("grad_norm"),
|
||||
m.get("eval_loss"),
|
||||
m.get("epoch"),
|
||||
m.get("num_tokens"),
|
||||
m.get("elapsed_seconds"),
|
||||
)
|
||||
for m in metrics
|
||||
],
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def list_runs(limit: int = 50, offset: int = 0) -> dict:
|
||||
conn = get_connection()
|
||||
try:
|
||||
total = conn.execute("SELECT COUNT(*) FROM training_runs").fetchone()[0]
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, status, model_name, dataset_name, started_at, ended_at,
|
||||
total_steps, final_step, final_loss, output_dir,
|
||||
duration_seconds, error_message, loss_sparkline
|
||||
FROM training_runs
|
||||
ORDER BY started_at DESC
|
||||
LIMIT ? OFFSET ?
|
||||
""",
|
||||
(limit, offset),
|
||||
).fetchall()
|
||||
runs = []
|
||||
for row in rows:
|
||||
run = dict(row)
|
||||
sparkline = run.get("loss_sparkline")
|
||||
if sparkline:
|
||||
try:
|
||||
run["loss_sparkline"] = json.loads(sparkline)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
logger.debug(
|
||||
"Failed to parse loss_sparkline for run %s", run.get("id")
|
||||
)
|
||||
run["loss_sparkline"] = None
|
||||
runs.append(run)
|
||||
return {"runs": runs, "total": total}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_run(id: str) -> Optional[dict]:
|
||||
conn = get_connection()
|
||||
try:
|
||||
row = conn.execute("SELECT * FROM training_runs WHERE id = ?", (id,)).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
run = dict(row)
|
||||
sparkline = run.get("loss_sparkline")
|
||||
if sparkline:
|
||||
try:
|
||||
run["loss_sparkline"] = json.loads(sparkline)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
logger.debug("Failed to parse loss_sparkline for run %s", id)
|
||||
run["loss_sparkline"] = None
|
||||
return run
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_run_metrics(id: str) -> dict:
|
||||
"""Return metric arrays for a run, using paired step arrays per metric."""
|
||||
conn = get_connection()
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT step, loss, learning_rate, grad_norm, eval_loss, epoch,
|
||||
num_tokens, elapsed_seconds
|
||||
FROM training_metrics
|
||||
WHERE run_id = ?
|
||||
ORDER BY step
|
||||
""",
|
||||
(id,),
|
||||
).fetchall()
|
||||
|
||||
step_history: list[int] = []
|
||||
loss_history: list[float] = []
|
||||
loss_step_history: list[int] = []
|
||||
lr_history: list[float] = []
|
||||
lr_step_history: list[int] = []
|
||||
grad_norm_history: list[float] = []
|
||||
grad_norm_step_history: list[int] = []
|
||||
eval_loss_history: list[float] = []
|
||||
eval_step_history: list[int] = []
|
||||
final_epoch: float | None = None
|
||||
final_num_tokens: int | None = None
|
||||
|
||||
for row in rows:
|
||||
step = row["step"]
|
||||
step_history.append(step)
|
||||
if step > 0 and row["loss"] is not None:
|
||||
loss_history.append(row["loss"])
|
||||
loss_step_history.append(step)
|
||||
if step > 0 and row["learning_rate"] is not None:
|
||||
lr_history.append(row["learning_rate"])
|
||||
lr_step_history.append(step)
|
||||
if step > 0 and row["grad_norm"] is not None:
|
||||
grad_norm_history.append(row["grad_norm"])
|
||||
grad_norm_step_history.append(step)
|
||||
if step > 0 and row["eval_loss"] is not None:
|
||||
eval_loss_history.append(row["eval_loss"])
|
||||
eval_step_history.append(step)
|
||||
if row["epoch"] is not None:
|
||||
final_epoch = row["epoch"]
|
||||
if row["num_tokens"] is not None:
|
||||
final_num_tokens = row["num_tokens"]
|
||||
|
||||
return {
|
||||
"step_history": step_history,
|
||||
"loss_history": loss_history,
|
||||
"loss_step_history": loss_step_history,
|
||||
"lr_history": lr_history,
|
||||
"lr_step_history": lr_step_history,
|
||||
"grad_norm_history": grad_norm_history,
|
||||
"grad_norm_step_history": grad_norm_step_history,
|
||||
"eval_loss_history": eval_loss_history,
|
||||
"eval_step_history": eval_step_history,
|
||||
"final_epoch": final_epoch,
|
||||
"final_num_tokens": final_num_tokens,
|
||||
}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def delete_run(id: str) -> None:
|
||||
conn = get_connection()
|
||||
try:
|
||||
conn.execute("DELETE FROM training_runs WHERE id = ?", (id,))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def cleanup_orphaned_runs() -> None:
|
||||
"""Mark any 'running' rows as errored on startup (server restarted mid-training)."""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
conn = get_connection()
|
||||
try:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE training_runs
|
||||
SET status = 'error',
|
||||
error_message = 'Server restarted during training',
|
||||
ended_at = ?
|
||||
WHERE status = 'running'
|
||||
""",
|
||||
(datetime.now(timezone.utc).isoformat(),),
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
18
studio/backend/utils/downsample.py
Normal file
18
studio/backend/utils/downsample.py
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
"""Generic numeric downsampling utility."""
|
||||
|
||||
|
||||
def downsample(values: list[float], target_count: int) -> list[float]:
|
||||
"""Reduce a list to target_count points via evenly-spaced index sampling."""
|
||||
if len(values) <= target_count:
|
||||
return list(values)
|
||||
if target_count <= 0:
|
||||
return []
|
||||
if target_count == 1:
|
||||
return [values[-1]]
|
||||
indices = [
|
||||
round(i * (len(values) - 1) / (target_count - 1)) for i in range(target_count)
|
||||
]
|
||||
return [values[i] for i in indices]
|
||||
|
|
@ -16,6 +16,7 @@ from .storage_roots import (
|
|||
exports_root,
|
||||
auth_root,
|
||||
auth_db_path,
|
||||
studio_db_path,
|
||||
tmp_root,
|
||||
seed_uploads_root,
|
||||
unstructured_seed_cache_root,
|
||||
|
|
@ -45,6 +46,7 @@ __all__ = [
|
|||
"exports_root",
|
||||
"auth_root",
|
||||
"auth_db_path",
|
||||
"studio_db_path",
|
||||
"tmp_root",
|
||||
"seed_uploads_root",
|
||||
"unstructured_seed_cache_root",
|
||||
|
|
|
|||
|
|
@ -49,6 +49,10 @@ def auth_db_path() -> Path:
|
|||
return auth_root() / "auth.db"
|
||||
|
||||
|
||||
def studio_db_path() -> Path:
|
||||
return studio_root() / "studio.db"
|
||||
|
||||
|
||||
def tmp_root() -> Path:
|
||||
return Path(tempfile.gettempdir()) / "unsloth-studio"
|
||||
|
||||
|
|
|
|||
168
studio/frontend/src/features/studio/historical-training-view.tsx
Normal file
168
studio/frontend/src/features/studio/historical-training-view.tsx
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import type { TrainingViewData } from "@/features/training";
|
||||
import { getTrainingRun } from "@/features/training";
|
||||
import type { TrainingRunDetailResponse } from "@/features/training";
|
||||
import { type ReactElement, useEffect, useState } from "react";
|
||||
import { ChartsSection } from "./sections/charts-section";
|
||||
import { ProgressSection } from "./sections/progress-section";
|
||||
|
||||
interface HistoricalTrainingViewProps {
|
||||
runId: string;
|
||||
}
|
||||
|
||||
function normalizeTrainingMethod(config: Record<string, unknown>): string {
|
||||
const type = config?.training_type as string | undefined;
|
||||
if (!type || type === "Full Finetuning") return "full";
|
||||
if (type === "LoRA/QLoRA") {
|
||||
return config?.load_in_4bit ? "qlora" : "lora";
|
||||
}
|
||||
return "full";
|
||||
}
|
||||
|
||||
function mapToViewData(detail: TrainingRunDetailResponse): TrainingViewData {
|
||||
const { run, metrics } = detail;
|
||||
|
||||
const lossHistory = metrics.loss_step_history
|
||||
.map((step, i) => ({ step, value: metrics.loss_history[i] }))
|
||||
.filter((p): p is { step: number; value: number } => p.value != null);
|
||||
|
||||
const lrHistory = metrics.lr_step_history
|
||||
.map((step, i) => ({ step, value: metrics.lr_history[i] }))
|
||||
.filter((p): p is { step: number; value: number } => p.value != null);
|
||||
|
||||
const gradNormHistory = metrics.grad_norm_step_history
|
||||
.map((step, i) => ({ step, value: metrics.grad_norm_history[i] }))
|
||||
.filter((p): p is { step: number; value: number } => p.value != null);
|
||||
|
||||
const evalLossHistory = metrics.eval_step_history
|
||||
.map((step, i) => ({ step, value: metrics.eval_loss_history[i] }))
|
||||
.filter((p): p is { step: number; value: number } => p.value != null);
|
||||
|
||||
const phase =
|
||||
run.status === "completed"
|
||||
? "completed"
|
||||
: run.status === "stopped"
|
||||
? "stopped"
|
||||
: run.status === "error"
|
||||
? "error"
|
||||
: run.status === "running"
|
||||
? "training"
|
||||
: "idle";
|
||||
|
||||
return {
|
||||
phase,
|
||||
currentStep: run.final_step ?? 0,
|
||||
totalSteps: run.total_steps ?? 0,
|
||||
currentLoss: run.final_loss,
|
||||
currentLearningRate: metrics.lr_history.at(-1) ?? null,
|
||||
currentGradNorm: metrics.grad_norm_history.at(-1) ?? null,
|
||||
currentEpoch: metrics.final_epoch,
|
||||
currentNumTokens: metrics.final_num_tokens ?? null,
|
||||
progressPercent:
|
||||
run.total_steps && run.final_step
|
||||
? (run.final_step / run.total_steps) * 100
|
||||
: 0,
|
||||
elapsedSeconds: run.duration_seconds,
|
||||
etaSeconds: null,
|
||||
evalEnabled: evalLossHistory.length > 0,
|
||||
message:
|
||||
run.status === "completed"
|
||||
? "Training completed"
|
||||
: run.status === "stopped"
|
||||
? "Training stopped"
|
||||
: run.status === "running"
|
||||
? "Training in progress"
|
||||
: run.error_message ?? "Training errored",
|
||||
error: run.status === "error" ? run.error_message : null,
|
||||
isTrainingRunning: false,
|
||||
modelName: run.model_name,
|
||||
trainingMethod: normalizeTrainingMethod(detail.config),
|
||||
lossHistory,
|
||||
lrHistory,
|
||||
gradNormHistory,
|
||||
evalLossHistory,
|
||||
};
|
||||
}
|
||||
|
||||
export function HistoricalTrainingView({
|
||||
runId,
|
||||
}: HistoricalTrainingViewProps): ReactElement {
|
||||
const [detail, setDetail] = useState<TrainingRunDetailResponse | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
// Derive loading from detail/error -- no separate state needed
|
||||
const loading = detail === null && error === null;
|
||||
|
||||
useEffect(() => {
|
||||
const controller = new AbortController();
|
||||
getTrainingRun(runId, controller.signal)
|
||||
.then((result) => {
|
||||
setDetail(result);
|
||||
})
|
||||
.catch((err) => {
|
||||
if (err instanceof DOMException && err.name === "AbortError") return;
|
||||
setError(err instanceof Error ? err.message : "Failed to load run");
|
||||
});
|
||||
return () => {
|
||||
controller.abort();
|
||||
// Reset on runId change so loading derives correctly for the next fetch
|
||||
setDetail(null);
|
||||
setError(null);
|
||||
};
|
||||
}, [runId]);
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<div className="rounded-xl border bg-card p-8 text-sm text-muted-foreground">
|
||||
Loading training run...
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (error || !detail) {
|
||||
return (
|
||||
<div className="rounded-xl border border-destructive/30 bg-destructive/5 p-8 text-sm text-red-500">
|
||||
{error ?? "Run not found"}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const viewData = mapToViewData(detail);
|
||||
const configOverride = detail.config
|
||||
? {
|
||||
epochs: detail.config.num_epochs as number | undefined,
|
||||
batchSize: detail.config.batch_size as number | undefined,
|
||||
learningRate: detail.config.learning_rate as string | undefined,
|
||||
maxSteps: detail.config.max_steps as number | undefined,
|
||||
contextLength: detail.config.max_seq_length as number | undefined,
|
||||
warmupSteps: detail.config.warmup_steps as number | undefined,
|
||||
optimizerType: detail.config.optim as string | undefined,
|
||||
loraRank: detail.config.lora_r as number | undefined,
|
||||
loraAlpha: detail.config.lora_alpha as number | undefined,
|
||||
loraDropout: detail.config.lora_dropout as number | undefined,
|
||||
loraVariant: detail.config.use_rslora ? "rsLoRA" : undefined,
|
||||
}
|
||||
: undefined;
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-6">
|
||||
<ProgressSection
|
||||
data={viewData}
|
||||
isHistorical
|
||||
configOverride={configOverride}
|
||||
/>
|
||||
<ChartsSection
|
||||
currentStep={viewData.currentStep}
|
||||
totalSteps={viewData.totalSteps}
|
||||
isTraining={false}
|
||||
evalEnabled={viewData.evalEnabled}
|
||||
lossHistory={viewData.lossHistory}
|
||||
lrHistory={viewData.lrHistory}
|
||||
gradNormHistory={viewData.gradNormHistory}
|
||||
evalLossHistory={viewData.evalLossHistory}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
401
studio/frontend/src/features/studio/history-card-grid.tsx
Normal file
401
studio/frontend/src/features/studio/history-card-grid.tsx
Normal file
|
|
@ -0,0 +1,401 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import {
|
||||
AlertDialog,
|
||||
AlertDialogAction,
|
||||
AlertDialogCancel,
|
||||
AlertDialogContent,
|
||||
AlertDialogDescription,
|
||||
AlertDialogFooter,
|
||||
AlertDialogHeader,
|
||||
AlertDialogTitle,
|
||||
} from "@/components/ui/alert-dialog";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import type { TrainingRunSummary } from "@/features/training";
|
||||
import { deleteTrainingRun, listTrainingRuns } from "@/features/training";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { Delete02Icon } from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { type ReactElement, useCallback, useEffect, useRef, useState } from "react";
|
||||
import { Spinner } from "@/components/ui/spinner";
|
||||
|
||||
const PAGE_SIZE = 12;
|
||||
const RUNNING_POLL_INTERVAL_MS = 5000;
|
||||
|
||||
const statusBadge: Record<
|
||||
string,
|
||||
{ label: string; className: string }
|
||||
> = {
|
||||
completed: {
|
||||
label: "Completed",
|
||||
className:
|
||||
"bg-emerald-100 text-emerald-700 dark:bg-emerald-950 dark:text-emerald-400",
|
||||
},
|
||||
stopped: {
|
||||
label: "Stopped",
|
||||
className:
|
||||
"bg-amber-100 text-amber-700 dark:bg-amber-950 dark:text-amber-400",
|
||||
},
|
||||
error: {
|
||||
label: "Error",
|
||||
className: "bg-red-100 text-red-700 dark:bg-red-950 dark:text-red-400",
|
||||
},
|
||||
running: {
|
||||
label: "Running",
|
||||
className:
|
||||
"bg-blue-100 text-blue-700 dark:bg-blue-950 dark:text-blue-400",
|
||||
},
|
||||
};
|
||||
|
||||
function catmullRomPath(points: { x: number; y: number }[]): string {
|
||||
if (points.length < 2) return "";
|
||||
const d = [`M${points[0]!.x.toFixed(1)},${points[0]!.y.toFixed(1)}`];
|
||||
for (let i = 0; i < points.length - 1; i++) {
|
||||
const p0 = points[Math.max(i - 1, 0)]!;
|
||||
const p1 = points[i]!;
|
||||
const p2 = points[i + 1]!;
|
||||
const p3 = points[Math.min(i + 2, points.length - 1)]!;
|
||||
const cp1x = p1.x + (p2.x - p0.x) / 6;
|
||||
const cp1y = p1.y + (p2.y - p0.y) / 6;
|
||||
const cp2x = p2.x - (p3.x - p1.x) / 6;
|
||||
const cp2y = p2.y - (p3.y - p1.y) / 6;
|
||||
d.push(
|
||||
`C${cp1x.toFixed(1)},${cp1y.toFixed(1)} ${cp2x.toFixed(1)},${cp2y.toFixed(1)} ${p2.x.toFixed(1)},${p2.y.toFixed(1)}`,
|
||||
);
|
||||
}
|
||||
return d.join(" ");
|
||||
}
|
||||
|
||||
function Sparkline({ values, id }: { values: number[]; id: string }): ReactElement | null {
|
||||
if (!values || values.length < 2) return null;
|
||||
let min = values[0]!;
|
||||
let max = values[0]!;
|
||||
for (let i = 1; i < values.length; i++) {
|
||||
if (values[i]! < min) min = values[i]!;
|
||||
if (values[i]! > max) max = values[i]!;
|
||||
}
|
||||
const range = max - min || 1;
|
||||
const pad = 1.5; // half stroke-width so peaks aren't clipped
|
||||
const h = 32;
|
||||
const w = 120;
|
||||
const gradientId = `sparkFill-${id}`;
|
||||
|
||||
// Build points with vertical padding so the stroke isn't clipped
|
||||
const pts = values.map((v, i) => ({
|
||||
x: (i / (values.length - 1)) * w,
|
||||
y: pad + (1 - (v - min) / range) * (h - pad * 2),
|
||||
}));
|
||||
|
||||
const linePath = catmullRomPath(pts);
|
||||
const last = pts[pts.length - 1]!;
|
||||
const first = pts[0]!;
|
||||
const fillPath = `${linePath} L${last.x.toFixed(1)},${h} L${first.x.toFixed(1)},${h} Z`;
|
||||
|
||||
return (
|
||||
<svg viewBox={`0 0 ${w} ${h}`} className="h-8 w-full" preserveAspectRatio="none" role="img" aria-label="Loss trend sparkline">
|
||||
<defs>
|
||||
<linearGradient id={gradientId} x1="0" y1="0" x2="0" y2="1">
|
||||
<stop offset="0%" stopColor="currentColor" stopOpacity="0.12" />
|
||||
<stop offset="100%" stopColor="currentColor" stopOpacity="0" />
|
||||
</linearGradient>
|
||||
</defs>
|
||||
<path
|
||||
d={fillPath}
|
||||
fill={`url(#${gradientId})`}
|
||||
className="text-emerald-500"
|
||||
/>
|
||||
<path
|
||||
d={linePath}
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="1.5"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
className="text-emerald-500"
|
||||
/>
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
|
||||
function formatRelativeTime(isoDate: string): string {
|
||||
const diff = Date.now() - new Date(isoDate).getTime();
|
||||
const mins = Math.floor(diff / 60000);
|
||||
if (mins < 1) return "just now";
|
||||
if (mins < 60) return `${mins}m ago`;
|
||||
const hrs = Math.floor(mins / 60);
|
||||
if (hrs < 24) return `${hrs}h ago`;
|
||||
const days = Math.floor(hrs / 24);
|
||||
return `${days}d ago`;
|
||||
}
|
||||
|
||||
function formatDuration(seconds: number | null): string {
|
||||
if (seconds == null) return "--";
|
||||
const total = Math.floor(seconds);
|
||||
if (total < 60) return `${total}s`;
|
||||
const min = Math.floor(total / 60);
|
||||
const sec = total % 60;
|
||||
if (min < 60) return `${min}m ${sec}s`;
|
||||
const hrs = Math.floor(min / 60);
|
||||
return `${hrs}h ${min % 60}m`;
|
||||
}
|
||||
|
||||
interface HistoryCardGridProps {
|
||||
onSelectRun: (runId: string) => void;
|
||||
}
|
||||
|
||||
export function HistoryCardGrid({
|
||||
onSelectRun,
|
||||
}: HistoryCardGridProps): ReactElement {
|
||||
const [runs, setRuns] = useState<TrainingRunSummary[]>([]);
|
||||
const [total, setTotal] = useState(0);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [deleteError, setDeleteError] = useState<string | null>(null);
|
||||
const [deleteTarget, setDeleteTarget] = useState<string | null>(null);
|
||||
const [manualFetchInFlight, setManualFetchInFlight] = useState(false);
|
||||
|
||||
const userControllerRef = useRef<AbortController | null>(null);
|
||||
const pollControllerRef = useRef<AbortController | null>(null);
|
||||
const fetchIdRef = useRef(0);
|
||||
const pollIdRef = useRef(0);
|
||||
|
||||
const fetchRuns = useCallback(async (offset = 0, append = false, limit = PAGE_SIZE) => {
|
||||
// Cancel any in-flight poll so its stale response can't clobber this fresher fetch
|
||||
pollControllerRef.current?.abort();
|
||||
userControllerRef.current?.abort();
|
||||
const controller = new AbortController();
|
||||
userControllerRef.current = controller;
|
||||
const id = ++fetchIdRef.current;
|
||||
|
||||
setManualFetchInFlight(true);
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
try {
|
||||
const result = await listTrainingRuns(limit, offset, controller.signal);
|
||||
if (fetchIdRef.current !== id) return;
|
||||
setRuns((prev) => (append ? [...prev, ...result.runs] : result.runs));
|
||||
setTotal(result.total);
|
||||
} catch (err) {
|
||||
if (err instanceof DOMException && err.name === "AbortError") return;
|
||||
if (fetchIdRef.current !== id) return;
|
||||
if (!append) setError("Failed to load training runs");
|
||||
} finally {
|
||||
if (fetchIdRef.current === id) {
|
||||
setLoading(false);
|
||||
setManualFetchInFlight(false);
|
||||
}
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
void fetchRuns(0);
|
||||
return () => {
|
||||
userControllerRef.current?.abort();
|
||||
};
|
||||
}, [fetchRuns]);
|
||||
|
||||
// Poll while any run is still "running" so the card shows live progress
|
||||
const hasRunningRun = runs.some((r) => r.status === "running");
|
||||
const visibleCount = runs.length;
|
||||
useEffect(() => {
|
||||
if (!hasRunningRun) return;
|
||||
const timer = setInterval(async () => {
|
||||
if (manualFetchInFlight) return;
|
||||
pollControllerRef.current?.abort();
|
||||
const controller = new AbortController();
|
||||
pollControllerRef.current = controller;
|
||||
const pid = ++pollIdRef.current;
|
||||
try {
|
||||
const limit = Math.max(PAGE_SIZE, visibleCount);
|
||||
const result = await listTrainingRuns(limit, 0, controller.signal);
|
||||
if (pollIdRef.current !== pid) return; // stale poll — discard
|
||||
setRuns(result.runs);
|
||||
setTotal(result.total);
|
||||
} catch {
|
||||
// silently handle — poll will retry
|
||||
}
|
||||
}, RUNNING_POLL_INTERVAL_MS);
|
||||
return () => {
|
||||
clearInterval(timer);
|
||||
pollControllerRef.current?.abort();
|
||||
};
|
||||
}, [hasRunningRun, visibleCount, manualFetchInFlight]);
|
||||
|
||||
const handleDelete = async () => {
|
||||
if (!deleteTarget) return;
|
||||
setDeleteError(null);
|
||||
try {
|
||||
await deleteTrainingRun(deleteTarget);
|
||||
// Optimistically remove the card so it disappears immediately
|
||||
setRuns((prev) => prev.filter((r) => r.id !== deleteTarget));
|
||||
setTotal((prev) => Math.max(0, prev - 1));
|
||||
// Re-fetch preserving visible count so offsets stay consistent for "Load more"
|
||||
const currentCount = runs.length - 1;
|
||||
const limit = Math.max(PAGE_SIZE, currentCount);
|
||||
fetchRuns(0, false, limit).catch(() => {
|
||||
// Refresh failed — card is already removed, no stale display
|
||||
});
|
||||
} catch {
|
||||
setDeleteError("Failed to delete training run. Please try again.");
|
||||
}
|
||||
setDeleteTarget(null);
|
||||
};
|
||||
|
||||
if (!loading && error && runs.length === 0) {
|
||||
return (
|
||||
<div className="flex flex-col items-center gap-2 py-16 text-center">
|
||||
<p className="text-sm text-destructive">{error}</p>
|
||||
<Button variant="outline" size="sm" onClick={() => void fetchRuns(0)}>
|
||||
Retry
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (!loading && runs.length === 0) {
|
||||
return (
|
||||
<div className="flex flex-col items-center gap-2 py-16 text-center">
|
||||
<p className="text-sm text-muted-foreground">
|
||||
No training runs yet. Start your first training run in the Configure
|
||||
tab.
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
{deleteError && (
|
||||
<div className="mb-4 rounded-lg border border-destructive/50 bg-destructive/10 px-4 py-2 text-sm text-destructive">
|
||||
{deleteError}
|
||||
</div>
|
||||
)}
|
||||
<div className="grid grid-cols-1 gap-4 sm:grid-cols-2 xl:grid-cols-3">
|
||||
{runs.map((run) => {
|
||||
const badge = statusBadge[run.status] ?? statusBadge.error;
|
||||
const isRunning = run.status === "running";
|
||||
return (
|
||||
<div
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
key={run.id}
|
||||
className={cn(
|
||||
"group relative flex cursor-pointer flex-col gap-3 rounded-xl border bg-card p-4 text-left transition-colors hover:border-border hover:bg-accent/30",
|
||||
isRunning
|
||||
? "border-blue-400/50 dark:border-blue-500/30"
|
||||
: "border-border/60",
|
||||
)}
|
||||
onClick={() => onSelectRun(run.id)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === "Enter" || e.key === " ") {
|
||||
e.preventDefault();
|
||||
onSelectRun(run.id);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<div className="flex items-center justify-between pr-6">
|
||||
<span
|
||||
className={cn(
|
||||
"inline-flex items-center gap-1.5 rounded-full px-2 py-0.5 text-[10px] font-semibold",
|
||||
badge.className,
|
||||
)}
|
||||
>
|
||||
{isRunning && <Spinner className="size-2.5" />}
|
||||
{badge.label}
|
||||
</span>
|
||||
<span className="text-[10px] text-muted-foreground">
|
||||
{formatRelativeTime(run.started_at)}
|
||||
</span>
|
||||
</div>
|
||||
<div className="min-w-0">
|
||||
<p
|
||||
className="truncate text-sm font-medium"
|
||||
title={run.model_name}
|
||||
>
|
||||
{run.model_name}
|
||||
</p>
|
||||
<p className="truncate text-xs text-muted-foreground">
|
||||
{run.dataset_name}
|
||||
</p>
|
||||
</div>
|
||||
{run.loss_sparkline && run.loss_sparkline.length >= 2 && (
|
||||
<Sparkline values={run.loss_sparkline} id={run.id} />
|
||||
)}
|
||||
<div className="flex flex-wrap gap-x-4 gap-y-1 text-[11px] text-muted-foreground">
|
||||
<span>
|
||||
Loss:{" "}
|
||||
{run.final_loss != null ? run.final_loss.toFixed(4) : "--"}
|
||||
</span>
|
||||
<span>
|
||||
Steps: {run.final_step ?? 0}/{run.total_steps ?? "--"}
|
||||
</span>
|
||||
<span>{formatDuration(run.duration_seconds)}</span>
|
||||
</div>
|
||||
{!isRunning && (
|
||||
<button
|
||||
type="button"
|
||||
className="absolute right-3 top-3 rounded-md p-1 text-muted-foreground/50 opacity-0 transition-opacity hover:bg-destructive/10 hover:text-destructive group-hover:opacity-100 focus-visible:opacity-100"
|
||||
aria-label="Delete run"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
setDeleteTarget(run.id);
|
||||
}}
|
||||
>
|
||||
<HugeiconsIcon icon={Delete02Icon} className="size-3.5" />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
{runs.length < total && (
|
||||
<div className="mt-4 flex justify-center">
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => void fetchRuns(runs.length, true)}
|
||||
disabled={loading}
|
||||
>
|
||||
{loading ? "Loading..." : "Load more"}
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
{loading && runs.length === 0 && (
|
||||
<div className="grid grid-cols-1 gap-4 sm:grid-cols-2 xl:grid-cols-3">
|
||||
{Array.from({ length: 3 }).map((_, i) => (
|
||||
<div
|
||||
key={`skeleton-${i}`}
|
||||
className="h-40 animate-pulse rounded-xl border bg-muted/30"
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
<AlertDialog
|
||||
open={deleteTarget !== null}
|
||||
onOpenChange={(open) => {
|
||||
if (!open) setDeleteTarget(null);
|
||||
}}
|
||||
>
|
||||
<AlertDialogContent>
|
||||
<AlertDialogHeader>
|
||||
<AlertDialogTitle>Delete training run?</AlertDialogTitle>
|
||||
<AlertDialogDescription>
|
||||
This will permanently delete this training run and all its metrics.
|
||||
This action cannot be undone.
|
||||
</AlertDialogDescription>
|
||||
</AlertDialogHeader>
|
||||
<AlertDialogFooter>
|
||||
<AlertDialogCancel>Cancel</AlertDialogCancel>
|
||||
<AlertDialogAction
|
||||
variant="destructive"
|
||||
onClick={() => void handleDelete()}
|
||||
>
|
||||
Delete
|
||||
</AlertDialogAction>
|
||||
</AlertDialogFooter>
|
||||
</AlertDialogContent>
|
||||
</AlertDialog>
|
||||
</>
|
||||
);
|
||||
}
|
||||
118
studio/frontend/src/features/studio/live-training-view.tsx
Normal file
118
studio/frontend/src/features/studio/live-training-view.tsx
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { cn } from "@/lib/utils";
|
||||
import {
|
||||
useTrainingConfigStore,
|
||||
useTrainingRuntimeStore,
|
||||
} from "@/features/training";
|
||||
import type { TrainingViewData } from "@/features/training";
|
||||
import type { ReactElement } from "react";
|
||||
import { useShallow } from "zustand/react/shallow";
|
||||
import { ChartsSection } from "./sections/charts-section";
|
||||
import { ProgressSection } from "./sections/progress-section";
|
||||
import { TrainingStartOverlay } from "./training-start-overlay";
|
||||
|
||||
export function LiveTrainingView(): ReactElement {
|
||||
const runtime = useTrainingRuntimeStore(
|
||||
useShallow((state) => ({
|
||||
jobId: state.jobId,
|
||||
phase: state.phase,
|
||||
message: state.message,
|
||||
error: state.error,
|
||||
currentStep: state.currentStep,
|
||||
totalSteps: state.totalSteps,
|
||||
currentEpoch: state.currentEpoch,
|
||||
currentLoss: state.currentLoss,
|
||||
currentLearningRate: state.currentLearningRate,
|
||||
currentGradNorm: state.currentGradNorm,
|
||||
currentNumTokens: state.currentNumTokens,
|
||||
progressPercent: state.progressPercent,
|
||||
elapsedSeconds: state.elapsedSeconds,
|
||||
etaSeconds: state.etaSeconds,
|
||||
evalEnabled: state.evalEnabled,
|
||||
isTrainingRunning: state.isTrainingRunning,
|
||||
lossHistory: state.lossHistory,
|
||||
lrHistory: state.lrHistory,
|
||||
gradNormHistory: state.gradNormHistory,
|
||||
evalLossHistory: state.evalLossHistory,
|
||||
firstStepReceived: state.firstStepReceived,
|
||||
isStarting: state.isStarting,
|
||||
})),
|
||||
);
|
||||
|
||||
const config = useTrainingConfigStore(
|
||||
useShallow((state) => ({
|
||||
selectedModel: state.selectedModel,
|
||||
trainingMethod: state.trainingMethod,
|
||||
})),
|
||||
);
|
||||
|
||||
const viewData: TrainingViewData = {
|
||||
phase: runtime.phase,
|
||||
currentStep: runtime.currentStep,
|
||||
totalSteps: runtime.totalSteps,
|
||||
currentLoss: runtime.currentLoss,
|
||||
currentLearningRate: runtime.currentLearningRate,
|
||||
currentGradNorm: runtime.currentGradNorm,
|
||||
currentEpoch: runtime.currentEpoch,
|
||||
currentNumTokens: runtime.currentNumTokens,
|
||||
progressPercent: runtime.progressPercent,
|
||||
elapsedSeconds: runtime.elapsedSeconds,
|
||||
etaSeconds: runtime.etaSeconds,
|
||||
evalEnabled: runtime.evalEnabled,
|
||||
message: runtime.message,
|
||||
error: runtime.error,
|
||||
isTrainingRunning: runtime.isTrainingRunning,
|
||||
modelName: config.selectedModel ?? "",
|
||||
trainingMethod: config.trainingMethod ?? "",
|
||||
lossHistory: runtime.lossHistory,
|
||||
lrHistory: runtime.lrHistory,
|
||||
gradNormHistory: runtime.gradNormHistory,
|
||||
evalLossHistory: runtime.evalLossHistory,
|
||||
};
|
||||
|
||||
const isPreparingPhase =
|
||||
runtime.phase === "downloading_model" ||
|
||||
runtime.phase === "downloading_dataset" ||
|
||||
runtime.phase === "loading_model" ||
|
||||
runtime.phase === "loading_dataset" ||
|
||||
runtime.phase === "configuring";
|
||||
const isWaitingForFirstStep =
|
||||
runtime.phase === "training" && !runtime.firstStepReceived;
|
||||
const showOverlay =
|
||||
runtime.isStarting ||
|
||||
isPreparingPhase ||
|
||||
(isWaitingForFirstStep && runtime.currentStep <= 0);
|
||||
|
||||
return (
|
||||
<div className={cn("relative", showOverlay && "min-h-[72vh]")}>
|
||||
<div
|
||||
className={cn(
|
||||
"relative z-10 flex flex-col gap-6 transition-[filter]",
|
||||
showOverlay && "blur",
|
||||
)}
|
||||
>
|
||||
<div data-tour="studio-training-progress">
|
||||
<ProgressSection key={runtime.jobId ?? "no-job"} data={viewData} />
|
||||
</div>
|
||||
<ChartsSection
|
||||
currentStep={viewData.currentStep}
|
||||
totalSteps={viewData.totalSteps}
|
||||
isTraining={viewData.isTrainingRunning}
|
||||
evalEnabled={viewData.evalEnabled}
|
||||
lossHistory={viewData.lossHistory}
|
||||
lrHistory={viewData.lrHistory}
|
||||
gradNormHistory={viewData.gradNormHistory}
|
||||
evalLossHistory={viewData.evalLossHistory}
|
||||
/>
|
||||
</div>
|
||||
{showOverlay ? (
|
||||
<TrainingStartOverlay
|
||||
message={runtime.message}
|
||||
currentStep={runtime.currentStep}
|
||||
/>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { useTrainingRuntimeStore } from "@/features/training";
|
||||
import type { TrainingSeriesPoint } from "@/features/training";
|
||||
import { type ReactElement, Suspense, lazy, useMemo } from "react";
|
||||
|
||||
const ChartsContent = lazy(() =>
|
||||
|
|
@ -16,42 +16,49 @@ const SKELETON_KEYS = [
|
|||
"chart-skeleton-4",
|
||||
];
|
||||
|
||||
export function ChartsSection(): ReactElement | null {
|
||||
const currentStep = useTrainingRuntimeStore((state) => state.currentStep);
|
||||
const totalSteps = useTrainingRuntimeStore((state) => state.totalSteps);
|
||||
const isTraining = useTrainingRuntimeStore((state) => state.isTrainingRunning);
|
||||
const evalEnabled = useTrainingRuntimeStore((state) => state.evalEnabled);
|
||||
const lossHistoryRaw = useTrainingRuntimeStore((state) => state.lossHistory);
|
||||
const lrHistoryRaw = useTrainingRuntimeStore((state) => state.lrHistory);
|
||||
const gradNormHistoryRaw = useTrainingRuntimeStore(
|
||||
(state) => state.gradNormHistory,
|
||||
);
|
||||
const evalLossHistoryRaw = useTrainingRuntimeStore(
|
||||
(state) => state.evalLossHistory,
|
||||
);
|
||||
interface ChartsSectionProps {
|
||||
currentStep: number;
|
||||
totalSteps: number;
|
||||
isTraining: boolean;
|
||||
evalEnabled: boolean;
|
||||
lossHistory: TrainingSeriesPoint[];
|
||||
lrHistory: TrainingSeriesPoint[];
|
||||
gradNormHistory: TrainingSeriesPoint[];
|
||||
evalLossHistory: TrainingSeriesPoint[];
|
||||
}
|
||||
|
||||
export function ChartsSection({
|
||||
currentStep,
|
||||
totalSteps,
|
||||
isTraining,
|
||||
evalEnabled,
|
||||
lossHistory,
|
||||
lrHistory,
|
||||
gradNormHistory,
|
||||
evalLossHistory,
|
||||
}: ChartsSectionProps): ReactElement | null {
|
||||
const series = useMemo(
|
||||
() => ({
|
||||
currentStep,
|
||||
totalSteps,
|
||||
lossHistory: lossHistoryRaw.map((point) => ({
|
||||
lossHistory: lossHistory.map((point) => ({
|
||||
step: point.step,
|
||||
loss: point.value,
|
||||
})),
|
||||
lrHistory: lrHistoryRaw.map((point) => ({
|
||||
lrHistory: lrHistory.map((point) => ({
|
||||
step: point.step,
|
||||
lr: point.value,
|
||||
})),
|
||||
gradNormHistory: gradNormHistoryRaw.map((point) => ({
|
||||
gradNormHistory: gradNormHistory.map((point) => ({
|
||||
step: point.step,
|
||||
gradNorm: point.value,
|
||||
})),
|
||||
evalLossHistory: evalLossHistoryRaw.map((point) => ({
|
||||
evalLossHistory: evalLossHistory.map((point) => ({
|
||||
step: point.step,
|
||||
loss: point.value,
|
||||
})),
|
||||
}),
|
||||
[currentStep, evalLossHistoryRaw, gradNormHistoryRaw, lossHistoryRaw, lrHistoryRaw, totalSteps],
|
||||
[currentStep, evalLossHistory, gradNormHistory, lossHistory, lrHistory, totalSteps],
|
||||
);
|
||||
|
||||
if (
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import {
|
|||
useTrainingConfigStore,
|
||||
useTrainingRuntimeStore,
|
||||
} from "@/features/training";
|
||||
import type { TrainingViewData } from "@/features/training";
|
||||
import { useGpuUtilization } from "@/hooks";
|
||||
import { cn } from "@/lib/utils";
|
||||
import {
|
||||
|
|
@ -39,7 +40,7 @@ import {
|
|||
} from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { Link, useNavigate } from "@tanstack/react-router";
|
||||
import { type ReactElement, type ReactNode, useEffect, useState } from "react";
|
||||
import { type ReactElement, type ReactNode, useState } from "react";
|
||||
import { useShallow } from "zustand/react/shallow";
|
||||
import { ChartSettingsSheet } from "./charts/chart-settings-sheet";
|
||||
import {
|
||||
|
|
@ -61,34 +62,33 @@ function configRow(
|
|||
return [label, value];
|
||||
}
|
||||
|
||||
export function ProgressSection(): ReactElement {
|
||||
interface ProgressSectionProps {
|
||||
data: TrainingViewData;
|
||||
isHistorical?: boolean;
|
||||
configOverride?: {
|
||||
epochs?: number;
|
||||
batchSize?: number;
|
||||
learningRate?: string;
|
||||
maxSteps?: number;
|
||||
contextLength?: number;
|
||||
warmupSteps?: number;
|
||||
optimizerType?: string;
|
||||
loraRank?: number;
|
||||
loraAlpha?: number;
|
||||
loraDropout?: number;
|
||||
loraVariant?: string;
|
||||
};
|
||||
}
|
||||
|
||||
export function ProgressSection({
|
||||
data,
|
||||
isHistorical = false,
|
||||
configOverride,
|
||||
}: ProgressSectionProps): ReactElement {
|
||||
const navigate = useNavigate();
|
||||
const runtime = useTrainingRuntimeStore(
|
||||
useShallow((state) => ({
|
||||
phase: state.phase,
|
||||
message: state.message,
|
||||
error: state.error,
|
||||
currentStep: state.currentStep,
|
||||
totalSteps: state.totalSteps,
|
||||
currentEpoch: state.currentEpoch,
|
||||
currentLoss: state.currentLoss,
|
||||
currentLearningRate: state.currentLearningRate,
|
||||
currentGradNorm: state.currentGradNorm,
|
||||
progressPercent: state.progressPercent,
|
||||
elapsedSeconds: state.elapsedSeconds,
|
||||
etaSeconds: state.etaSeconds,
|
||||
currentNumTokens: state.currentNumTokens,
|
||||
isTrainingRunning: state.isTrainingRunning,
|
||||
lossHistory: state.lossHistory,
|
||||
lrHistory: state.lrHistory,
|
||||
gradNormHistory: state.gradNormHistory,
|
||||
})),
|
||||
);
|
||||
|
||||
const config = useTrainingConfigStore(
|
||||
useShallow((state) => ({
|
||||
selectedModel: state.selectedModel,
|
||||
trainingMethod: state.trainingMethod,
|
||||
epochs: state.epochs,
|
||||
batchSize: state.batchSize,
|
||||
learningRate: state.learningRate,
|
||||
|
|
@ -103,98 +103,92 @@ export function ProgressSection(): ReactElement {
|
|||
})),
|
||||
);
|
||||
|
||||
const { stopTrainingRun } = useTrainingActions();
|
||||
const gpu = useGpuUtilization(runtime.isTrainingRunning);
|
||||
const [stopDialogOpen, setStopDialogOpen] = useState(false);
|
||||
const [stopRequested, setStopRequested] = useState(false);
|
||||
const [stopRequestedLocal, setStopRequestedLocal] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (!runtime.isTrainingRunning) {
|
||||
setStopRequested(false);
|
||||
}
|
||||
}, [runtime.isTrainingRunning]);
|
||||
// Auto-reset when training stops -- no useEffect needed
|
||||
const stopRequested = data.isTrainingRunning && stopRequestedLocal;
|
||||
|
||||
const pct =
|
||||
runtime.totalSteps > 0
|
||||
data.totalSteps > 0
|
||||
? Math.min(
|
||||
100,
|
||||
Math.max(
|
||||
0,
|
||||
Math.round((runtime.currentStep / runtime.totalSteps) * 100),
|
||||
Math.round((data.currentStep / data.totalSteps) * 100),
|
||||
),
|
||||
)
|
||||
: Math.round(runtime.progressPercent);
|
||||
: Math.round(data.progressPercent);
|
||||
|
||||
const elapsed = runtime.elapsedSeconds;
|
||||
const elapsed = data.elapsedSeconds;
|
||||
const derivedEta =
|
||||
elapsed != null && pct > 0
|
||||
? Math.round((elapsed * (100 - pct)) / Math.max(pct, 1))
|
||||
: null;
|
||||
const eta = runtime.etaSeconds ?? derivedEta;
|
||||
const eta = data.etaSeconds ?? derivedEta;
|
||||
|
||||
const stepsPerSecond =
|
||||
elapsed != null && elapsed > 0 ? runtime.currentStep / elapsed : null;
|
||||
elapsed != null && elapsed > 0 ? data.currentStep / elapsed : null;
|
||||
const showHalfwayHint =
|
||||
runtime.phase === "training" && pct >= 50 && pct < 100;
|
||||
const showCompletedHint = runtime.phase === "completed";
|
||||
data.phase === "training" && pct >= 50 && pct < 100;
|
||||
const showCompletedHint = data.phase === "completed";
|
||||
const handleCompareInChat = async () => {
|
||||
setTrainingCompareHandoff(config.selectedModel);
|
||||
setTrainingCompareHandoff(data.modelName);
|
||||
await navigate({ to: "/chat" });
|
||||
};
|
||||
const requestStop = async (saveCheckpoint: boolean) => {
|
||||
setStopRequested(true);
|
||||
setStopDialogOpen(false);
|
||||
useTrainingRuntimeStore.getState().setStopRequested(true);
|
||||
try {
|
||||
const ok = await stopTrainingRun(saveCheckpoint);
|
||||
if (!ok) {
|
||||
setStopRequested(false);
|
||||
}
|
||||
} catch {
|
||||
setStopRequested(false);
|
||||
}
|
||||
};
|
||||
|
||||
const stoppedLoss = getDisplayMetric(
|
||||
runtime.isTrainingRunning,
|
||||
runtime.currentLoss,
|
||||
runtime.lossHistory,
|
||||
data.isTrainingRunning,
|
||||
data.currentLoss,
|
||||
data.lossHistory,
|
||||
);
|
||||
const stoppedLr = getDisplayMetric(
|
||||
runtime.isTrainingRunning,
|
||||
runtime.currentLearningRate,
|
||||
runtime.lrHistory,
|
||||
data.isTrainingRunning,
|
||||
data.currentLearningRate,
|
||||
data.lrHistory,
|
||||
);
|
||||
const stoppedGradNorm = runtime.isTrainingRunning
|
||||
? runtime.currentGradNorm
|
||||
: (lastNonZeroValue(runtime.gradNormHistory) ?? runtime.currentGradNorm);
|
||||
const stoppedGradNorm = data.isTrainingRunning
|
||||
? data.currentGradNorm
|
||||
: (lastValue(data.gradNormHistory) ?? data.currentGradNorm);
|
||||
|
||||
const cfgEpochs = isHistorical ? configOverride?.epochs : config.epochs;
|
||||
const cfgBatchSize = isHistorical ? configOverride?.batchSize : config.batchSize;
|
||||
const cfgLearningRate = isHistorical ? configOverride?.learningRate : config.learningRate;
|
||||
const cfgMaxSteps = isHistorical ? configOverride?.maxSteps : config.maxSteps;
|
||||
const cfgContextLength = isHistorical ? configOverride?.contextLength : config.contextLength;
|
||||
const cfgWarmupSteps = isHistorical ? configOverride?.warmupSteps : config.warmupSteps;
|
||||
const cfgOptimizerType = isHistorical ? configOverride?.optimizerType : config.optimizerType;
|
||||
const cfgLoraRank = isHistorical ? configOverride?.loraRank : config.loraRank;
|
||||
const cfgLoraAlpha = isHistorical ? configOverride?.loraAlpha : config.loraAlpha;
|
||||
const cfgLoraDropout = isHistorical ? configOverride?.loraDropout : config.loraDropout;
|
||||
const cfgLoraVariant = isHistorical ? configOverride?.loraVariant : config.loraVariant;
|
||||
|
||||
const optimizerLabel =
|
||||
OPTIMIZER_OPTIONS.find((o) => o.value === config.optimizerType)?.label ??
|
||||
config.optimizerType;
|
||||
OPTIMIZER_OPTIONS.find((o) => o.value === cfgOptimizerType)?.label ??
|
||||
cfgOptimizerType;
|
||||
|
||||
const configItems: ConfigGroup[] = [
|
||||
{
|
||||
section: "Hyperparams",
|
||||
rows: [
|
||||
configRow("Epochs", config.epochs),
|
||||
configRow("Batch size", config.batchSize),
|
||||
configRow("Learning rate", config.learningRate),
|
||||
configRow("Epochs", cfgEpochs),
|
||||
configRow("Batch size", cfgBatchSize),
|
||||
configRow("Learning rate", cfgLearningRate),
|
||||
configRow("Optimizer", optimizerLabel),
|
||||
configRow("Max steps", config.maxSteps),
|
||||
configRow("Context length", config.contextLength),
|
||||
configRow("Warmup steps", config.warmupSteps),
|
||||
configRow("Max steps", cfgMaxSteps),
|
||||
configRow("Context length", cfgContextLength),
|
||||
configRow("Warmup steps", cfgWarmupSteps),
|
||||
],
|
||||
},
|
||||
...(config.trainingMethod !== "full"
|
||||
...(data.trainingMethod !== "full"
|
||||
? [
|
||||
{
|
||||
section: "LoRA",
|
||||
rows: [
|
||||
configRow("Rank", config.loraRank),
|
||||
configRow("Alpha", config.loraAlpha),
|
||||
configRow("Dropout", config.loraDropout),
|
||||
configRow("Variant", config.loraVariant),
|
||||
configRow("Rank", cfgLoraRank),
|
||||
configRow("Alpha", cfgLoraAlpha),
|
||||
configRow("Dropout", cfgLoraDropout),
|
||||
configRow("Variant", cfgLoraVariant),
|
||||
],
|
||||
},
|
||||
]
|
||||
|
|
@ -205,30 +199,34 @@ export function ProgressSection(): ReactElement {
|
|||
<SectionCard
|
||||
icon={<HugeiconsIcon icon={ChartAverageIcon} className="size-5" />}
|
||||
title="Training Progress"
|
||||
description={runtime.message || "Live training metrics"}
|
||||
description={data.message || "Live training metrics"}
|
||||
accent="emerald"
|
||||
className="shadow-border border border-border/60 bg-card/90 ring-0 backdrop-blur-sm"
|
||||
headerAction={
|
||||
<TrainingHeaderActions
|
||||
configItems={configItems}
|
||||
isTrainingRunning={runtime.isTrainingRunning}
|
||||
onOpenStopDialog={setStopDialogOpen}
|
||||
onRequestStop={requestStop}
|
||||
stopDialogOpen={stopDialogOpen}
|
||||
stopRequested={stopRequested}
|
||||
/>
|
||||
isHistorical ? (
|
||||
<ConfigPopoverButton configItems={configItems} />
|
||||
) : (
|
||||
<LiveTrainingHeaderActions
|
||||
configItems={configItems}
|
||||
isTrainingRunning={data.isTrainingRunning}
|
||||
onOpenStopDialog={setStopDialogOpen}
|
||||
stopDialogOpen={stopDialogOpen}
|
||||
stopRequested={stopRequested}
|
||||
onSetStopRequested={setStopRequestedLocal}
|
||||
/>
|
||||
)
|
||||
}
|
||||
>
|
||||
<div className="grid grid-cols-1 gap-5 lg:grid-cols-[minmax(0,1.2fr)_minmax(18rem,0.8fr)]">
|
||||
<div className="flex flex-col gap-4">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<span
|
||||
className={`rounded-full px-2.5 py-1 text-[10px] font-semibold ${phaseColors[runtime.phase]}`}
|
||||
className={`rounded-full px-2.5 py-1 text-[10px] font-semibold ${phaseColors[data.phase]}`}
|
||||
>
|
||||
{phaseLabel[runtime.phase]}
|
||||
{phaseLabel[data.phase]}
|
||||
</span>
|
||||
<span className="text-[10px] tabular-nums text-muted-foreground">
|
||||
Epoch {runtime.currentEpoch.toFixed(2)}
|
||||
Epoch {formatNumber(data.currentEpoch, 2)}
|
||||
</span>
|
||||
<span className="rounded-full border border-border/60 px-2.5 py-1 text-[10px] font-medium tabular-nums text-muted-foreground">
|
||||
{pct}% complete
|
||||
|
|
@ -238,22 +236,24 @@ export function ProgressSection(): ReactElement {
|
|||
<div className="flex flex-col gap-2">
|
||||
<div className="flex justify-between text-xs text-muted-foreground">
|
||||
<span>
|
||||
Step {runtime.currentStep} / {runtime.totalSteps || "--"}
|
||||
Step {data.currentStep} / {data.totalSteps || "--"}
|
||||
</span>
|
||||
<span>{pct}%</span>
|
||||
</div>
|
||||
<Progress value={pct} className="h-2 bg-foreground/[0.05]" />
|
||||
</div>
|
||||
|
||||
<MilestoneCallout
|
||||
showCompletedHint={showCompletedHint}
|
||||
showHalfwayHint={showHalfwayHint}
|
||||
onCompareInChat={handleCompareInChat}
|
||||
/>
|
||||
{!isHistorical && (
|
||||
<MilestoneCallout
|
||||
showCompletedHint={showCompletedHint}
|
||||
showHalfwayHint={showHalfwayHint}
|
||||
onCompareInChat={handleCompareInChat}
|
||||
/>
|
||||
)}
|
||||
|
||||
{runtime.error && (
|
||||
{data.error && (
|
||||
<p className="rounded-2xl border border-destructive/30 bg-destructive/5 px-3 py-2 text-xs text-red-500 leading-relaxed">
|
||||
{runtime.error}
|
||||
{data.error}
|
||||
</p>
|
||||
)}
|
||||
|
||||
|
|
@ -262,97 +262,196 @@ export function ProgressSection(): ReactElement {
|
|||
label="Loss"
|
||||
valueClassName="text-2xl font-bold tracking-tight"
|
||||
>
|
||||
{stoppedLoss.toFixed(4)}
|
||||
{stoppedLoss != null ? stoppedLoss.toFixed(4) : "--"}
|
||||
</MetricStat>
|
||||
<MetricStat label="LR">{stoppedLr.toExponential(2)}</MetricStat>
|
||||
<MetricStat label="LR">{stoppedLr != null ? stoppedLr.toExponential(2) : "--"}</MetricStat>
|
||||
<MetricStat label="Grad Norm">
|
||||
{formatNumber(stoppedGradNorm, 3)}
|
||||
</MetricStat>
|
||||
<MetricStat label="Model" valueClassName="truncate">
|
||||
{config.selectedModel ?? "--"}
|
||||
{data.modelName || "--"}
|
||||
</MetricStat>
|
||||
<MetricStat label="Method">
|
||||
{config.trainingMethod === "qlora" ? "QLoRA" : config.trainingMethod === "lora" ? "LoRA" : "Full"}
|
||||
{data.trainingMethod === "qlora" ? "QLoRA" : data.trainingMethod === "lora" ? "LoRA" : "Full"}
|
||||
</MetricStat>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-wrap gap-x-4 gap-y-1 text-xs text-muted-foreground">
|
||||
<span>Elapsed: {formatDuration(elapsed)}</span>
|
||||
<span>ETA: {formatDuration(eta)}</span>
|
||||
{!isHistorical && <span>ETA: {formatDuration(eta)}</span>}
|
||||
<span>
|
||||
{stepsPerSecond == null
|
||||
? "-- steps/s"
|
||||
: `${stepsPerSecond.toFixed(2)} steps/s`}
|
||||
</span>
|
||||
{runtime.currentNumTokens != null && (
|
||||
<span>Tokens: {runtime.currentNumTokens}</span>
|
||||
{data.currentNumTokens != null && (
|
||||
<span>Tokens: {data.currentNumTokens}</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<p className="text-xs font-medium text-muted-foreground">
|
||||
GPU Monitor
|
||||
</p>
|
||||
<span className="text-[11px] text-muted-foreground">Live</span>
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-2.5">
|
||||
<GpuStat
|
||||
label="Utilization"
|
||||
icon={
|
||||
<HugeiconsIcon
|
||||
icon={DashboardSpeed01Icon}
|
||||
className="size-3.5"
|
||||
/>
|
||||
}
|
||||
value={
|
||||
gpu.gpu_utilization_pct != null
|
||||
? `${gpu.gpu_utilization_pct}%`
|
||||
: "--"
|
||||
}
|
||||
pct={gpu.gpu_utilization_pct ?? 0}
|
||||
/>
|
||||
<GpuStat
|
||||
label="Temperature"
|
||||
icon={
|
||||
<HugeiconsIcon icon={TemperatureIcon} className="size-3.5" />
|
||||
}
|
||||
value={
|
||||
gpu.temperature_c != null ? `${gpu.temperature_c}°C` : "--"
|
||||
}
|
||||
pct={gpu.temperature_c ?? 0}
|
||||
max={100}
|
||||
/>
|
||||
<GpuStat
|
||||
label="VRAM"
|
||||
icon={<HugeiconsIcon icon={RamMemoryIcon} className="size-3.5" />}
|
||||
value={
|
||||
gpu.vram_used_gb != null && gpu.vram_total_gb != null
|
||||
? `${gpu.vram_used_gb} / ${gpu.vram_total_gb} GB`
|
||||
: "--"
|
||||
}
|
||||
pct={gpu.vram_utilization_pct ?? 0}
|
||||
/>
|
||||
<GpuStat
|
||||
label="Power"
|
||||
icon={<HugeiconsIcon icon={ZapIcon} className="size-3.5" />}
|
||||
value={
|
||||
gpu.power_draw_w != null
|
||||
? gpu.power_limit_w != null
|
||||
? `${gpu.power_draw_w} / ${gpu.power_limit_w} W`
|
||||
: `${gpu.power_draw_w} W`
|
||||
: "--"
|
||||
}
|
||||
pct={gpu.power_utilization_pct ?? 0}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
{!isHistorical && (
|
||||
<LiveGpuPanel isTrainingRunning={data.isTrainingRunning} />
|
||||
)}
|
||||
</div>
|
||||
</SectionCard>
|
||||
);
|
||||
}
|
||||
|
||||
function LiveGpuPanel({
|
||||
isTrainingRunning,
|
||||
}: {
|
||||
isTrainingRunning: boolean;
|
||||
}): ReactElement {
|
||||
const gpu = useGpuUtilization(isTrainingRunning);
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<p className="text-xs font-medium text-muted-foreground">
|
||||
GPU Monitor
|
||||
</p>
|
||||
<span className="text-[11px] text-muted-foreground">Live</span>
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-2.5">
|
||||
<GpuStat
|
||||
label="Utilization"
|
||||
icon={
|
||||
<HugeiconsIcon
|
||||
icon={DashboardSpeed01Icon}
|
||||
className="size-3.5"
|
||||
/>
|
||||
}
|
||||
value={
|
||||
gpu.gpu_utilization_pct != null
|
||||
? `${gpu.gpu_utilization_pct}%`
|
||||
: "--"
|
||||
}
|
||||
pct={gpu.gpu_utilization_pct ?? 0}
|
||||
/>
|
||||
<GpuStat
|
||||
label="Temperature"
|
||||
icon={
|
||||
<HugeiconsIcon icon={TemperatureIcon} className="size-3.5" />
|
||||
}
|
||||
value={
|
||||
gpu.temperature_c != null ? `${gpu.temperature_c}°C` : "--"
|
||||
}
|
||||
pct={gpu.temperature_c ?? 0}
|
||||
max={100}
|
||||
/>
|
||||
<GpuStat
|
||||
label="VRAM"
|
||||
icon={<HugeiconsIcon icon={RamMemoryIcon} className="size-3.5" />}
|
||||
value={
|
||||
gpu.vram_used_gb != null && gpu.vram_total_gb != null
|
||||
? `${gpu.vram_used_gb} / ${gpu.vram_total_gb} GB`
|
||||
: "--"
|
||||
}
|
||||
pct={gpu.vram_utilization_pct ?? 0}
|
||||
/>
|
||||
<GpuStat
|
||||
label="Power"
|
||||
icon={<HugeiconsIcon icon={ZapIcon} className="size-3.5" />}
|
||||
value={
|
||||
gpu.power_draw_w != null
|
||||
? gpu.power_limit_w != null
|
||||
? `${gpu.power_draw_w} / ${gpu.power_limit_w} W`
|
||||
: `${gpu.power_draw_w} W`
|
||||
: "--"
|
||||
}
|
||||
pct={gpu.power_utilization_pct ?? 0}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function LiveTrainingHeaderActions({
|
||||
configItems,
|
||||
isTrainingRunning,
|
||||
onOpenStopDialog,
|
||||
stopDialogOpen,
|
||||
stopRequested,
|
||||
onSetStopRequested,
|
||||
}: {
|
||||
configItems: ConfigGroup[];
|
||||
isTrainingRunning: boolean;
|
||||
onOpenStopDialog: (open: boolean) => void;
|
||||
stopDialogOpen: boolean;
|
||||
stopRequested: boolean;
|
||||
onSetStopRequested: (v: boolean) => void;
|
||||
}): ReactElement {
|
||||
const { stopTrainingRun } = useTrainingActions();
|
||||
|
||||
const requestStop = async (saveCheckpoint: boolean) => {
|
||||
onSetStopRequested(true);
|
||||
onOpenStopDialog(false);
|
||||
useTrainingRuntimeStore.getState().setStopRequested(true);
|
||||
try {
|
||||
const ok = await stopTrainingRun(saveCheckpoint);
|
||||
if (!ok) {
|
||||
onSetStopRequested(false);
|
||||
}
|
||||
} catch {
|
||||
onSetStopRequested(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<TrainingHeaderActions
|
||||
configItems={configItems}
|
||||
isTrainingRunning={isTrainingRunning}
|
||||
onOpenStopDialog={onOpenStopDialog}
|
||||
onRequestStop={requestStop}
|
||||
stopDialogOpen={stopDialogOpen}
|
||||
stopRequested={stopRequested}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
function ConfigPopoverButton({
|
||||
configItems,
|
||||
}: {
|
||||
configItems: ConfigGroup[];
|
||||
}): ReactElement {
|
||||
return (
|
||||
<Popover>
|
||||
<PopoverTrigger asChild={true}>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
className="rounded-full text-muted-foreground hover:bg-muted hover:text-foreground"
|
||||
aria-label="Open training config"
|
||||
>
|
||||
<HugeiconsIcon icon={Notebook01Icon} className="size-4" />
|
||||
</Button>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent className="w-72" align="end">
|
||||
<div className="flex flex-col gap-3">
|
||||
<p className="text-xs font-semibold">Training Config</p>
|
||||
{configItems.map((group) => (
|
||||
<div key={group.section} className="flex flex-col gap-1">
|
||||
<p className="text-[10px] font-semibold uppercase tracking-wider text-muted-foreground">
|
||||
{group.section}
|
||||
</p>
|
||||
{group.rows.map(([label, value]) => (
|
||||
<div key={label} className="flex justify-between text-xs">
|
||||
<span className="text-muted-foreground">{label}</span>
|
||||
<span className="font-medium tabular-nums">
|
||||
{value == null || value === "" ? "--" : String(value)}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
);
|
||||
}
|
||||
|
||||
function TrainingHeaderActions({
|
||||
configItems,
|
||||
isTrainingRunning,
|
||||
|
|
@ -370,39 +469,7 @@ function TrainingHeaderActions({
|
|||
}): ReactElement {
|
||||
return (
|
||||
<div className="flex items-center gap-2">
|
||||
<Popover>
|
||||
<PopoverTrigger asChild={true}>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
className="rounded-full text-muted-foreground hover:bg-muted hover:text-foreground"
|
||||
aria-label="Open training config"
|
||||
>
|
||||
<HugeiconsIcon icon={Notebook01Icon} className="size-4" />
|
||||
</Button>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent className="w-72" align="end">
|
||||
<div className="flex flex-col gap-3">
|
||||
<p className="text-xs font-semibold">Training Config</p>
|
||||
{configItems.map((group) => (
|
||||
<div key={group.section} className="flex flex-col gap-1">
|
||||
<p className="text-[10px] font-semibold uppercase tracking-wider text-muted-foreground">
|
||||
{group.section}
|
||||
</p>
|
||||
{group.rows.map(([label, value]) => (
|
||||
<div key={label} className="flex justify-between text-xs">
|
||||
<span className="text-muted-foreground">{label}</span>
|
||||
<span className="font-medium tabular-nums">
|
||||
{String(value)}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
<ConfigPopoverButton configItems={configItems} />
|
||||
<ChartSettingsSheet />
|
||||
<AlertDialog open={stopDialogOpen} onOpenChange={onOpenStopDialog}>
|
||||
<Button
|
||||
|
|
@ -518,25 +585,21 @@ function MetricStat({
|
|||
);
|
||||
}
|
||||
|
||||
function lastNonZeroValue(points: { value: number }[]): number | null {
|
||||
for (let i = points.length - 1; i >= 0; i -= 1) {
|
||||
const value = points[i]?.value;
|
||||
if (Number.isFinite(value) && value !== 0) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
function lastValue(points: { value: number }[]): number | null {
|
||||
if (points.length === 0) return null;
|
||||
const v = points[points.length - 1]?.value;
|
||||
return v != null && Number.isFinite(v) ? v : null;
|
||||
}
|
||||
|
||||
function getDisplayMetric(
|
||||
isTrainingRunning: boolean,
|
||||
currentValue: number,
|
||||
currentValue: number | null,
|
||||
history: { value: number }[],
|
||||
): number {
|
||||
): number | null {
|
||||
if (isTrainingRunning) {
|
||||
return currentValue;
|
||||
return currentValue != null ? currentValue : null;
|
||||
}
|
||||
return lastNonZeroValue(history) ?? currentValue;
|
||||
return lastValue(history) ?? (currentValue != null ? currentValue : null);
|
||||
}
|
||||
|
||||
function GpuStat({
|
||||
|
|
|
|||
|
|
@ -1,26 +1,28 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
shouldShowTrainingView,
|
||||
useDatasetPreviewDialogStore,
|
||||
useTrainingActions,
|
||||
useTrainingConfigStore,
|
||||
useTrainingRuntimeLifecycle,
|
||||
useTrainingRuntimeStore,
|
||||
} from "@/features/training";
|
||||
import { GuidedTour, useGuidedTourController } from "@/features/tour";
|
||||
import { studioTourSteps, studioTrainingTourSteps } from "./tour";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { ArrowLeft01Icon } from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { type ReactElement, useEffect } from "react";
|
||||
import { type ReactElement, useEffect, useState } from "react";
|
||||
import { DatasetPreviewDialog } from "./sections/dataset-preview-dialog";
|
||||
import { DatasetSection } from "./sections/dataset-section";
|
||||
import { ModelSection } from "./sections/model-section";
|
||||
import { ParamsSection } from "./sections/params-section";
|
||||
import { TrainingSection } from "./sections/training-section";
|
||||
import { TrainingView } from "./training-view";
|
||||
import { LiveTrainingView } from "./live-training-view";
|
||||
import { HistoricalTrainingView } from "./historical-training-view";
|
||||
import { HistoryCardGrid } from "./history-card-grid";
|
||||
|
||||
const STUDIO_TOUR_KEY = "tour:studio:v1";
|
||||
|
||||
|
|
@ -28,11 +30,10 @@ export function StudioPage(): ReactElement {
|
|||
useTrainingRuntimeLifecycle();
|
||||
const showTrainingView = useTrainingRuntimeStore(shouldShowTrainingView);
|
||||
const isTrainingRunning = useTrainingRuntimeStore((state) => state.isTrainingRunning);
|
||||
const currentJobId = useTrainingRuntimeStore((state) => state.jobId);
|
||||
const runtimeMessage = useTrainingRuntimeStore((state) => state.message);
|
||||
const runtimePhase = useTrainingRuntimeStore((state) => state.phase);
|
||||
const isHydratingRuntime = useTrainingRuntimeStore((state) => state.isHydrating);
|
||||
const hasHydratedRuntime = useTrainingRuntimeStore((state) => state.hasHydrated);
|
||||
const { dismissTrainingRun } = useTrainingActions();
|
||||
|
||||
const config = useTrainingConfigStore();
|
||||
const selectedModel = useTrainingConfigStore((s) => s.selectedModel);
|
||||
|
|
@ -47,19 +48,22 @@ export function StudioPage(): ReactElement {
|
|||
const dialogInitial = useDatasetPreviewDialogStore((s) => s.initialData);
|
||||
const closeDialog = useDatasetPreviewDialogStore((s) => s.close);
|
||||
|
||||
const stopRequested = useTrainingRuntimeStore((state) => state.stopRequested);
|
||||
const canGoBack =
|
||||
showTrainingView &&
|
||||
!isHydratingRuntime &&
|
||||
(stopRequested ||
|
||||
(!isTrainingRunning &&
|
||||
(runtimePhase === "stopped" ||
|
||||
runtimePhase === "error" ||
|
||||
runtimePhase === "completed" ||
|
||||
runtimePhase === "idle")));
|
||||
const [requestedTab, setRequestedTab] = useState("configure");
|
||||
const [selectedHistoryRunId, setSelectedHistoryRunId] = useState<string | null>(null);
|
||||
|
||||
// Derive activeTab: auto-switch to "current-run" only while training is
|
||||
// genuinely running. Once training ends, honour whatever tab the user clicks.
|
||||
// If requestedTab is "current-run" but there's nothing to show, fall back to "configure".
|
||||
const activeTab =
|
||||
isTrainingRunning && requestedTab !== "history"
|
||||
? "current-run"
|
||||
: requestedTab === "current-run" && !showTrainingView
|
||||
? "configure"
|
||||
: requestedTab;
|
||||
|
||||
const tourEnabled = hasHydratedRuntime && !isHydratingRuntime;
|
||||
const isConfigTour = !showTrainingView;
|
||||
const tourSteps = showTrainingView ? studioTrainingTourSteps : studioTourSteps;
|
||||
const isConfigTour = activeTab === "configure";
|
||||
const tourSteps = activeTab === "current-run" ? studioTrainingTourSteps : studioTourSteps;
|
||||
const tour = useGuidedTourController({
|
||||
id: "studio",
|
||||
steps: tourSteps,
|
||||
|
|
@ -71,13 +75,36 @@ export function StudioPage(): ReactElement {
|
|||
const setTourOpen = tour.setOpen;
|
||||
useEffect(() => {
|
||||
setTourOpen(false);
|
||||
}, [showTrainingView, setTourOpen]);
|
||||
}, [activeTab, setTourOpen]);
|
||||
|
||||
// When training auto-switches us to "current-run", persist that in
|
||||
// requestedTab so the user stays on results after training ends.
|
||||
useEffect(() => {
|
||||
if (isTrainingRunning && requestedTab !== "history" && requestedTab !== "current-run") {
|
||||
setRequestedTab("current-run");
|
||||
setSelectedHistoryRunId(null);
|
||||
}
|
||||
}, [isTrainingRunning, requestedTab]);
|
||||
|
||||
useEffect(() => {
|
||||
ensureModelDefaultsLoaded();
|
||||
ensureDatasetChecked();
|
||||
}, [selectedModel, ensureModelDefaultsLoaded, ensureDatasetChecked]);
|
||||
|
||||
function handleTabChange(value: string) {
|
||||
setRequestedTab(value);
|
||||
if (value !== "history") {
|
||||
setSelectedHistoryRunId(null);
|
||||
}
|
||||
}
|
||||
|
||||
const subtitle = (() => {
|
||||
if (activeTab === "current-run") return runtimeMessage || "Training in progress";
|
||||
if (activeTab === "history")
|
||||
return selectedHistoryRunId ? "Viewing past run" : "View past training runs";
|
||||
return "Configure and start training";
|
||||
})();
|
||||
|
||||
return (
|
||||
<div className="relative min-h-screen overflow-hidden bg-background">
|
||||
<main className="relative z-10 mx-auto max-w-7xl px-4 py-4 sm:px-6">
|
||||
|
|
@ -100,42 +127,69 @@ export function StudioPage(): ReactElement {
|
|||
isVlm={config.isVisionModel && config.isDatasetImage === true}
|
||||
/>
|
||||
|
||||
{canGoBack && (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
className="mb-2 cursor-pointer gap-1.5 text-muted-foreground"
|
||||
onClick={() => void dismissTrainingRun()}
|
||||
>
|
||||
<HugeiconsIcon icon={ArrowLeft01Icon} className="size-4" />
|
||||
Back to configuration
|
||||
</Button>
|
||||
)}
|
||||
|
||||
<div className="mb-6 flex flex-col gap-0.5 sm:mb-8">
|
||||
<h1 className="text-2xl font-semibold tracking-tight">
|
||||
Fine-tuning Studio
|
||||
</h1>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{showTrainingView
|
||||
? runtimeMessage || "Training in progress"
|
||||
: "Configure and start training"}
|
||||
</p>
|
||||
<p className="text-sm text-muted-foreground">{subtitle}</p>
|
||||
</div>
|
||||
|
||||
{!hasHydratedRuntime && isHydratingRuntime ? (
|
||||
<div className="rounded-xl border bg-card p-8 text-sm text-muted-foreground">
|
||||
Loading training runtime...
|
||||
</div>
|
||||
) : showTrainingView ? (
|
||||
<TrainingView />
|
||||
) : (
|
||||
<div className="grid grid-cols-1 items-start gap-4 md:grid-cols-2 md:gap-6 xl:grid-cols-12">
|
||||
<ModelSection />
|
||||
<DatasetSection />
|
||||
<ParamsSection />
|
||||
<TrainingSection />
|
||||
</div>
|
||||
<Tabs value={activeTab} onValueChange={handleTabChange}>
|
||||
<div className="flex items-center gap-3">
|
||||
{selectedHistoryRunId && activeTab === "history" && (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
className="rounded-full text-muted-foreground"
|
||||
onClick={() => setSelectedHistoryRunId(null)}
|
||||
aria-label="Back to history"
|
||||
>
|
||||
<HugeiconsIcon icon={ArrowLeft01Icon} className="size-4" />
|
||||
</Button>
|
||||
)}
|
||||
<TabsList variant="line">
|
||||
<TabsTrigger value="configure" disabled={isTrainingRunning}>
|
||||
Configure
|
||||
</TabsTrigger>
|
||||
<TabsTrigger value="current-run" disabled={!showTrainingView}>
|
||||
Current Run
|
||||
</TabsTrigger>
|
||||
<TabsTrigger value="history">History</TabsTrigger>
|
||||
</TabsList>
|
||||
</div>
|
||||
|
||||
<TabsContent value="configure">
|
||||
<div className="grid grid-cols-1 items-start gap-4 md:grid-cols-2 md:gap-6 xl:grid-cols-12">
|
||||
<ModelSection />
|
||||
<DatasetSection />
|
||||
<ParamsSection />
|
||||
<TrainingSection />
|
||||
</div>
|
||||
</TabsContent>
|
||||
|
||||
<TabsContent value="current-run">
|
||||
<LiveTrainingView />
|
||||
</TabsContent>
|
||||
|
||||
<TabsContent value="history">
|
||||
{selectedHistoryRunId ? (
|
||||
<HistoricalTrainingView runId={selectedHistoryRunId} />
|
||||
) : (
|
||||
<HistoryCardGrid onSelectRun={(runId) => {
|
||||
if (runId === currentJobId && isTrainingRunning) {
|
||||
handleTabChange("current-run");
|
||||
} else {
|
||||
setSelectedHistoryRunId(runId);
|
||||
}
|
||||
}} />
|
||||
)}
|
||||
</TabsContent>
|
||||
</Tabs>
|
||||
)}
|
||||
</main>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,57 +0,0 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { cn } from "@/lib/utils";
|
||||
import { useTrainingRuntimeStore } from "@/features/training";
|
||||
import type { ReactElement } from "react";
|
||||
import { useShallow } from "zustand/react/shallow";
|
||||
import { ChartsSection } from "./sections/charts-section";
|
||||
import { ProgressSection } from "./sections/progress-section";
|
||||
import { TrainingStartOverlay } from "./training-start-overlay";
|
||||
|
||||
export function TrainingView(): ReactElement {
|
||||
const runtime = useTrainingRuntimeStore(
|
||||
useShallow((state) => ({
|
||||
phase: state.phase,
|
||||
message: state.message,
|
||||
currentStep: state.currentStep,
|
||||
firstStepReceived: state.firstStepReceived,
|
||||
isStarting: state.isStarting,
|
||||
})),
|
||||
);
|
||||
|
||||
const isPreparingPhase =
|
||||
runtime.phase === "downloading_model" ||
|
||||
runtime.phase === "downloading_dataset" ||
|
||||
runtime.phase === "loading_model" ||
|
||||
runtime.phase === "loading_dataset" ||
|
||||
runtime.phase === "configuring";
|
||||
const isWaitingForFirstStep =
|
||||
runtime.phase === "training" && !runtime.firstStepReceived;
|
||||
const showOverlay =
|
||||
runtime.isStarting ||
|
||||
isPreparingPhase ||
|
||||
(isWaitingForFirstStep && runtime.currentStep <= 0);
|
||||
|
||||
return (
|
||||
<div className={cn("relative", showOverlay && "min-h-[72vh]")}>
|
||||
<div
|
||||
className={cn(
|
||||
"relative z-10 flex flex-col gap-6 transition-[filter]",
|
||||
showOverlay && "blur",
|
||||
)}
|
||||
>
|
||||
<div data-tour="studio-training-progress">
|
||||
<ProgressSection />
|
||||
</div>
|
||||
<ChartsSection />
|
||||
</div>
|
||||
{showOverlay ? (
|
||||
<TrainingStartOverlay
|
||||
message={runtime.message}
|
||||
currentStep={runtime.currentStep}
|
||||
/>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
59
studio/frontend/src/features/training/api/history-api.ts
Normal file
59
studio/frontend/src/features/training/api/history-api.ts
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { authFetch } from "@/features/auth";
|
||||
import type {
|
||||
TrainingRunDeleteResponse,
|
||||
TrainingRunDetailResponse,
|
||||
TrainingRunListResponse,
|
||||
} from "../types/history";
|
||||
|
||||
async function readError(response: Response): Promise<string> {
|
||||
try {
|
||||
const payload = (await response.json()) as { detail?: string; message?: string };
|
||||
return payload.detail || payload.message || `Request failed (${response.status})`;
|
||||
} catch {
|
||||
return `Request failed (${response.status})`;
|
||||
}
|
||||
}
|
||||
|
||||
async function parseJson<T>(response: Response): Promise<T> {
|
||||
if (!response.ok) {
|
||||
throw new Error(await readError(response));
|
||||
}
|
||||
return (await response.json()) as T;
|
||||
}
|
||||
|
||||
export async function listTrainingRuns(
|
||||
limit = 50,
|
||||
offset = 0,
|
||||
signal?: AbortSignal,
|
||||
): Promise<TrainingRunListResponse> {
|
||||
const response = await authFetch(
|
||||
`/api/train/runs?limit=${limit}&offset=${offset}`,
|
||||
{ signal },
|
||||
);
|
||||
return parseJson<TrainingRunListResponse>(response);
|
||||
}
|
||||
|
||||
export async function getTrainingRun(
|
||||
runId: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<TrainingRunDetailResponse> {
|
||||
const response = await authFetch(
|
||||
`/api/train/runs/${encodeURIComponent(runId)}`,
|
||||
{ signal },
|
||||
);
|
||||
return parseJson<TrainingRunDetailResponse>(response);
|
||||
}
|
||||
|
||||
export async function deleteTrainingRun(
|
||||
runId: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<TrainingRunDeleteResponse> {
|
||||
const response = await authFetch(
|
||||
`/api/train/runs/${encodeURIComponent(runId)}`,
|
||||
{ method: "DELETE", signal },
|
||||
);
|
||||
return parseJson<TrainingRunDeleteResponse>(response);
|
||||
}
|
||||
|
|
@ -14,6 +14,14 @@ export { useDatasetPreviewDialogStore } from "./stores/dataset-preview-dialog-st
|
|||
export { uploadTrainingDataset } from "./api/datasets-api";
|
||||
export { listLocalModels } from "./api/models-api";
|
||||
export type { LocalModelInfo } from "./api/models-api";
|
||||
export type { TrainingPhase } from "./types/runtime";
|
||||
export type { TrainingPhase, TrainingViewData, TrainingSeriesPoint } from "./types/runtime";
|
||||
export type {
|
||||
TrainingRunSummary,
|
||||
TrainingRunListResponse,
|
||||
TrainingRunMetrics,
|
||||
TrainingRunDetailResponse,
|
||||
TrainingRunDeleteResponse,
|
||||
} from "./types/history";
|
||||
export { listTrainingRuns, getTrainingRun, deleteTrainingRun } from "./api/history-api";
|
||||
export { parseYamlConfig, serializeConfigToYaml } from "./lib/yaml-config";
|
||||
export { validateTrainingConfig } from "./lib/validation";
|
||||
|
|
|
|||
48
studio/frontend/src/features/training/types/history.ts
Normal file
48
studio/frontend/src/features/training/types/history.ts
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
export interface TrainingRunSummary {
|
||||
id: string;
|
||||
status: "running" | "completed" | "stopped" | "error";
|
||||
model_name: string;
|
||||
dataset_name: string;
|
||||
started_at: string;
|
||||
ended_at: string | null;
|
||||
total_steps: number | null;
|
||||
final_step: number | null;
|
||||
final_loss: number | null;
|
||||
output_dir: string | null;
|
||||
duration_seconds: number | null;
|
||||
error_message: string | null;
|
||||
loss_sparkline: number[] | null;
|
||||
}
|
||||
|
||||
export interface TrainingRunListResponse {
|
||||
runs: TrainingRunSummary[];
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface TrainingRunMetrics {
|
||||
step_history: number[];
|
||||
loss_history: number[];
|
||||
loss_step_history: number[];
|
||||
lr_history: number[];
|
||||
lr_step_history: number[];
|
||||
grad_norm_history: number[];
|
||||
grad_norm_step_history: number[];
|
||||
eval_loss_history: number[];
|
||||
eval_step_history: number[];
|
||||
final_epoch: number | null;
|
||||
final_num_tokens: number | null;
|
||||
}
|
||||
|
||||
export interface TrainingRunDetailResponse {
|
||||
run: TrainingRunSummary;
|
||||
config: Record<string, unknown>;
|
||||
metrics: TrainingRunMetrics;
|
||||
}
|
||||
|
||||
export interface TrainingRunDeleteResponse {
|
||||
status: string;
|
||||
message: string;
|
||||
}
|
||||
|
|
@ -53,8 +53,8 @@ export interface TrainingProgressPayload {
|
|||
job_id: string;
|
||||
step: number;
|
||||
total_steps: number;
|
||||
loss: number;
|
||||
learning_rate: number;
|
||||
loss: number | null;
|
||||
learning_rate: number | null;
|
||||
progress_percent: number;
|
||||
epoch: number | null;
|
||||
elapsed_seconds: number | null;
|
||||
|
|
@ -118,3 +118,32 @@ export interface TrainingRuntimeActions {
|
|||
}
|
||||
|
||||
export type TrainingRuntimeStore = TrainingRuntimeState & TrainingRuntimeActions;
|
||||
|
||||
export interface TrainingViewData {
|
||||
// Current metrics (for ProgressSection)
|
||||
phase: TrainingPhase;
|
||||
currentStep: number;
|
||||
totalSteps: number;
|
||||
currentLoss: number | null;
|
||||
currentLearningRate: number | null;
|
||||
currentGradNorm: number | null;
|
||||
currentEpoch: number | null;
|
||||
currentNumTokens: number | null;
|
||||
progressPercent: number;
|
||||
elapsedSeconds: number | null;
|
||||
etaSeconds: number | null;
|
||||
evalEnabled: boolean;
|
||||
message: string;
|
||||
error: string | null;
|
||||
isTrainingRunning: boolean;
|
||||
|
||||
// Config summary
|
||||
modelName: string;
|
||||
trainingMethod: string;
|
||||
|
||||
// Time-series (for ChartsSection)
|
||||
lossHistory: TrainingSeriesPoint[];
|
||||
lrHistory: TrainingSeriesPoint[];
|
||||
gradNormHistory: TrainingSeriesPoint[];
|
||||
evalLossHistory: TrainingSeriesPoint[];
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue