diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py index 5f504bbdf4..2324916236 100644 --- a/studio/backend/core/training/trainer.py +++ b/studio/backend/core/training/trainer.py @@ -81,8 +81,8 @@ class TrainingProgress: epoch: float = 0 step: int = 0 total_steps: int = 0 - loss: float = 0.0 - learning_rate: float = 0.0 + loss: Optional[float] = None + learning_rate: Optional[float] = None is_training: bool = False is_completed: bool = False error: Optional[str] = None @@ -244,7 +244,7 @@ class UnslothTrainer: def on_log(self, args, state, control, logs = None, **kwargs): if not logs: return - loss_value = logs.get("loss", logs.get("train_loss", 0.0)) + loss_value = logs.get("loss", logs.get("train_loss", None)) current_step = state.global_step grad_norm = logs.get("grad_norm", None) @@ -268,7 +268,7 @@ class UnslothTrainer: step = current_step, epoch = round(state.epoch, 2) if state.epoch else 0, loss = loss_value, - learning_rate = logs.get("learning_rate", 0.0), + learning_rate = logs.get("learning_rate", None), elapsed_seconds = elapsed_seconds, eta_seconds = eta_seconds, grad_norm = grad_norm, diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.py index 9626f9df2e..4439e4e173 100644 --- a/studio/backend/core/training/training.py +++ b/studio/backend/core/training/training.py @@ -14,12 +14,14 @@ worker's mp.Queue, and exposes the same API surface to routes/training.py. Pattern follows core/data_recipe/jobs/manager.py. """ +import json as _json import math import multiprocessing as mp import queue import threading import time import structlog +from datetime import datetime, timezone from loggers import get_logger from dataclasses import dataclass, field from pathlib import Path @@ -44,8 +46,8 @@ class TrainingProgress: epoch: float = 0 step: int = 0 total_steps: int = 0 - loss: float = 0.0 - learning_rate: float = 0.0 + loss: Optional[float] = None + learning_rate: Optional[float] = None is_training: bool = False is_completed: bool = False error: Optional[str] = None @@ -63,6 +65,8 @@ class TrainingBackend: Launches a fresh subprocess per training job, communicates via mp.Queue. """ + FLUSH_THRESHOLD: int = 10 + def __init__(self): # Subprocess state self._proc: Optional[mp.Process] = None @@ -91,13 +95,21 @@ class TrainingBackend: self.current_job_id: Optional[str] = None self._output_dir: Optional[str] = None + # DB persistence + self._metric_buffer: list[dict] = [] + self._run_finalized: bool = False + self._db_run_created: bool = False + self._db_total_steps_set: bool = False + self._db_config: Optional[dict] = None + self._db_started_at: Optional[str] = None + logger.info("TrainingBackend initialized (subprocess mode)") # ------------------------------------------------------------------ # Public API (called by routes/training.py) # ------------------------------------------------------------------ - def start_training(self, **kwargs) -> bool: + def start_training(self, job_id: str, **kwargs) -> bool: """Spawn a subprocess to run the full training pipeline. All kwargs are serialized into a config dict and sent to the worker. @@ -108,30 +120,16 @@ class TrainingBackend: logger.warning("Training subprocess already running") return False - # Join prior pump thread to prevent it from consuming events - # from the new job's queue (it reads self._event_queue dynamically). + # Join prior pump thread — refuse to start if it won't die if self._pump_thread is not None and self._pump_thread.is_alive(): self._pump_thread.join(timeout = 5.0) if self._pump_thread.is_alive(): - logger.warning("Previous pump thread did not exit within 5s") + logger.warning( + "Previous pump thread did not exit within 5s — refusing to start" + ) + return False self._pump_thread = None - # Reset state - self._should_stop = False - self._cancel_requested = False - self._progress = TrainingProgress( - is_training = True, status_message = "Initializing training..." - ) - self.loss_history.clear() - self.lr_history.clear() - self.step_history.clear() - self.grad_norm_history.clear() - self.grad_norm_step_history.clear() - self.eval_loss_history.clear() - self.eval_step_history.clear() - self.eval_enabled = False - self._output_dir = None - # Build config dict for the subprocess config = { "model_name": kwargs["model_name"], @@ -193,23 +191,62 @@ class TrainingBackend: if config["training_type"] != "LoRA/QLoRA": config["load_in_4bit"] = False - # Spawn subprocess + # Spawn subprocess — use locals so state is untouched on failure from .worker import run_training_process - self._event_queue = _CTX.Queue() - self._stop_queue = _CTX.Queue() + event_queue = _CTX.Queue() + stop_queue = _CTX.Queue() - self._proc = _CTX.Process( + proc = _CTX.Process( target = run_training_process, kwargs = { - "event_queue": self._event_queue, - "stop_queue": self._stop_queue, + "event_queue": event_queue, + "stop_queue": stop_queue, "config": config, }, daemon = True, ) - self._proc.start() - logger.info("Training subprocess started (pid=%s)", self._proc.pid) + try: + proc.start() + except Exception: + logger.error("Failed to start training subprocess", exc_info = True) + return False + + logger.info("Training subprocess started (pid=%s)", proc.pid) + + # Reset state — safe because old pump thread is confirmed dead + # and proc.start() succeeded + self.current_job_id = job_id + self._should_stop = False + self._cancel_requested = False + self._progress = TrainingProgress( + is_training = True, status_message = "Initializing training..." + ) + self.loss_history.clear() + self.lr_history.clear() + self.step_history.clear() + self.grad_norm_history.clear() + self.grad_norm_step_history.clear() + self.eval_loss_history.clear() + self.eval_step_history.clear() + self.eval_enabled = False + self._output_dir = None + self._metric_buffer.clear() + self._run_finalized = False + self._db_run_created = False + self._db_total_steps_set = False + self._db_config = { + k: v for k, v in config.items() if k not in {"hf_token", "wandb_token"} + } + self._db_started_at = datetime.now(timezone.utc).isoformat() + + # Assign subprocess handles after state reset + self._event_queue = event_queue + self._stop_queue = stop_queue + self._proc = proc + + # Eagerly create DB run row so the run appears in history during model loading + self._ensure_db_run_created() # Start event pump thread self._pump_thread = threading.Thread(target = self._pump_loop, daemon = True) @@ -252,6 +289,11 @@ class TrainingBackend: proc.kill() proc.join(timeout = 2.0) + # Wait for pump thread to finish DB finalization before returning + # (8s covers SQLite's default 5s lock timeout plus execution overhead) + if self._pump_thread is not None and self._pump_thread.is_alive(): + self._pump_thread.join(timeout = 8.0) + def is_training_active(self) -> bool: """Check if training is currently active.""" with self._lock: @@ -389,20 +431,54 @@ class TrainingBackend: self._progress.error or "Training process exited unexpectedly" ) + + self._ensure_db_run_created() + self._finalize_run_in_db( + status = "stopped" if self._should_stop else "error", + error_message = None + if self._should_stop + else "Training process terminated unexpectedly", + ) return def _handle_event(self, event: dict) -> None: - """Apply a subprocess event to local state.""" + """Apply a subprocess event to local state. + + State updates happen inside self._lock; DB I/O happens after + releasing it so status-polling API endpoints are never blocked + by slow SQLite writes. + """ etype = event.get("type") + db_action: Optional[str] = None + db_action_kwargs: dict = {} with self._lock: if etype == "progress": self._progress.step = event.get("step", self._progress.step) self._progress.epoch = event.get("epoch", self._progress.epoch) - self._progress.loss = event.get("loss", self._progress.loss) - self._progress.learning_rate = event.get( - "learning_rate", self._progress.learning_rate - ) + # loss/lr are sanitized below; update progress after coercion + _raw_loss = event.get("loss") + _raw_lr = event.get("learning_rate") + try: + _safe_loss = float(_raw_loss) if _raw_loss is not None else None + except (TypeError, ValueError): + logger.debug("Could not convert loss to float: %s", _raw_loss) + _safe_loss = None + if _safe_loss is not None and not math.isfinite(_safe_loss): + _safe_loss = None + try: + _safe_lr = float(_raw_lr) if _raw_lr is not None else None + except (TypeError, ValueError): + logger.debug( + "Could not convert learning_rate to float: %s", _raw_lr + ) + _safe_lr = None + if _safe_lr is not None and not math.isfinite(_safe_lr): + _safe_lr = None + if _safe_loss is not None: + self._progress.loss = _safe_loss + if _safe_lr is not None: + self._progress.learning_rate = _safe_lr self._progress.total_steps = event.get( "total_steps", self._progress.total_steps ) @@ -416,30 +492,85 @@ class TrainingBackend: if status: self._progress.status_message = status - # Update metric histories + # Update metric histories — reuse sanitized values from above step = event.get("step", 0) - loss = event.get("loss", 0.0) - lr = event.get("learning_rate", 0.0) - if step >= 0 and loss > 0: + loss = _safe_loss + lr = _safe_lr + if step > 0 and loss is not None: self.loss_history.append(loss) - self.lr_history.append(lr) + self.lr_history.append(lr if lr is not None else 0.0) self.step_history.append(step) grad_norm = event.get("grad_norm") + gn = None if grad_norm is not None: try: gn = float(grad_norm) except (TypeError, ValueError): gn = None - if gn is not None and math.isfinite(gn): + if step > 0 and gn is not None and math.isfinite(gn): self.grad_norm_history.append(gn) self.grad_norm_step_history.append(step) + else: + gn = None eval_loss = event.get("eval_loss") if eval_loss is not None: - self.eval_loss_history.append(eval_loss) - self.eval_step_history.append(step) - self.eval_enabled = True + try: + eval_loss = float(eval_loss) + except (TypeError, ValueError): + logger.debug( + "Could not convert eval_loss to float: %s", eval_loss + ) + eval_loss = None + if step > 0 and eval_loss is not None and math.isfinite(eval_loss): + self.eval_loss_history.append(eval_loss) + self.eval_step_history.append(step) + self.eval_enabled = True + else: + eval_loss = None + + # Buffer metric for DB flush (loss/lr already sanitized above) + self._metric_buffer.append( + { + "step": step, + "loss": loss, + "learning_rate": lr, + "grad_norm": gn, + "eval_loss": eval_loss, + "epoch": event.get("epoch"), + "num_tokens": event.get("num_tokens"), + "elapsed_seconds": event.get("elapsed_seconds"), + } + ) + + # Decide which DB action to take after releasing the lock + if not self._db_run_created and self.current_job_id and self._db_config: + db_action = "create_run" + db_action_kwargs = { + "job_id": self.current_job_id, + "model_name": self._db_config["model_name"], + "dataset_name": self._db_config.get("hf_dataset") + or next( + iter(self._db_config.get("local_datasets") or []), "unknown" + ), + "config_json": _json.dumps(self._db_config), + "started_at": self._db_started_at + or datetime.now(timezone.utc).isoformat(), + "total_steps": event.get("total_steps"), + } + elif ( + event.get("total_steps") + and self._db_run_created + and not self._db_total_steps_set + ): + db_action = "update_total_steps" + db_action_kwargs = { + "job_id": self.current_job_id, + "total_steps": event["total_steps"], + } + elif len(self._metric_buffer) >= self.FLUSH_THRESHOLD: + db_action = "flush" elif etype == "eval_configured": self.eval_enabled = True @@ -454,6 +585,14 @@ class TrainingBackend: self._output_dir = event.get("output_dir") msg = event.get("status_message", "Training completed") self._progress.status_message = msg + if not self._db_run_created and self.current_job_id and self._db_config: + db_action = "create_and_finalize" + else: + db_action = "finalize" + db_action_kwargs = { + "status": "stopped" if self._should_stop else "completed", + "output_dir": self._output_dir, + } elif etype == "error": self._progress.is_training = False @@ -462,6 +601,149 @@ class TrainingBackend: stack = event.get("stack", "") if stack: logger.error("Stack trace:\n%s", stack) + if not self._db_run_created and self.current_job_id and self._db_config: + db_action = "create_and_finalize" + else: + db_action = "finalize" + db_action_kwargs = { + "status": "stopped" if self._should_stop else "error", + "error_message": event.get("error", "Unknown error"), + } + + # --- DB I/O outside the lock --- + if db_action == "create_run": + try: + from storage.studio_db import create_run + + create_run( + id = db_action_kwargs["job_id"], + model_name = db_action_kwargs["model_name"], + dataset_name = db_action_kwargs["dataset_name"], + config_json = db_action_kwargs["config_json"], + started_at = db_action_kwargs["started_at"], + total_steps = db_action_kwargs["total_steps"], + ) + self._db_run_created = True + if db_action_kwargs["total_steps"]: + self._db_total_steps_set = True + except Exception: + logger.warning("Failed to create DB run record", exc_info = True) + elif db_action == "create_and_finalize": + self._ensure_db_run_created() + self._finalize_run_in_db(**db_action_kwargs) + elif db_action == "update_total_steps": + try: + from storage.studio_db import update_run_total_steps + + update_run_total_steps( + db_action_kwargs["job_id"], db_action_kwargs["total_steps"] + ) + self._db_total_steps_set = True + except Exception: + logger.warning("Failed to update total_steps in DB", exc_info = True) + elif db_action == "flush": + self._flush_metrics_to_db() + elif db_action == "finalize": + self._finalize_run_in_db(**db_action_kwargs) + + def _ensure_db_run_created(self) -> None: + """Create the DB row if it doesn't exist yet. Called outside the lock.""" + if self._db_run_created or not self.current_job_id or not self._db_config: + return + try: + from storage.studio_db import create_run + + dataset_name = self._db_config.get("hf_dataset") or next( + iter(self._db_config.get("local_datasets") or []), "unknown" + ) + create_run( + id = self.current_job_id, + model_name = self._db_config["model_name"], + dataset_name = dataset_name, + config_json = _json.dumps(self._db_config), + started_at = self._db_started_at + or datetime.now(timezone.utc).isoformat(), + total_steps = self._progress.total_steps or None, + ) + self._db_run_created = True + except Exception: + logger.warning( + "Failed to create DB run record for early failure", exc_info = True + ) + + def _finalize_run_in_db( + self, + status: str, + error_message: Optional[str] = None, + output_dir: Optional[str] = None, + ) -> None: + """Flush remaining metrics and mark a run as finished in the DB.""" + if not self.current_job_id or not self._db_run_created or self._run_finalized: + return + self._flush_metrics_to_db() + try: + from storage.studio_db import finish_run + from utils.downsample import downsample + + sparkline = downsample(self.loss_history, 50) + finish_run( + id = self.current_job_id, + status = status, + ended_at = datetime.now(timezone.utc).isoformat(), + final_step = self._progress.step, + final_loss = self._progress.loss + if ( + self._progress.loss is not None + and math.isfinite(self._progress.loss) + ) + else None, + duration_seconds = self._progress.elapsed_seconds, + loss_sparkline = _json.dumps(sparkline), + output_dir = output_dir, + error_message = error_message, + ) + self._run_finalized = True + except Exception: + logger.warning( + "Failed to finalize run in DB (status=%s)", status, exc_info = True + ) + + def _flush_metrics_to_db(self) -> None: + """Flush buffered metrics to the database and update live progress.""" + if ( + not self._metric_buffer + or not self.current_job_id + or not self._db_run_created + ): + return + # Cap buffer to prevent unbounded memory growth + if len(self._metric_buffer) > 500: + logger.warning( + "Metric buffer exceeded 500 entries (%d) — trimming oldest", + len(self._metric_buffer), + ) + self._metric_buffer = self._metric_buffer[-500:] + # Snapshot before insert so metrics arriving during the write are preserved + batch = list(self._metric_buffer) + try: + from storage.studio_db import insert_metrics_batch, update_run_progress + + insert_metrics_batch(self.current_job_id, batch) + del self._metric_buffer[: len(batch)] + update_run_progress( + id = self.current_job_id, + step = self._progress.step, + loss = self._progress.loss + if ( + self._progress.loss is not None + and math.isfinite(self._progress.loss) + ) + else None, + duration_seconds = self._progress.elapsed_seconds, + ) + except Exception: + # Leave buffer intact for retry on next flush + logger.warning("Failed to flush metrics to DB", exc_info = True) @staticmethod def _read_queue(q: Any, timeout_sec: float) -> Optional[dict]: @@ -561,11 +843,13 @@ class TrainingBackend: if progress.error: title = f"Error: {progress.error}" elif progress.is_completed: - title = f"Training completed! Final loss: {progress.loss:.4f}" + loss_str = f"{progress.loss:.4f}" if progress.loss is not None else "--" + title = f"Training completed! Final loss: {loss_str}" elif progress.status_message: title = progress.status_message elif progress.step > 0: - title = f"Epoch: {progress.epoch} | Step: {progress.step}/{progress.total_steps} | Loss: {progress.loss:.4f}" + loss_str = f"{progress.loss:.4f}" if progress.loss is not None else "--" + title = f"Epoch: {progress.epoch} | Step: {progress.step}/{progress.total_steps} | Loss: {loss_str}" else: title = "Training Loss" diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py index ccd805b7ac..d06dd6d358 100644 --- a/studio/backend/core/training/worker.py +++ b/studio/backend/core/training/worker.py @@ -242,7 +242,7 @@ def run_training_process( # Wire up progress callback → event_queue def _on_progress(progress: TrainingProgress): - has_train_loss = progress.step >= 0 and progress.loss > 0 + has_train_loss = progress.step > 0 and progress.loss is not None has_eval_loss = progress.eval_loss is not None if has_train_loss or has_eval_loss: event_queue.put( @@ -918,7 +918,7 @@ def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) -> def on_log(self, args, state, control, logs = None, **kwargs): if not logs: return - loss_value = logs.get("loss", logs.get("train_loss", 0.0)) + loss_value = logs.get("loss", logs.get("train_loss", None)) current_step = state.global_step elapsed = time.time() - training_start_time @@ -934,7 +934,7 @@ def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) -> "step": current_step, "epoch": round(state.epoch, 2) if state.epoch else 0, "loss": loss_value, - "learning_rate": logs.get("learning_rate", 0.0), + "learning_rate": logs.get("learning_rate", None), "total_steps": total_steps, "elapsed_seconds": elapsed, "eta_seconds": eta, diff --git a/studio/backend/main.py b/studio/backend/main.py index 7134c5a783..5e647f6312 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -49,6 +49,7 @@ from routes import ( export_router, inference_router, models_router, + training_history_router, training_router, ) from auth import storage @@ -73,6 +74,17 @@ async def lifespan(app: FastAPI): # Detect hardware first — sets DEVICE global used everywhere detect_hardware() + from storage.studio_db import cleanup_orphaned_runs + + try: + cleanup_orphaned_runs() + except Exception as exc: + import structlog + + structlog.get_logger(__name__).warning( + "cleanup_orphaned_runs failed at startup: %s", exc + ) + # Pre-cache the helper GGUF model for LLM-assisted dataset detection. # Runs in a background thread so it doesn't block server startup. import threading @@ -149,6 +161,9 @@ app.include_router(inference_router, prefix = "/v1", tags = ["openai-compat"]) app.include_router(datasets_router, prefix = "/api/datasets", tags = ["datasets"]) app.include_router(data_recipe_router, prefix = "/api/data-recipe", tags = ["data-recipe"]) app.include_router(export_router, prefix = "/api/export", tags = ["export"]) +app.include_router( + training_history_router, prefix = "/api/train", tags = ["training-history"] +) # ============ Health and System Endpoints ============ diff --git a/studio/backend/models/__init__.py b/studio/backend/models/__init__.py index 11cf215f54..a4fbbbe6ee 100644 --- a/studio/backend/models/__init__.py +++ b/studio/backend/models/__init__.py @@ -10,6 +10,11 @@ from .training import ( TrainingJobResponse, TrainingStatus, TrainingProgress, + TrainingRunSummary, + TrainingRunListResponse, + TrainingRunMetrics, + TrainingRunDetailResponse, + TrainingRunDeleteResponse, ) from .models import ( CheckpointInfo, @@ -71,6 +76,11 @@ __all__ = [ "TrainingJobResponse", "TrainingStatus", "TrainingProgress", + "TrainingRunSummary", + "TrainingRunListResponse", + "TrainingRunMetrics", + "TrainingRunDetailResponse", + "TrainingRunDeleteResponse", # Model management schemas "ModelDetails", "LocalModelInfo", diff --git a/studio/backend/models/training.py b/studio/backend/models/training.py index 342e44cc09..68791aa7a8 100644 --- a/studio/backend/models/training.py +++ b/studio/backend/models/training.py @@ -177,8 +177,8 @@ class TrainingProgress(BaseModel): job_id: str = Field(..., description = "Training job identifier") step: int = Field(..., description = "Current training step") total_steps: int = Field(..., description = "Total training steps") - loss: float = Field(..., description = "Current loss value") - learning_rate: float = Field(..., description = "Current learning rate") + loss: Optional[float] = Field(None, description = "Current loss value") + learning_rate: Optional[float] = Field(None, description = "Current learning rate") progress_percent: float = Field( ..., description = "Progress percentage (0.0 to 100.0)" ) @@ -196,3 +196,59 @@ class TrainingProgress(BaseModel): eval_loss: Optional[float] = Field( None, description = "Eval loss from the most recent evaluation step" ) + + +class TrainingRunSummary(BaseModel): + """Summary of a training run for list views.""" + + id: str + status: Literal["running", "completed", "stopped", "error"] + model_name: str + dataset_name: str + started_at: str + ended_at: Optional[str] = None + total_steps: Optional[int] = None + final_step: Optional[int] = None + final_loss: Optional[float] = None + output_dir: Optional[str] = None + duration_seconds: Optional[float] = None + error_message: Optional[str] = None + loss_sparkline: Optional[List[float]] = None + + +class TrainingRunListResponse(BaseModel): + """Response for listing training runs.""" + + runs: List[TrainingRunSummary] + total: int + + +class TrainingRunMetrics(BaseModel): + """Metrics arrays for a training run, using paired step arrays per metric.""" + + step_history: List[int] = Field(default_factory = list) + loss_history: List[float] = Field(default_factory = list) + loss_step_history: List[int] = Field(default_factory = list) + lr_history: List[float] = Field(default_factory = list) + lr_step_history: List[int] = Field(default_factory = list) + grad_norm_history: List[float] = Field(default_factory = list) + grad_norm_step_history: List[int] = Field(default_factory = list) + eval_loss_history: List[float] = Field(default_factory = list) + eval_step_history: List[int] = Field(default_factory = list) + final_epoch: Optional[float] = None + final_num_tokens: Optional[int] = None + + +class TrainingRunDetailResponse(BaseModel): + """Response for a single training run with config and metrics.""" + + run: TrainingRunSummary + config: dict + metrics: TrainingRunMetrics + + +class TrainingRunDeleteResponse(BaseModel): + """Response for deleting a training run.""" + + status: str + message: str diff --git a/studio/backend/routes/__init__.py b/studio/backend/routes/__init__.py index b45eff821b..e79f6553f9 100644 --- a/studio/backend/routes/__init__.py +++ b/studio/backend/routes/__init__.py @@ -12,6 +12,7 @@ from routes.datasets import router as datasets_router from routes.auth import router as auth_router from routes.data_recipe import router as data_recipe_router from routes.export import router as export_router +from routes.training_history import router as training_history_router __all__ = [ "training_router", @@ -21,4 +22,5 @@ __all__ = [ "auth_router", "data_recipe_router", "export_router", + "training_history_router", ] diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py index 4f8054f80e..4cfb060dee 100644 --- a/studio/backend/routes/training.py +++ b/studio/backend/routes/training.py @@ -14,6 +14,7 @@ import structlog from loggers import get_logger import asyncio from datetime import datetime +import uuid as _uuid # Add backend directory to path # The backend code should be in the same directory structure @@ -115,15 +116,11 @@ async def start_training( backend = get_training_backend() - # Generate job ID and attach to backend for later status/progress calls - job_id = f"job_{datetime.now().strftime('%Y%m%d_%H%M%S')}" - backend.current_job_id = job_id - - # Check if training is already active + # Check if training is already active (before mutating any state) if backend.is_training_active(): existing_job_id: Optional[str] = getattr(backend, "current_job_id", "") return TrainingJobResponse( - job_id = existing_job_id or job_id, + job_id = existing_job_id or "", status = "error", message = ( "Training is already in progress. " @@ -132,6 +129,12 @@ async def start_training( error = "Training already active", ) + # Generate job ID — passed into start_training() which sets it on the + # backend only after confirming the old pump thread is dead. + job_id = ( + f"job_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{_uuid.uuid4().hex[:8]}" + ) + # Validate dataset paths if provided if request.local_datasets: request.local_datasets = _validate_local_dataset_paths( @@ -248,12 +251,12 @@ async def start_training( logger.warning("Could not shut down export subprocess: %s", e) # start_training now spawns a subprocess (non-blocking) - success = backend.start_training(**training_kwargs) + success = backend.start_training(job_id = job_id, **training_kwargs) if not success: progress_error = backend.trainer.training_progress.error return TrainingJobResponse( - job_id = job_id, + job_id = backend.current_job_id or "", status = "error", message = progress_error or "Failed to start training subprocess", error = progress_error or "subprocess_start_failed", @@ -345,7 +348,7 @@ async def reset_training( error = None, status_message = "Ready to train", step = 0, - loss = 0.0, + loss = None, epoch = 0, total_steps = 0, ) @@ -419,8 +422,8 @@ async def get_training_status( "epoch": getattr(progress, "epoch", 0), "step": getattr(progress, "step", 0), "total_steps": getattr(progress, "total_steps", 0), - "loss": getattr(progress, "loss", 0.0), - "learning_rate": getattr(progress, "learning_rate", 0.0), + "loss": getattr(progress, "loss", None), + "learning_rate": getattr(progress, "learning_rate", None), } # Build metric history for chart recovery after SSE reconnection @@ -526,8 +529,8 @@ async def stream_training_progress( # ── Helpers ────────────────────────────────────────────── def build_progress( step: int, - loss: float, - learning_rate: float, + loss: Optional[float], + learning_rate: Optional[float], total_steps: int, epoch: Optional[float] = None, progress: Optional[Any] = None, @@ -604,10 +607,10 @@ async def stream_training_progress( loss_val = ( backend.loss_history[i] if i < len(backend.loss_history) - else 0.0 + else None ) lr_val = ( - backend.lr_history[i] if i < len(backend.lr_history) else 0.0 + backend.lr_history[i] if i < len(backend.lr_history) else None ) tp_replay = getattr( getattr(backend, "trainer", None), "training_progress", None @@ -645,8 +648,8 @@ async def stream_training_progress( initial_progress = build_progress( step = 0, - loss = 0.0, - learning_rate = 0.0, + loss = None, + learning_rate = None, total_steps = initial_total_steps, epoch = initial_epoch, progress = tp, @@ -660,9 +663,9 @@ async def stream_training_progress( if backend.step_history: final_step = backend.step_history[-1] final_loss = ( - backend.loss_history[-1] if backend.loss_history else 0.0 + backend.loss_history[-1] if backend.loss_history else None ) - final_lr = backend.lr_history[-1] if backend.lr_history else 0.0 + final_lr = backend.lr_history[-1] if backend.lr_history else None final_total_steps = ( getattr(tp, "total_steps", final_step) if tp else final_step ) @@ -680,7 +683,9 @@ async def stream_training_progress( ) else: yield format_sse( - build_progress(-1, 0.0, 0.0, 0, progress = tp).model_dump_json(), + build_progress( + -1, None, None, 0, progress = tp + ).model_dump_json(), event = "complete", event_id = 0, ) @@ -698,9 +703,9 @@ async def stream_training_progress( if backend.step_history: current_step = backend.step_history[-1] current_loss = ( - backend.loss_history[-1] if backend.loss_history else 0.0 + backend.loss_history[-1] if backend.loss_history else None ) - current_lr = backend.lr_history[-1] if backend.lr_history else 0.0 + current_lr = backend.lr_history[-1] if backend.lr_history else None tp_inner = getattr( getattr(backend, "trainer", None), "training_progress", None ) @@ -763,8 +768,8 @@ async def stream_training_progress( ) preparing_payload = build_progress( 0, - 0.0, - 0.0, + None, + None, prep_total, progress = tp_prep, ) @@ -781,7 +786,7 @@ async def stream_training_progress( getattr(backend, "trainer", None), "training_progress", None ) timeout_payload = build_progress( - last_step, 0.0, 0.0, 0, progress = tp_timeout + last_step, None, None, 0, progress = tp_timeout ) yield format_sse( timeout_payload.model_dump_json(), @@ -797,7 +802,7 @@ async def stream_training_progress( tp_error = getattr( getattr(backend, "trainer", None), "training_progress", None ) - error_payload = build_progress(0, 0.0, 0.0, 0, progress = tp_error) + error_payload = build_progress(0, None, None, 0, progress = tp_error) yield format_sse( error_payload.model_dump_json(), event = "error", @@ -807,8 +812,8 @@ async def stream_training_progress( # ── Final "complete" event ─────────────────────────────── final_step = backend.step_history[-1] if backend.step_history else last_step - final_loss = backend.loss_history[-1] if backend.loss_history else 0.0 - final_lr = backend.lr_history[-1] if backend.lr_history else 0.0 + final_loss = backend.loss_history[-1] if backend.loss_history else None + final_lr = backend.lr_history[-1] if backend.lr_history else None final_tp = getattr(getattr(backend, "trainer", None), "training_progress", None) final_total_steps = ( getattr(final_tp, "total_steps", final_step) if final_tp else final_step diff --git a/studio/backend/routes/training_history.py b/studio/backend/routes/training_history.py new file mode 100644 index 0000000000..597c4424c0 --- /dev/null +++ b/studio/backend/routes/training_history.py @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +""" +Training history API routes — browse, view, and delete past training runs. +""" + +import json + +from fastapi import APIRouter, Depends, HTTPException, Query +from loggers import get_logger + +from auth.authentication import get_current_subject +from models import ( + TrainingRunDeleteResponse, + TrainingRunDetailResponse, + TrainingRunListResponse, + TrainingRunMetrics, + TrainingRunSummary, +) +from storage.studio_db import delete_run, get_run, get_run_metrics, list_runs + +logger = get_logger(__name__) + +router = APIRouter() + + +@router.get("/runs", response_model = TrainingRunListResponse) +async def list_training_runs( + limit: int = Query(50, ge = 1, le = 200), + offset: int = Query(0, ge = 0), + current_subject: str = Depends(get_current_subject), +): + """List training runs, newest first.""" + result = list_runs(limit = limit, offset = offset) + return TrainingRunListResponse( + runs = [TrainingRunSummary(**r) for r in result["runs"]], + total = result["total"], + ) + + +@router.get("/runs/{run_id}", response_model = TrainingRunDetailResponse) +async def get_training_run_detail( + run_id: str, + current_subject: str = Depends(get_current_subject), +): + """Get a single training run with full config and metrics.""" + run = get_run(run_id) + if run is None: + raise HTTPException(status_code = 404, detail = f"Run {run_id} not found") + + try: + config = json.loads(run.get("config_json", "{}")) + except (json.JSONDecodeError, TypeError): + logger.debug("Failed to parse config_json for run %s", run_id) + config = {} + + metrics_data = get_run_metrics(run_id) + + return TrainingRunDetailResponse( + run = TrainingRunSummary(**{k: v for k, v in run.items() if k != "config_json"}), + config = config, + metrics = TrainingRunMetrics(**metrics_data), + ) + + +@router.delete("/runs/{run_id}", response_model = TrainingRunDeleteResponse) +async def delete_training_run( + run_id: str, + current_subject: str = Depends(get_current_subject), +): + """Delete a training run and its metrics (CASCADE).""" + run = get_run(run_id) + if run is None: + raise HTTPException(status_code = 404, detail = f"Run {run_id} not found") + if run["status"] == "running": + raise HTTPException( + status_code = 409, detail = "Cannot delete a running training run" + ) + logger.info("Deleting training run %s", run_id) + delete_run(run_id) + return TrainingRunDeleteResponse( + status = "deleted", + message = f"Run {run_id} deleted", + ) diff --git a/studio/backend/storage/__init__.py b/studio/backend/storage/__init__.py new file mode 100644 index 0000000000..32014236c6 --- /dev/null +++ b/studio/backend/storage/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 diff --git a/studio/backend/storage/studio_db.py b/studio/backend/storage/studio_db.py new file mode 100644 index 0000000000..4af19df42b --- /dev/null +++ b/studio/backend/storage/studio_db.py @@ -0,0 +1,362 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +""" +SQLite storage for training run history and metrics. + +Follows the same pattern as auth/storage.py — module-level functions, +raw sqlite3, per-function connections. Enhancements over auth: + - WAL mode for concurrent read/write access + - PRAGMA foreign_keys = ON for CASCADE deletes +""" + +import json +import logging +import sqlite3 +import threading + +logger = logging.getLogger(__name__) +from typing import Optional + +from utils.paths import studio_db_path, ensure_dir + +_schema_lock = threading.Lock() +_schema_ready = False + + +def _ensure_schema(conn: sqlite3.Connection) -> None: + """Create tables and indexes if they don't exist. Called once per process.""" + conn.execute("PRAGMA journal_mode=WAL") + conn.execute( + """ + CREATE TABLE IF NOT EXISTS training_runs ( + id TEXT NOT NULL PRIMARY KEY, + status TEXT NOT NULL DEFAULT 'running', + model_name TEXT NOT NULL, + dataset_name TEXT NOT NULL, + config_json TEXT NOT NULL, + started_at TEXT NOT NULL, + ended_at TEXT, + total_steps INTEGER, + final_step INTEGER, + final_loss REAL, + output_dir TEXT, + error_message TEXT, + duration_seconds REAL, + loss_sparkline TEXT + ) + """ + ) + conn.execute( + """ + CREATE TABLE IF NOT EXISTS training_metrics ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id TEXT NOT NULL REFERENCES training_runs(id) ON DELETE CASCADE, + step INTEGER NOT NULL, + loss REAL, + learning_rate REAL, + grad_norm REAL, + eval_loss REAL, + epoch REAL, + num_tokens INTEGER, + elapsed_seconds REAL, + UNIQUE(run_id, step) + ) + """ + ) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_metrics_run_id ON training_metrics(run_id)" + ) + + +def get_connection() -> sqlite3.Connection: + """Open studio.db with WAL mode, create tables once per process, enable foreign keys.""" + global _schema_ready + db_path = studio_db_path() + ensure_dir(db_path.parent) + conn = sqlite3.connect(str(db_path)) + conn.row_factory = sqlite3.Row + # foreign_keys is session-scoped, must be set per connection + conn.execute("PRAGMA foreign_keys=ON") + if not _schema_ready: + with _schema_lock: + if not _schema_ready: + try: + _ensure_schema(conn) + _schema_ready = True + except Exception: + conn.close() + raise + return conn + + +def create_run( + id: str, + model_name: str, + dataset_name: str, + config_json: str, + started_at: str, + total_steps: Optional[int], +) -> None: + conn = get_connection() + try: + conn.execute( + """ + INSERT INTO training_runs (id, model_name, dataset_name, config_json, started_at, total_steps) + VALUES (?, ?, ?, ?, ?, ?) + """, + (id, model_name, dataset_name, config_json, started_at, total_steps), + ) + conn.commit() + finally: + conn.close() + + +def update_run_total_steps(id: str, total_steps: int) -> None: + conn = get_connection() + try: + conn.execute( + "UPDATE training_runs SET total_steps = ? WHERE id = ?", + (total_steps, id), + ) + conn.commit() + finally: + conn.close() + + +def update_run_progress( + id: str, step: int, loss: Optional[float], duration_seconds: Optional[float] +) -> None: + """Update current progress on a running training run (called on each metric flush).""" + conn = get_connection() + try: + conn.execute( + "UPDATE training_runs SET final_step = ?, final_loss = ?, duration_seconds = ? WHERE id = ?", + (step, loss, duration_seconds, id), + ) + conn.commit() + finally: + conn.close() + + +def finish_run( + id: str, + status: str, + ended_at: str, + final_step: Optional[int], + final_loss: Optional[float], + duration_seconds: Optional[float], + loss_sparkline: Optional[str] = None, + output_dir: Optional[str] = None, + error_message: Optional[str] = None, +) -> None: + conn = get_connection() + try: + conn.execute( + """ + UPDATE training_runs + SET status = ?, ended_at = ?, final_step = ?, final_loss = ?, + duration_seconds = ?, loss_sparkline = ?, output_dir = ?, + error_message = ? + WHERE id = ? + """, + ( + status, + ended_at, + final_step, + final_loss, + duration_seconds, + loss_sparkline, + output_dir, + error_message, + id, + ), + ) + conn.commit() + finally: + conn.close() + + +def insert_metrics_batch(run_id: str, metrics: list[dict]) -> None: + if not metrics: + return + conn = get_connection() + try: + conn.executemany( + """ + INSERT INTO training_metrics + (run_id, step, loss, learning_rate, grad_norm, eval_loss, epoch, num_tokens, elapsed_seconds) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(run_id, step) DO UPDATE SET + loss = COALESCE(excluded.loss, loss), + learning_rate = COALESCE(excluded.learning_rate, learning_rate), + grad_norm = COALESCE(excluded.grad_norm, grad_norm), + eval_loss = COALESCE(excluded.eval_loss, eval_loss), + epoch = COALESCE(excluded.epoch, epoch), + num_tokens = COALESCE(excluded.num_tokens, num_tokens), + elapsed_seconds = COALESCE(excluded.elapsed_seconds, elapsed_seconds) + """, + [ + ( + run_id, + m.get("step"), + m.get("loss"), + m.get("learning_rate"), + m.get("grad_norm"), + m.get("eval_loss"), + m.get("epoch"), + m.get("num_tokens"), + m.get("elapsed_seconds"), + ) + for m in metrics + ], + ) + conn.commit() + finally: + conn.close() + + +def list_runs(limit: int = 50, offset: int = 0) -> dict: + conn = get_connection() + try: + total = conn.execute("SELECT COUNT(*) FROM training_runs").fetchone()[0] + rows = conn.execute( + """ + SELECT id, status, model_name, dataset_name, started_at, ended_at, + total_steps, final_step, final_loss, output_dir, + duration_seconds, error_message, loss_sparkline + FROM training_runs + ORDER BY started_at DESC + LIMIT ? OFFSET ? + """, + (limit, offset), + ).fetchall() + runs = [] + for row in rows: + run = dict(row) + sparkline = run.get("loss_sparkline") + if sparkline: + try: + run["loss_sparkline"] = json.loads(sparkline) + except (json.JSONDecodeError, TypeError): + logger.debug( + "Failed to parse loss_sparkline for run %s", run.get("id") + ) + run["loss_sparkline"] = None + runs.append(run) + return {"runs": runs, "total": total} + finally: + conn.close() + + +def get_run(id: str) -> Optional[dict]: + conn = get_connection() + try: + row = conn.execute("SELECT * FROM training_runs WHERE id = ?", (id,)).fetchone() + if row is None: + return None + run = dict(row) + sparkline = run.get("loss_sparkline") + if sparkline: + try: + run["loss_sparkline"] = json.loads(sparkline) + except (json.JSONDecodeError, TypeError): + logger.debug("Failed to parse loss_sparkline for run %s", id) + run["loss_sparkline"] = None + return run + finally: + conn.close() + + +def get_run_metrics(id: str) -> dict: + """Return metric arrays for a run, using paired step arrays per metric.""" + conn = get_connection() + try: + rows = conn.execute( + """ + SELECT step, loss, learning_rate, grad_norm, eval_loss, epoch, + num_tokens, elapsed_seconds + FROM training_metrics + WHERE run_id = ? + ORDER BY step + """, + (id,), + ).fetchall() + + step_history: list[int] = [] + loss_history: list[float] = [] + loss_step_history: list[int] = [] + lr_history: list[float] = [] + lr_step_history: list[int] = [] + grad_norm_history: list[float] = [] + grad_norm_step_history: list[int] = [] + eval_loss_history: list[float] = [] + eval_step_history: list[int] = [] + final_epoch: float | None = None + final_num_tokens: int | None = None + + for row in rows: + step = row["step"] + step_history.append(step) + if step > 0 and row["loss"] is not None: + loss_history.append(row["loss"]) + loss_step_history.append(step) + if step > 0 and row["learning_rate"] is not None: + lr_history.append(row["learning_rate"]) + lr_step_history.append(step) + if step > 0 and row["grad_norm"] is not None: + grad_norm_history.append(row["grad_norm"]) + grad_norm_step_history.append(step) + if step > 0 and row["eval_loss"] is not None: + eval_loss_history.append(row["eval_loss"]) + eval_step_history.append(step) + if row["epoch"] is not None: + final_epoch = row["epoch"] + if row["num_tokens"] is not None: + final_num_tokens = row["num_tokens"] + + return { + "step_history": step_history, + "loss_history": loss_history, + "loss_step_history": loss_step_history, + "lr_history": lr_history, + "lr_step_history": lr_step_history, + "grad_norm_history": grad_norm_history, + "grad_norm_step_history": grad_norm_step_history, + "eval_loss_history": eval_loss_history, + "eval_step_history": eval_step_history, + "final_epoch": final_epoch, + "final_num_tokens": final_num_tokens, + } + finally: + conn.close() + + +def delete_run(id: str) -> None: + conn = get_connection() + try: + conn.execute("DELETE FROM training_runs WHERE id = ?", (id,)) + conn.commit() + finally: + conn.close() + + +def cleanup_orphaned_runs() -> None: + """Mark any 'running' rows as errored on startup (server restarted mid-training).""" + from datetime import datetime, timezone + + conn = get_connection() + try: + conn.execute( + """ + UPDATE training_runs + SET status = 'error', + error_message = 'Server restarted during training', + ended_at = ? + WHERE status = 'running' + """, + (datetime.now(timezone.utc).isoformat(),), + ) + conn.commit() + finally: + conn.close() diff --git a/studio/backend/utils/downsample.py b/studio/backend/utils/downsample.py new file mode 100644 index 0000000000..bccf6a23b7 --- /dev/null +++ b/studio/backend/utils/downsample.py @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Generic numeric downsampling utility.""" + + +def downsample(values: list[float], target_count: int) -> list[float]: + """Reduce a list to target_count points via evenly-spaced index sampling.""" + if len(values) <= target_count: + return list(values) + if target_count <= 0: + return [] + if target_count == 1: + return [values[-1]] + indices = [ + round(i * (len(values) - 1) / (target_count - 1)) for i in range(target_count) + ] + return [values[i] for i in indices] diff --git a/studio/backend/utils/paths/__init__.py b/studio/backend/utils/paths/__init__.py index 90df216f96..789052f372 100644 --- a/studio/backend/utils/paths/__init__.py +++ b/studio/backend/utils/paths/__init__.py @@ -16,6 +16,7 @@ from .storage_roots import ( exports_root, auth_root, auth_db_path, + studio_db_path, tmp_root, seed_uploads_root, unstructured_seed_cache_root, @@ -45,6 +46,7 @@ __all__ = [ "exports_root", "auth_root", "auth_db_path", + "studio_db_path", "tmp_root", "seed_uploads_root", "unstructured_seed_cache_root", diff --git a/studio/backend/utils/paths/storage_roots.py b/studio/backend/utils/paths/storage_roots.py index f14887e119..626e868275 100644 --- a/studio/backend/utils/paths/storage_roots.py +++ b/studio/backend/utils/paths/storage_roots.py @@ -49,6 +49,10 @@ def auth_db_path() -> Path: return auth_root() / "auth.db" +def studio_db_path() -> Path: + return studio_root() / "studio.db" + + def tmp_root() -> Path: return Path(tempfile.gettempdir()) / "unsloth-studio" diff --git a/studio/frontend/src/features/studio/historical-training-view.tsx b/studio/frontend/src/features/studio/historical-training-view.tsx new file mode 100644 index 0000000000..d3ac434bd8 --- /dev/null +++ b/studio/frontend/src/features/studio/historical-training-view.tsx @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import type { TrainingViewData } from "@/features/training"; +import { getTrainingRun } from "@/features/training"; +import type { TrainingRunDetailResponse } from "@/features/training"; +import { type ReactElement, useEffect, useState } from "react"; +import { ChartsSection } from "./sections/charts-section"; +import { ProgressSection } from "./sections/progress-section"; + +interface HistoricalTrainingViewProps { + runId: string; +} + +function normalizeTrainingMethod(config: Record): string { + const type = config?.training_type as string | undefined; + if (!type || type === "Full Finetuning") return "full"; + if (type === "LoRA/QLoRA") { + return config?.load_in_4bit ? "qlora" : "lora"; + } + return "full"; +} + +function mapToViewData(detail: TrainingRunDetailResponse): TrainingViewData { + const { run, metrics } = detail; + + const lossHistory = metrics.loss_step_history + .map((step, i) => ({ step, value: metrics.loss_history[i] })) + .filter((p): p is { step: number; value: number } => p.value != null); + + const lrHistory = metrics.lr_step_history + .map((step, i) => ({ step, value: metrics.lr_history[i] })) + .filter((p): p is { step: number; value: number } => p.value != null); + + const gradNormHistory = metrics.grad_norm_step_history + .map((step, i) => ({ step, value: metrics.grad_norm_history[i] })) + .filter((p): p is { step: number; value: number } => p.value != null); + + const evalLossHistory = metrics.eval_step_history + .map((step, i) => ({ step, value: metrics.eval_loss_history[i] })) + .filter((p): p is { step: number; value: number } => p.value != null); + + const phase = + run.status === "completed" + ? "completed" + : run.status === "stopped" + ? "stopped" + : run.status === "error" + ? "error" + : run.status === "running" + ? "training" + : "idle"; + + return { + phase, + currentStep: run.final_step ?? 0, + totalSteps: run.total_steps ?? 0, + currentLoss: run.final_loss, + currentLearningRate: metrics.lr_history.at(-1) ?? null, + currentGradNorm: metrics.grad_norm_history.at(-1) ?? null, + currentEpoch: metrics.final_epoch, + currentNumTokens: metrics.final_num_tokens ?? null, + progressPercent: + run.total_steps && run.final_step + ? (run.final_step / run.total_steps) * 100 + : 0, + elapsedSeconds: run.duration_seconds, + etaSeconds: null, + evalEnabled: evalLossHistory.length > 0, + message: + run.status === "completed" + ? "Training completed" + : run.status === "stopped" + ? "Training stopped" + : run.status === "running" + ? "Training in progress" + : run.error_message ?? "Training errored", + error: run.status === "error" ? run.error_message : null, + isTrainingRunning: false, + modelName: run.model_name, + trainingMethod: normalizeTrainingMethod(detail.config), + lossHistory, + lrHistory, + gradNormHistory, + evalLossHistory, + }; +} + +export function HistoricalTrainingView({ + runId, +}: HistoricalTrainingViewProps): ReactElement { + const [detail, setDetail] = useState(null); + const [error, setError] = useState(null); + + // Derive loading from detail/error -- no separate state needed + const loading = detail === null && error === null; + + useEffect(() => { + const controller = new AbortController(); + getTrainingRun(runId, controller.signal) + .then((result) => { + setDetail(result); + }) + .catch((err) => { + if (err instanceof DOMException && err.name === "AbortError") return; + setError(err instanceof Error ? err.message : "Failed to load run"); + }); + return () => { + controller.abort(); + // Reset on runId change so loading derives correctly for the next fetch + setDetail(null); + setError(null); + }; + }, [runId]); + + if (loading) { + return ( +
+ Loading training run... +
+ ); + } + + if (error || !detail) { + return ( +
+ {error ?? "Run not found"} +
+ ); + } + + const viewData = mapToViewData(detail); + const configOverride = detail.config + ? { + epochs: detail.config.num_epochs as number | undefined, + batchSize: detail.config.batch_size as number | undefined, + learningRate: detail.config.learning_rate as string | undefined, + maxSteps: detail.config.max_steps as number | undefined, + contextLength: detail.config.max_seq_length as number | undefined, + warmupSteps: detail.config.warmup_steps as number | undefined, + optimizerType: detail.config.optim as string | undefined, + loraRank: detail.config.lora_r as number | undefined, + loraAlpha: detail.config.lora_alpha as number | undefined, + loraDropout: detail.config.lora_dropout as number | undefined, + loraVariant: detail.config.use_rslora ? "rsLoRA" : undefined, + } + : undefined; + + return ( +
+ + +
+ ); +} diff --git a/studio/frontend/src/features/studio/history-card-grid.tsx b/studio/frontend/src/features/studio/history-card-grid.tsx new file mode 100644 index 0000000000..78859d2f81 --- /dev/null +++ b/studio/frontend/src/features/studio/history-card-grid.tsx @@ -0,0 +1,401 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { Button } from "@/components/ui/button"; +import type { TrainingRunSummary } from "@/features/training"; +import { deleteTrainingRun, listTrainingRuns } from "@/features/training"; +import { cn } from "@/lib/utils"; +import { Delete02Icon } from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { type ReactElement, useCallback, useEffect, useRef, useState } from "react"; +import { Spinner } from "@/components/ui/spinner"; + +const PAGE_SIZE = 12; +const RUNNING_POLL_INTERVAL_MS = 5000; + +const statusBadge: Record< + string, + { label: string; className: string } +> = { + completed: { + label: "Completed", + className: + "bg-emerald-100 text-emerald-700 dark:bg-emerald-950 dark:text-emerald-400", + }, + stopped: { + label: "Stopped", + className: + "bg-amber-100 text-amber-700 dark:bg-amber-950 dark:text-amber-400", + }, + error: { + label: "Error", + className: "bg-red-100 text-red-700 dark:bg-red-950 dark:text-red-400", + }, + running: { + label: "Running", + className: + "bg-blue-100 text-blue-700 dark:bg-blue-950 dark:text-blue-400", + }, +}; + +function catmullRomPath(points: { x: number; y: number }[]): string { + if (points.length < 2) return ""; + const d = [`M${points[0]!.x.toFixed(1)},${points[0]!.y.toFixed(1)}`]; + for (let i = 0; i < points.length - 1; i++) { + const p0 = points[Math.max(i - 1, 0)]!; + const p1 = points[i]!; + const p2 = points[i + 1]!; + const p3 = points[Math.min(i + 2, points.length - 1)]!; + const cp1x = p1.x + (p2.x - p0.x) / 6; + const cp1y = p1.y + (p2.y - p0.y) / 6; + const cp2x = p2.x - (p3.x - p1.x) / 6; + const cp2y = p2.y - (p3.y - p1.y) / 6; + d.push( + `C${cp1x.toFixed(1)},${cp1y.toFixed(1)} ${cp2x.toFixed(1)},${cp2y.toFixed(1)} ${p2.x.toFixed(1)},${p2.y.toFixed(1)}`, + ); + } + return d.join(" "); +} + +function Sparkline({ values, id }: { values: number[]; id: string }): ReactElement | null { + if (!values || values.length < 2) return null; + let min = values[0]!; + let max = values[0]!; + for (let i = 1; i < values.length; i++) { + if (values[i]! < min) min = values[i]!; + if (values[i]! > max) max = values[i]!; + } + const range = max - min || 1; + const pad = 1.5; // half stroke-width so peaks aren't clipped + const h = 32; + const w = 120; + const gradientId = `sparkFill-${id}`; + + // Build points with vertical padding so the stroke isn't clipped + const pts = values.map((v, i) => ({ + x: (i / (values.length - 1)) * w, + y: pad + (1 - (v - min) / range) * (h - pad * 2), + })); + + const linePath = catmullRomPath(pts); + const last = pts[pts.length - 1]!; + const first = pts[0]!; + const fillPath = `${linePath} L${last.x.toFixed(1)},${h} L${first.x.toFixed(1)},${h} Z`; + + return ( + + + + + + + + + + + ); +} + +function formatRelativeTime(isoDate: string): string { + const diff = Date.now() - new Date(isoDate).getTime(); + const mins = Math.floor(diff / 60000); + if (mins < 1) return "just now"; + if (mins < 60) return `${mins}m ago`; + const hrs = Math.floor(mins / 60); + if (hrs < 24) return `${hrs}h ago`; + const days = Math.floor(hrs / 24); + return `${days}d ago`; +} + +function formatDuration(seconds: number | null): string { + if (seconds == null) return "--"; + const total = Math.floor(seconds); + if (total < 60) return `${total}s`; + const min = Math.floor(total / 60); + const sec = total % 60; + if (min < 60) return `${min}m ${sec}s`; + const hrs = Math.floor(min / 60); + return `${hrs}h ${min % 60}m`; +} + +interface HistoryCardGridProps { + onSelectRun: (runId: string) => void; +} + +export function HistoryCardGrid({ + onSelectRun, +}: HistoryCardGridProps): ReactElement { + const [runs, setRuns] = useState([]); + const [total, setTotal] = useState(0); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [deleteError, setDeleteError] = useState(null); + const [deleteTarget, setDeleteTarget] = useState(null); + const [manualFetchInFlight, setManualFetchInFlight] = useState(false); + + const userControllerRef = useRef(null); + const pollControllerRef = useRef(null); + const fetchIdRef = useRef(0); + const pollIdRef = useRef(0); + + const fetchRuns = useCallback(async (offset = 0, append = false, limit = PAGE_SIZE) => { + // Cancel any in-flight poll so its stale response can't clobber this fresher fetch + pollControllerRef.current?.abort(); + userControllerRef.current?.abort(); + const controller = new AbortController(); + userControllerRef.current = controller; + const id = ++fetchIdRef.current; + + setManualFetchInFlight(true); + setLoading(true); + setError(null); + try { + const result = await listTrainingRuns(limit, offset, controller.signal); + if (fetchIdRef.current !== id) return; + setRuns((prev) => (append ? [...prev, ...result.runs] : result.runs)); + setTotal(result.total); + } catch (err) { + if (err instanceof DOMException && err.name === "AbortError") return; + if (fetchIdRef.current !== id) return; + if (!append) setError("Failed to load training runs"); + } finally { + if (fetchIdRef.current === id) { + setLoading(false); + setManualFetchInFlight(false); + } + } + }, []); + + useEffect(() => { + void fetchRuns(0); + return () => { + userControllerRef.current?.abort(); + }; + }, [fetchRuns]); + + // Poll while any run is still "running" so the card shows live progress + const hasRunningRun = runs.some((r) => r.status === "running"); + const visibleCount = runs.length; + useEffect(() => { + if (!hasRunningRun) return; + const timer = setInterval(async () => { + if (manualFetchInFlight) return; + pollControllerRef.current?.abort(); + const controller = new AbortController(); + pollControllerRef.current = controller; + const pid = ++pollIdRef.current; + try { + const limit = Math.max(PAGE_SIZE, visibleCount); + const result = await listTrainingRuns(limit, 0, controller.signal); + if (pollIdRef.current !== pid) return; // stale poll — discard + setRuns(result.runs); + setTotal(result.total); + } catch { + // silently handle — poll will retry + } + }, RUNNING_POLL_INTERVAL_MS); + return () => { + clearInterval(timer); + pollControllerRef.current?.abort(); + }; + }, [hasRunningRun, visibleCount, manualFetchInFlight]); + + const handleDelete = async () => { + if (!deleteTarget) return; + setDeleteError(null); + try { + await deleteTrainingRun(deleteTarget); + // Optimistically remove the card so it disappears immediately + setRuns((prev) => prev.filter((r) => r.id !== deleteTarget)); + setTotal((prev) => Math.max(0, prev - 1)); + // Re-fetch preserving visible count so offsets stay consistent for "Load more" + const currentCount = runs.length - 1; + const limit = Math.max(PAGE_SIZE, currentCount); + fetchRuns(0, false, limit).catch(() => { + // Refresh failed — card is already removed, no stale display + }); + } catch { + setDeleteError("Failed to delete training run. Please try again."); + } + setDeleteTarget(null); + }; + + if (!loading && error && runs.length === 0) { + return ( +
+

{error}

+ +
+ ); + } + + if (!loading && runs.length === 0) { + return ( +
+

+ No training runs yet. Start your first training run in the Configure + tab. +

+
+ ); + } + + return ( + <> + {deleteError && ( +
+ {deleteError} +
+ )} +
+ {runs.map((run) => { + const badge = statusBadge[run.status] ?? statusBadge.error; + const isRunning = run.status === "running"; + return ( +
onSelectRun(run.id)} + onKeyDown={(e) => { + if (e.key === "Enter" || e.key === " ") { + e.preventDefault(); + onSelectRun(run.id); + } + }} + > +
+ + {isRunning && } + {badge.label} + + + {formatRelativeTime(run.started_at)} + +
+
+

+ {run.model_name} +

+

+ {run.dataset_name} +

+
+ {run.loss_sparkline && run.loss_sparkline.length >= 2 && ( + + )} +
+ + Loss:{" "} + {run.final_loss != null ? run.final_loss.toFixed(4) : "--"} + + + Steps: {run.final_step ?? 0}/{run.total_steps ?? "--"} + + {formatDuration(run.duration_seconds)} +
+ {!isRunning && ( + + )} +
+ ); + })} +
+ {runs.length < total && ( +
+ +
+ )} + {loading && runs.length === 0 && ( +
+ {Array.from({ length: 3 }).map((_, i) => ( +
+ ))} +
+ )} + { + if (!open) setDeleteTarget(null); + }} + > + + + Delete training run? + + This will permanently delete this training run and all its metrics. + This action cannot be undone. + + + + Cancel + void handleDelete()} + > + Delete + + + + + + ); +} diff --git a/studio/frontend/src/features/studio/live-training-view.tsx b/studio/frontend/src/features/studio/live-training-view.tsx new file mode 100644 index 0000000000..9f930ce77b --- /dev/null +++ b/studio/frontend/src/features/studio/live-training-view.tsx @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { cn } from "@/lib/utils"; +import { + useTrainingConfigStore, + useTrainingRuntimeStore, +} from "@/features/training"; +import type { TrainingViewData } from "@/features/training"; +import type { ReactElement } from "react"; +import { useShallow } from "zustand/react/shallow"; +import { ChartsSection } from "./sections/charts-section"; +import { ProgressSection } from "./sections/progress-section"; +import { TrainingStartOverlay } from "./training-start-overlay"; + +export function LiveTrainingView(): ReactElement { + const runtime = useTrainingRuntimeStore( + useShallow((state) => ({ + jobId: state.jobId, + phase: state.phase, + message: state.message, + error: state.error, + currentStep: state.currentStep, + totalSteps: state.totalSteps, + currentEpoch: state.currentEpoch, + currentLoss: state.currentLoss, + currentLearningRate: state.currentLearningRate, + currentGradNorm: state.currentGradNorm, + currentNumTokens: state.currentNumTokens, + progressPercent: state.progressPercent, + elapsedSeconds: state.elapsedSeconds, + etaSeconds: state.etaSeconds, + evalEnabled: state.evalEnabled, + isTrainingRunning: state.isTrainingRunning, + lossHistory: state.lossHistory, + lrHistory: state.lrHistory, + gradNormHistory: state.gradNormHistory, + evalLossHistory: state.evalLossHistory, + firstStepReceived: state.firstStepReceived, + isStarting: state.isStarting, + })), + ); + + const config = useTrainingConfigStore( + useShallow((state) => ({ + selectedModel: state.selectedModel, + trainingMethod: state.trainingMethod, + })), + ); + + const viewData: TrainingViewData = { + phase: runtime.phase, + currentStep: runtime.currentStep, + totalSteps: runtime.totalSteps, + currentLoss: runtime.currentLoss, + currentLearningRate: runtime.currentLearningRate, + currentGradNorm: runtime.currentGradNorm, + currentEpoch: runtime.currentEpoch, + currentNumTokens: runtime.currentNumTokens, + progressPercent: runtime.progressPercent, + elapsedSeconds: runtime.elapsedSeconds, + etaSeconds: runtime.etaSeconds, + evalEnabled: runtime.evalEnabled, + message: runtime.message, + error: runtime.error, + isTrainingRunning: runtime.isTrainingRunning, + modelName: config.selectedModel ?? "", + trainingMethod: config.trainingMethod ?? "", + lossHistory: runtime.lossHistory, + lrHistory: runtime.lrHistory, + gradNormHistory: runtime.gradNormHistory, + evalLossHistory: runtime.evalLossHistory, + }; + + const isPreparingPhase = + runtime.phase === "downloading_model" || + runtime.phase === "downloading_dataset" || + runtime.phase === "loading_model" || + runtime.phase === "loading_dataset" || + runtime.phase === "configuring"; + const isWaitingForFirstStep = + runtime.phase === "training" && !runtime.firstStepReceived; + const showOverlay = + runtime.isStarting || + isPreparingPhase || + (isWaitingForFirstStep && runtime.currentStep <= 0); + + return ( +
+
+
+ +
+ +
+ {showOverlay ? ( + + ) : null} +
+ ); +} diff --git a/studio/frontend/src/features/studio/sections/charts-section.tsx b/studio/frontend/src/features/studio/sections/charts-section.tsx index a7dacf7e5b..4c7bf46d9d 100644 --- a/studio/frontend/src/features/studio/sections/charts-section.tsx +++ b/studio/frontend/src/features/studio/sections/charts-section.tsx @@ -1,7 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -import { useTrainingRuntimeStore } from "@/features/training"; +import type { TrainingSeriesPoint } from "@/features/training"; import { type ReactElement, Suspense, lazy, useMemo } from "react"; const ChartsContent = lazy(() => @@ -16,42 +16,49 @@ const SKELETON_KEYS = [ "chart-skeleton-4", ]; -export function ChartsSection(): ReactElement | null { - const currentStep = useTrainingRuntimeStore((state) => state.currentStep); - const totalSteps = useTrainingRuntimeStore((state) => state.totalSteps); - const isTraining = useTrainingRuntimeStore((state) => state.isTrainingRunning); - const evalEnabled = useTrainingRuntimeStore((state) => state.evalEnabled); - const lossHistoryRaw = useTrainingRuntimeStore((state) => state.lossHistory); - const lrHistoryRaw = useTrainingRuntimeStore((state) => state.lrHistory); - const gradNormHistoryRaw = useTrainingRuntimeStore( - (state) => state.gradNormHistory, - ); - const evalLossHistoryRaw = useTrainingRuntimeStore( - (state) => state.evalLossHistory, - ); +interface ChartsSectionProps { + currentStep: number; + totalSteps: number; + isTraining: boolean; + evalEnabled: boolean; + lossHistory: TrainingSeriesPoint[]; + lrHistory: TrainingSeriesPoint[]; + gradNormHistory: TrainingSeriesPoint[]; + evalLossHistory: TrainingSeriesPoint[]; +} +export function ChartsSection({ + currentStep, + totalSteps, + isTraining, + evalEnabled, + lossHistory, + lrHistory, + gradNormHistory, + evalLossHistory, +}: ChartsSectionProps): ReactElement | null { const series = useMemo( () => ({ currentStep, totalSteps, - lossHistory: lossHistoryRaw.map((point) => ({ + lossHistory: lossHistory.map((point) => ({ step: point.step, loss: point.value, })), - lrHistory: lrHistoryRaw.map((point) => ({ + lrHistory: lrHistory.map((point) => ({ step: point.step, lr: point.value, })), - gradNormHistory: gradNormHistoryRaw.map((point) => ({ + gradNormHistory: gradNormHistory.map((point) => ({ step: point.step, gradNorm: point.value, })), - evalLossHistory: evalLossHistoryRaw.map((point) => ({ + evalLossHistory: evalLossHistory.map((point) => ({ step: point.step, loss: point.value, })), }), - [currentStep, evalLossHistoryRaw, gradNormHistoryRaw, lossHistoryRaw, lrHistoryRaw, totalSteps], + [currentStep, evalLossHistory, gradNormHistory, lossHistory, lrHistory, totalSteps], ); if ( diff --git a/studio/frontend/src/features/studio/sections/progress-section.tsx b/studio/frontend/src/features/studio/sections/progress-section.tsx index da89ce66ee..8255283183 100644 --- a/studio/frontend/src/features/studio/sections/progress-section.tsx +++ b/studio/frontend/src/features/studio/sections/progress-section.tsx @@ -26,6 +26,7 @@ import { useTrainingConfigStore, useTrainingRuntimeStore, } from "@/features/training"; +import type { TrainingViewData } from "@/features/training"; import { useGpuUtilization } from "@/hooks"; import { cn } from "@/lib/utils"; import { @@ -39,7 +40,7 @@ import { } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { Link, useNavigate } from "@tanstack/react-router"; -import { type ReactElement, type ReactNode, useEffect, useState } from "react"; +import { type ReactElement, type ReactNode, useState } from "react"; import { useShallow } from "zustand/react/shallow"; import { ChartSettingsSheet } from "./charts/chart-settings-sheet"; import { @@ -61,34 +62,33 @@ function configRow( return [label, value]; } -export function ProgressSection(): ReactElement { +interface ProgressSectionProps { + data: TrainingViewData; + isHistorical?: boolean; + configOverride?: { + epochs?: number; + batchSize?: number; + learningRate?: string; + maxSteps?: number; + contextLength?: number; + warmupSteps?: number; + optimizerType?: string; + loraRank?: number; + loraAlpha?: number; + loraDropout?: number; + loraVariant?: string; + }; +} + +export function ProgressSection({ + data, + isHistorical = false, + configOverride, +}: ProgressSectionProps): ReactElement { const navigate = useNavigate(); - const runtime = useTrainingRuntimeStore( - useShallow((state) => ({ - phase: state.phase, - message: state.message, - error: state.error, - currentStep: state.currentStep, - totalSteps: state.totalSteps, - currentEpoch: state.currentEpoch, - currentLoss: state.currentLoss, - currentLearningRate: state.currentLearningRate, - currentGradNorm: state.currentGradNorm, - progressPercent: state.progressPercent, - elapsedSeconds: state.elapsedSeconds, - etaSeconds: state.etaSeconds, - currentNumTokens: state.currentNumTokens, - isTrainingRunning: state.isTrainingRunning, - lossHistory: state.lossHistory, - lrHistory: state.lrHistory, - gradNormHistory: state.gradNormHistory, - })), - ); const config = useTrainingConfigStore( useShallow((state) => ({ - selectedModel: state.selectedModel, - trainingMethod: state.trainingMethod, epochs: state.epochs, batchSize: state.batchSize, learningRate: state.learningRate, @@ -103,98 +103,92 @@ export function ProgressSection(): ReactElement { })), ); - const { stopTrainingRun } = useTrainingActions(); - const gpu = useGpuUtilization(runtime.isTrainingRunning); const [stopDialogOpen, setStopDialogOpen] = useState(false); - const [stopRequested, setStopRequested] = useState(false); + const [stopRequestedLocal, setStopRequestedLocal] = useState(false); - useEffect(() => { - if (!runtime.isTrainingRunning) { - setStopRequested(false); - } - }, [runtime.isTrainingRunning]); + // Auto-reset when training stops -- no useEffect needed + const stopRequested = data.isTrainingRunning && stopRequestedLocal; const pct = - runtime.totalSteps > 0 + data.totalSteps > 0 ? Math.min( 100, Math.max( 0, - Math.round((runtime.currentStep / runtime.totalSteps) * 100), + Math.round((data.currentStep / data.totalSteps) * 100), ), ) - : Math.round(runtime.progressPercent); + : Math.round(data.progressPercent); - const elapsed = runtime.elapsedSeconds; + const elapsed = data.elapsedSeconds; const derivedEta = elapsed != null && pct > 0 ? Math.round((elapsed * (100 - pct)) / Math.max(pct, 1)) : null; - const eta = runtime.etaSeconds ?? derivedEta; + const eta = data.etaSeconds ?? derivedEta; const stepsPerSecond = - elapsed != null && elapsed > 0 ? runtime.currentStep / elapsed : null; + elapsed != null && elapsed > 0 ? data.currentStep / elapsed : null; const showHalfwayHint = - runtime.phase === "training" && pct >= 50 && pct < 100; - const showCompletedHint = runtime.phase === "completed"; + data.phase === "training" && pct >= 50 && pct < 100; + const showCompletedHint = data.phase === "completed"; const handleCompareInChat = async () => { - setTrainingCompareHandoff(config.selectedModel); + setTrainingCompareHandoff(data.modelName); await navigate({ to: "/chat" }); }; - const requestStop = async (saveCheckpoint: boolean) => { - setStopRequested(true); - setStopDialogOpen(false); - useTrainingRuntimeStore.getState().setStopRequested(true); - try { - const ok = await stopTrainingRun(saveCheckpoint); - if (!ok) { - setStopRequested(false); - } - } catch { - setStopRequested(false); - } - }; const stoppedLoss = getDisplayMetric( - runtime.isTrainingRunning, - runtime.currentLoss, - runtime.lossHistory, + data.isTrainingRunning, + data.currentLoss, + data.lossHistory, ); const stoppedLr = getDisplayMetric( - runtime.isTrainingRunning, - runtime.currentLearningRate, - runtime.lrHistory, + data.isTrainingRunning, + data.currentLearningRate, + data.lrHistory, ); - const stoppedGradNorm = runtime.isTrainingRunning - ? runtime.currentGradNorm - : (lastNonZeroValue(runtime.gradNormHistory) ?? runtime.currentGradNorm); + const stoppedGradNorm = data.isTrainingRunning + ? data.currentGradNorm + : (lastValue(data.gradNormHistory) ?? data.currentGradNorm); + + const cfgEpochs = isHistorical ? configOverride?.epochs : config.epochs; + const cfgBatchSize = isHistorical ? configOverride?.batchSize : config.batchSize; + const cfgLearningRate = isHistorical ? configOverride?.learningRate : config.learningRate; + const cfgMaxSteps = isHistorical ? configOverride?.maxSteps : config.maxSteps; + const cfgContextLength = isHistorical ? configOverride?.contextLength : config.contextLength; + const cfgWarmupSteps = isHistorical ? configOverride?.warmupSteps : config.warmupSteps; + const cfgOptimizerType = isHistorical ? configOverride?.optimizerType : config.optimizerType; + const cfgLoraRank = isHistorical ? configOverride?.loraRank : config.loraRank; + const cfgLoraAlpha = isHistorical ? configOverride?.loraAlpha : config.loraAlpha; + const cfgLoraDropout = isHistorical ? configOverride?.loraDropout : config.loraDropout; + const cfgLoraVariant = isHistorical ? configOverride?.loraVariant : config.loraVariant; const optimizerLabel = - OPTIMIZER_OPTIONS.find((o) => o.value === config.optimizerType)?.label ?? - config.optimizerType; + OPTIMIZER_OPTIONS.find((o) => o.value === cfgOptimizerType)?.label ?? + cfgOptimizerType; const configItems: ConfigGroup[] = [ { section: "Hyperparams", rows: [ - configRow("Epochs", config.epochs), - configRow("Batch size", config.batchSize), - configRow("Learning rate", config.learningRate), + configRow("Epochs", cfgEpochs), + configRow("Batch size", cfgBatchSize), + configRow("Learning rate", cfgLearningRate), configRow("Optimizer", optimizerLabel), - configRow("Max steps", config.maxSteps), - configRow("Context length", config.contextLength), - configRow("Warmup steps", config.warmupSteps), + configRow("Max steps", cfgMaxSteps), + configRow("Context length", cfgContextLength), + configRow("Warmup steps", cfgWarmupSteps), ], }, - ...(config.trainingMethod !== "full" + ...(data.trainingMethod !== "full" ? [ { section: "LoRA", rows: [ - configRow("Rank", config.loraRank), - configRow("Alpha", config.loraAlpha), - configRow("Dropout", config.loraDropout), - configRow("Variant", config.loraVariant), + configRow("Rank", cfgLoraRank), + configRow("Alpha", cfgLoraAlpha), + configRow("Dropout", cfgLoraDropout), + configRow("Variant", cfgLoraVariant), ], }, ] @@ -205,30 +199,34 @@ export function ProgressSection(): ReactElement { } title="Training Progress" - description={runtime.message || "Live training metrics"} + description={data.message || "Live training metrics"} accent="emerald" className="shadow-border border border-border/60 bg-card/90 ring-0 backdrop-blur-sm" headerAction={ - + isHistorical ? ( + + ) : ( + + ) } >
- {phaseLabel[runtime.phase]} + {phaseLabel[data.phase]} - Epoch {runtime.currentEpoch.toFixed(2)} + Epoch {formatNumber(data.currentEpoch, 2)} {pct}% complete @@ -238,22 +236,24 @@ export function ProgressSection(): ReactElement {
- Step {runtime.currentStep} / {runtime.totalSteps || "--"} + Step {data.currentStep} / {data.totalSteps || "--"} {pct}%
- + {!isHistorical && ( + + )} - {runtime.error && ( + {data.error && (

- {runtime.error} + {data.error}

)} @@ -262,97 +262,196 @@ export function ProgressSection(): ReactElement { label="Loss" valueClassName="text-2xl font-bold tracking-tight" > - {stoppedLoss.toFixed(4)} + {stoppedLoss != null ? stoppedLoss.toFixed(4) : "--"} - {stoppedLr.toExponential(2)} + {stoppedLr != null ? stoppedLr.toExponential(2) : "--"} {formatNumber(stoppedGradNorm, 3)} - {config.selectedModel ?? "--"} + {data.modelName || "--"} - {config.trainingMethod === "qlora" ? "QLoRA" : config.trainingMethod === "lora" ? "LoRA" : "Full"} + {data.trainingMethod === "qlora" ? "QLoRA" : data.trainingMethod === "lora" ? "LoRA" : "Full"}
Elapsed: {formatDuration(elapsed)} - ETA: {formatDuration(eta)} + {!isHistorical && ETA: {formatDuration(eta)}} {stepsPerSecond == null ? "-- steps/s" : `${stepsPerSecond.toFixed(2)} steps/s`} - {runtime.currentNumTokens != null && ( - Tokens: {runtime.currentNumTokens} + {data.currentNumTokens != null && ( + Tokens: {data.currentNumTokens} )}
-
-
-

- GPU Monitor -

- Live -
-
- - } - value={ - gpu.gpu_utilization_pct != null - ? `${gpu.gpu_utilization_pct}%` - : "--" - } - pct={gpu.gpu_utilization_pct ?? 0} - /> - - } - value={ - gpu.temperature_c != null ? `${gpu.temperature_c}°C` : "--" - } - pct={gpu.temperature_c ?? 0} - max={100} - /> - } - value={ - gpu.vram_used_gb != null && gpu.vram_total_gb != null - ? `${gpu.vram_used_gb} / ${gpu.vram_total_gb} GB` - : "--" - } - pct={gpu.vram_utilization_pct ?? 0} - /> - } - value={ - gpu.power_draw_w != null - ? gpu.power_limit_w != null - ? `${gpu.power_draw_w} / ${gpu.power_limit_w} W` - : `${gpu.power_draw_w} W` - : "--" - } - pct={gpu.power_utilization_pct ?? 0} - /> -
-
+ {!isHistorical && ( + + )}
); } +function LiveGpuPanel({ + isTrainingRunning, +}: { + isTrainingRunning: boolean; +}): ReactElement { + const gpu = useGpuUtilization(isTrainingRunning); + + return ( +
+
+

+ GPU Monitor +

+ Live +
+
+ + } + value={ + gpu.gpu_utilization_pct != null + ? `${gpu.gpu_utilization_pct}%` + : "--" + } + pct={gpu.gpu_utilization_pct ?? 0} + /> + + } + value={ + gpu.temperature_c != null ? `${gpu.temperature_c}°C` : "--" + } + pct={gpu.temperature_c ?? 0} + max={100} + /> + } + value={ + gpu.vram_used_gb != null && gpu.vram_total_gb != null + ? `${gpu.vram_used_gb} / ${gpu.vram_total_gb} GB` + : "--" + } + pct={gpu.vram_utilization_pct ?? 0} + /> + } + value={ + gpu.power_draw_w != null + ? gpu.power_limit_w != null + ? `${gpu.power_draw_w} / ${gpu.power_limit_w} W` + : `${gpu.power_draw_w} W` + : "--" + } + pct={gpu.power_utilization_pct ?? 0} + /> +
+
+ ); +} + +function LiveTrainingHeaderActions({ + configItems, + isTrainingRunning, + onOpenStopDialog, + stopDialogOpen, + stopRequested, + onSetStopRequested, +}: { + configItems: ConfigGroup[]; + isTrainingRunning: boolean; + onOpenStopDialog: (open: boolean) => void; + stopDialogOpen: boolean; + stopRequested: boolean; + onSetStopRequested: (v: boolean) => void; +}): ReactElement { + const { stopTrainingRun } = useTrainingActions(); + + const requestStop = async (saveCheckpoint: boolean) => { + onSetStopRequested(true); + onOpenStopDialog(false); + useTrainingRuntimeStore.getState().setStopRequested(true); + try { + const ok = await stopTrainingRun(saveCheckpoint); + if (!ok) { + onSetStopRequested(false); + } + } catch { + onSetStopRequested(false); + } + }; + + return ( + + ); +} + +function ConfigPopoverButton({ + configItems, +}: { + configItems: ConfigGroup[]; +}): ReactElement { + return ( + + + + + +
+

Training Config

+ {configItems.map((group) => ( +
+

+ {group.section} +

+ {group.rows.map(([label, value]) => ( +
+ {label} + + {value == null || value === "" ? "--" : String(value)} + +
+ ))} +
+ ))} +
+
+
+ ); +} + function TrainingHeaderActions({ configItems, isTrainingRunning, @@ -370,39 +469,7 @@ function TrainingHeaderActions({ }): ReactElement { return (
- - - - - -
-

Training Config

- {configItems.map((group) => ( -
-

- {group.section} -

- {group.rows.map(([label, value]) => ( -
- {label} - - {String(value)} - -
- ))} -
- ))} -
-
-
+ - )} -

Fine-tuning Studio

-

- {showTrainingView - ? runtimeMessage || "Training in progress" - : "Configure and start training"} -

+

{subtitle}

{!hasHydratedRuntime && isHydratingRuntime ? (
Loading training runtime...
- ) : showTrainingView ? ( - ) : ( -
- - - - -
+ +
+ {selectedHistoryRunId && activeTab === "history" && ( + + )} + + + Configure + + + Current Run + + History + +
+ + +
+ + + + +
+
+ + + + + + + {selectedHistoryRunId ? ( + + ) : ( + { + if (runId === currentJobId && isTrainingRunning) { + handleTabChange("current-run"); + } else { + setSelectedHistoryRunId(runId); + } + }} /> + )} + +
)}
diff --git a/studio/frontend/src/features/studio/training-view.tsx b/studio/frontend/src/features/studio/training-view.tsx deleted file mode 100644 index b995b2f528..0000000000 --- a/studio/frontend/src/features/studio/training-view.tsx +++ /dev/null @@ -1,57 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only -// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 - -import { cn } from "@/lib/utils"; -import { useTrainingRuntimeStore } from "@/features/training"; -import type { ReactElement } from "react"; -import { useShallow } from "zustand/react/shallow"; -import { ChartsSection } from "./sections/charts-section"; -import { ProgressSection } from "./sections/progress-section"; -import { TrainingStartOverlay } from "./training-start-overlay"; - -export function TrainingView(): ReactElement { - const runtime = useTrainingRuntimeStore( - useShallow((state) => ({ - phase: state.phase, - message: state.message, - currentStep: state.currentStep, - firstStepReceived: state.firstStepReceived, - isStarting: state.isStarting, - })), - ); - - const isPreparingPhase = - runtime.phase === "downloading_model" || - runtime.phase === "downloading_dataset" || - runtime.phase === "loading_model" || - runtime.phase === "loading_dataset" || - runtime.phase === "configuring"; - const isWaitingForFirstStep = - runtime.phase === "training" && !runtime.firstStepReceived; - const showOverlay = - runtime.isStarting || - isPreparingPhase || - (isWaitingForFirstStep && runtime.currentStep <= 0); - - return ( -
-
-
- -
- -
- {showOverlay ? ( - - ) : null} -
- ); -} diff --git a/studio/frontend/src/features/training/api/history-api.ts b/studio/frontend/src/features/training/api/history-api.ts new file mode 100644 index 0000000000..8f279eb439 --- /dev/null +++ b/studio/frontend/src/features/training/api/history-api.ts @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { authFetch } from "@/features/auth"; +import type { + TrainingRunDeleteResponse, + TrainingRunDetailResponse, + TrainingRunListResponse, +} from "../types/history"; + +async function readError(response: Response): Promise { + try { + const payload = (await response.json()) as { detail?: string; message?: string }; + return payload.detail || payload.message || `Request failed (${response.status})`; + } catch { + return `Request failed (${response.status})`; + } +} + +async function parseJson(response: Response): Promise { + if (!response.ok) { + throw new Error(await readError(response)); + } + return (await response.json()) as T; +} + +export async function listTrainingRuns( + limit = 50, + offset = 0, + signal?: AbortSignal, +): Promise { + const response = await authFetch( + `/api/train/runs?limit=${limit}&offset=${offset}`, + { signal }, + ); + return parseJson(response); +} + +export async function getTrainingRun( + runId: string, + signal?: AbortSignal, +): Promise { + const response = await authFetch( + `/api/train/runs/${encodeURIComponent(runId)}`, + { signal }, + ); + return parseJson(response); +} + +export async function deleteTrainingRun( + runId: string, + signal?: AbortSignal, +): Promise { + const response = await authFetch( + `/api/train/runs/${encodeURIComponent(runId)}`, + { method: "DELETE", signal }, + ); + return parseJson(response); +} diff --git a/studio/frontend/src/features/training/index.ts b/studio/frontend/src/features/training/index.ts index 9a70e0a71c..af34b306cf 100644 --- a/studio/frontend/src/features/training/index.ts +++ b/studio/frontend/src/features/training/index.ts @@ -14,6 +14,14 @@ export { useDatasetPreviewDialogStore } from "./stores/dataset-preview-dialog-st export { uploadTrainingDataset } from "./api/datasets-api"; export { listLocalModels } from "./api/models-api"; export type { LocalModelInfo } from "./api/models-api"; -export type { TrainingPhase } from "./types/runtime"; +export type { TrainingPhase, TrainingViewData, TrainingSeriesPoint } from "./types/runtime"; +export type { + TrainingRunSummary, + TrainingRunListResponse, + TrainingRunMetrics, + TrainingRunDetailResponse, + TrainingRunDeleteResponse, +} from "./types/history"; +export { listTrainingRuns, getTrainingRun, deleteTrainingRun } from "./api/history-api"; export { parseYamlConfig, serializeConfigToYaml } from "./lib/yaml-config"; export { validateTrainingConfig } from "./lib/validation"; diff --git a/studio/frontend/src/features/training/types/history.ts b/studio/frontend/src/features/training/types/history.ts new file mode 100644 index 0000000000..8b89db539b --- /dev/null +++ b/studio/frontend/src/features/training/types/history.ts @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +export interface TrainingRunSummary { + id: string; + status: "running" | "completed" | "stopped" | "error"; + model_name: string; + dataset_name: string; + started_at: string; + ended_at: string | null; + total_steps: number | null; + final_step: number | null; + final_loss: number | null; + output_dir: string | null; + duration_seconds: number | null; + error_message: string | null; + loss_sparkline: number[] | null; +} + +export interface TrainingRunListResponse { + runs: TrainingRunSummary[]; + total: number; +} + +export interface TrainingRunMetrics { + step_history: number[]; + loss_history: number[]; + loss_step_history: number[]; + lr_history: number[]; + lr_step_history: number[]; + grad_norm_history: number[]; + grad_norm_step_history: number[]; + eval_loss_history: number[]; + eval_step_history: number[]; + final_epoch: number | null; + final_num_tokens: number | null; +} + +export interface TrainingRunDetailResponse { + run: TrainingRunSummary; + config: Record; + metrics: TrainingRunMetrics; +} + +export interface TrainingRunDeleteResponse { + status: string; + message: string; +} diff --git a/studio/frontend/src/features/training/types/runtime.ts b/studio/frontend/src/features/training/types/runtime.ts index 7669c8b2f3..1bf319a5d1 100644 --- a/studio/frontend/src/features/training/types/runtime.ts +++ b/studio/frontend/src/features/training/types/runtime.ts @@ -53,8 +53,8 @@ export interface TrainingProgressPayload { job_id: string; step: number; total_steps: number; - loss: number; - learning_rate: number; + loss: number | null; + learning_rate: number | null; progress_percent: number; epoch: number | null; elapsed_seconds: number | null; @@ -118,3 +118,32 @@ export interface TrainingRuntimeActions { } export type TrainingRuntimeStore = TrainingRuntimeState & TrainingRuntimeActions; + +export interface TrainingViewData { + // Current metrics (for ProgressSection) + phase: TrainingPhase; + currentStep: number; + totalSteps: number; + currentLoss: number | null; + currentLearningRate: number | null; + currentGradNorm: number | null; + currentEpoch: number | null; + currentNumTokens: number | null; + progressPercent: number; + elapsedSeconds: number | null; + etaSeconds: number | null; + evalEnabled: boolean; + message: string; + error: string | null; + isTrainingRunning: boolean; + + // Config summary + modelName: string; + trainingMethod: string; + + // Time-series (for ChartsSection) + lossHistory: TrainingSeriesPoint[]; + lrHistory: TrainingSeriesPoint[]; + gradNormHistory: TrainingSeriesPoint[]; + evalLossHistory: TrainingSeriesPoint[]; +}