diff --git a/studio/frontend/src/features/studio/sections/progress-section.tsx b/studio/frontend/src/features/studio/sections/progress-section.tsx index b8656a7867..f1a10c2684 100644 --- a/studio/frontend/src/features/studio/sections/progress-section.tsx +++ b/studio/frontend/src/features/studio/sections/progress-section.tsx @@ -1,4 +1,14 @@ import { SectionCard } from "@/components/section-card"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; import { Button } from "@/components/ui/button"; import { Popover, @@ -104,6 +114,7 @@ export function ProgressSection(): ReactElement { ); const { stopTrainingRun } = useTrainingActions(); + const [stopDialogOpen, setStopDialogOpen] = useState(false); const localStartAtRef = useRef(null); const [, setLocalTick] = useState(0); @@ -225,15 +236,39 @@ export function ProgressSection(): ReactElement { - + + + + + Stop Training + + Choose how you want to stop the current training run. + + + + Continue Training + void stopTrainingRun(false)} + > + Cancel Training + + void stopTrainingRun(true)} + > + Stop and Save + + + + } > diff --git a/studio/frontend/src/features/studio/studio-page.tsx b/studio/frontend/src/features/studio/studio-page.tsx index d5e20fc194..5811f161db 100644 --- a/studio/frontend/src/features/studio/studio-page.tsx +++ b/studio/frontend/src/features/studio/studio-page.tsx @@ -1,8 +1,12 @@ +import { Button } from "@/components/ui/button"; import { shouldShowTrainingView, + useTrainingActions, useTrainingRuntimeLifecycle, useTrainingRuntimeStore, } from "@/features/training"; +import { ArrowLeft01Icon } from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; import type { ReactElement } from "react"; import { DatasetSection } from "./sections/dataset-section"; import { ModelSection } from "./sections/model-section"; @@ -14,12 +18,28 @@ export function StudioPage(): ReactElement { useTrainingRuntimeLifecycle(); const showTrainingView = useTrainingRuntimeStore(shouldShowTrainingView); const runtimeMessage = useTrainingRuntimeStore((state) => state.message); + const runtimePhase = useTrainingRuntimeStore((state) => state.phase); const isHydratingRuntime = useTrainingRuntimeStore((state) => state.isHydrating); const hasHydratedRuntime = useTrainingRuntimeStore((state) => state.hasHydrated); + const { dismissTrainingRun } = useTrainingActions(); + + const canGoBack = runtimePhase === "stopped" || runtimePhase === "error"; return (
+ {canGoBack && ( + + )} + {/* Header */}

diff --git a/studio/frontend/src/features/training/api/train-api.ts b/studio/frontend/src/features/training/api/train-api.ts index e3c53b0bc2..d2f589c298 100644 --- a/studio/frontend/src/features/training/api/train-api.ts +++ b/studio/frontend/src/features/training/api/train-api.ts @@ -41,11 +41,22 @@ export async function startTraining( return parseJson(response); } -export async function stopTraining(): Promise { - const response = await authFetch("/api/train/stop", { method: "POST" }); +export async function stopTraining(save = true): Promise { + const response = await authFetch("/api/train/stop", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ save }), + }); return parseJson(response); } +export async function resetTraining(): Promise { + const response = await authFetch("/api/train/reset", { method: "POST" }); + if (!response.ok) { + throw new Error(await readError(response)); + } +} + export async function getTrainingStatus(): Promise { const response = await authFetch("/api/train/status"); return parseJson(response); diff --git a/studio/frontend/src/features/training/hooks/use-training-actions.ts b/studio/frontend/src/features/training/hooks/use-training-actions.ts index 510c45ed4a..4ace9dd5ed 100644 --- a/studio/frontend/src/features/training/hooks/use-training-actions.ts +++ b/studio/frontend/src/features/training/hooks/use-training-actions.ts @@ -1,7 +1,7 @@ import { useCallback } from "react"; import { useTrainingConfigStore } from "../stores/training-config-store"; import { useTrainingRuntimeStore } from "../stores/training-runtime-store"; -import { startTraining, stopTraining } from "../api/train-api"; +import { startTraining, stopTraining, resetTraining } from "../api/train-api"; import { buildTrainingStartPayload } from "../api/mappers"; import { syncTrainingRuntimeFromBackend } from "../lib/sync-runtime"; import { validateTrainingConfig } from "../lib/validation"; @@ -45,12 +45,12 @@ export function useTrainingActions() { } }, []); - const stopTrainingRun = useCallback(async (): Promise => { + const stopTrainingRun = useCallback(async (save = true): Promise => { const runtimeStore = useTrainingRuntimeStore.getState(); runtimeStore.setStartError(null); try { - await stopTraining(); + await stopTraining(save); await syncTrainingRuntimeFromBackend(); return true; } catch (error) { @@ -61,10 +61,20 @@ export function useTrainingActions() { } }, []); + const dismissTrainingRun = useCallback(async (): Promise => { + useTrainingRuntimeStore.getState().resetRuntime(); + try { + await resetTraining(); + } catch { + // Frontend already reset; backend will catch up on next poll + } + }, []); + return { isStarting, startError, startTrainingRun, stopTrainingRun, + dismissTrainingRun, }; } diff --git a/studio/frontend/src/features/training/hooks/use-training-runtime-lifecycle.ts b/studio/frontend/src/features/training/hooks/use-training-runtime-lifecycle.ts index 3baff3a62d..8d0eb9c756 100644 --- a/studio/frontend/src/features/training/hooks/use-training-runtime-lifecycle.ts +++ b/studio/frontend/src/features/training/hooks/use-training-runtime-lifecycle.ts @@ -48,9 +48,10 @@ export function useTrainingRuntimeLifecycle(): void { }; const pollMetrics = async () => { + const gen = runtimeStore.getState().resetGeneration; try { const metrics = await getTrainingMetrics(); - if (disposed) { + if (disposed || runtimeStore.getState().resetGeneration !== gen) { return; } runtimeStore.getState().applyMetrics(metrics); @@ -62,9 +63,10 @@ export function useTrainingRuntimeLifecycle(): void { }; const pollStatus = async () => { + const gen = runtimeStore.getState().resetGeneration; try { const status = await getTrainingStatus(); - if (disposed) { + if (disposed || runtimeStore.getState().resetGeneration !== gen) { return; } diff --git a/studio/frontend/src/features/training/lib/sync-runtime.ts b/studio/frontend/src/features/training/lib/sync-runtime.ts index b5fbd0bafb..bf255f58c6 100644 --- a/studio/frontend/src/features/training/lib/sync-runtime.ts +++ b/studio/frontend/src/features/training/lib/sync-runtime.ts @@ -6,12 +6,17 @@ import { useTrainingRuntimeStore } from "../stores/training-runtime-store"; import type { TrainingStatusResponse } from "../types/runtime"; export async function syncTrainingRuntimeFromBackend(): Promise { + const gen = useTrainingRuntimeStore.getState().resetGeneration; + const [status, metrics] = await Promise.all([ getTrainingStatus(), getTrainingMetrics(), ]); const runtimeStore = useTrainingRuntimeStore.getState(); + if (runtimeStore.resetGeneration !== gen) { + return status; + } runtimeStore.applyStatus(status); runtimeStore.applyMetrics(metrics); diff --git a/studio/frontend/src/features/training/stores/training-runtime-store.ts b/studio/frontend/src/features/training/stores/training-runtime-store.ts index bc40019346..94db0f28f5 100644 --- a/studio/frontend/src/features/training/stores/training-runtime-store.ts +++ b/studio/frontend/src/features/training/stores/training-runtime-store.ts @@ -34,6 +34,7 @@ const initialState: TrainingRuntimeState = { lossHistory: [], lrHistory: [], gradNormHistory: [], + resetGeneration: 0, }; function sortSeries(points: TrainingSeriesPoint[]): TrainingSeriesPoint[] { @@ -95,12 +96,13 @@ export const useTrainingRuntimeStore = create()((set) => ( setLastEventId: (value) => set({ lastEventId: value }), resetRuntime: () => - set({ + set((state) => ({ ...initialState, lossHistory: [], lrHistory: [], gradNormHistory: [], - }), + resetGeneration: state.resetGeneration + 1, + })), setStartQueued: (jobId, message) => set({ diff --git a/studio/frontend/src/features/training/types/runtime.ts b/studio/frontend/src/features/training/types/runtime.ts index fe2afbd36d..389418680a 100644 --- a/studio/frontend/src/features/training/types/runtime.ts +++ b/studio/frontend/src/features/training/types/runtime.ts @@ -82,6 +82,7 @@ export interface TrainingRuntimeState { lossHistory: TrainingSeriesPoint[]; lrHistory: TrainingSeriesPoint[]; gradNormHistory: TrainingSeriesPoint[]; + resetGeneration: number; } export interface TrainingRuntimeActions {