feat: enhance training stop and reset flow with detailed checks

This commit is contained in:
Shine1i 2026-02-17 23:32:22 +01:00
commit dc0cec772d
7 changed files with 112 additions and 27 deletions

View file

@ -586,8 +586,14 @@ class UnslothTrainer:
)
self.should_stop = False
self.training_thread.start()
return True
self.is_training = True
try:
self.training_thread.start()
return True
except Exception as e:
self.is_training = False
logger.error(f"Failed to start training thread: {e}")
return False
def _train_worker(self, dataset: Dataset, **training_args):
"""Worker function for training (runs in separate thread)"""
@ -990,9 +996,12 @@ class UnslothTrainer:
print(f"\nStopping training (save={save})...")
self.should_stop = True
self.save_on_stop = save
self.is_training = False
# Clear the status message so timer doesn't show stale status
self._update_progress(is_training=False, status_message="")
stop_msg = (
"Stopping training and saving checkpoint..."
if save
else "Cancelling training..."
)
self._update_progress(status_message=stop_msg)
# If trainer exists, try to stop it gracefully
if self.trainer:

View file

@ -369,8 +369,13 @@ class TrainingBackend:
True if training is in progress, False otherwise
"""
try:
# If user requested stop, training is no longer considered active
if self.trainer.should_stop:
training_thread = getattr(self.trainer, "training_thread", None)
if training_thread and training_thread.is_alive():
return True
# Stop requested and worker already exited => inactive.
# This allows UI to show stopped state + "Back to configuration".
if getattr(self.trainer, "should_stop", False):
return False
progress = self.trainer.get_training_progress()
@ -381,7 +386,23 @@ class TrainingBackend:
# but haven't completed or errored yet
if not is_active and not progress.is_completed and not progress.error:
status = progress.status_message or ""
if any(keyword in status.lower() for keyword in ["loading", "preparing", "training"]):
status_lower = status.lower()
if any(
keyword in status_lower
for keyword in ["cancelled", "canceled", "stopped", "completed", "ready to train"]
):
return False
if any(
keyword in status_lower
for keyword in [
"loading",
"preparing",
"training",
"configuring",
"tokenizing",
"starting",
]
):
is_active = True
return is_active
except Exception as e:

View file

@ -288,21 +288,31 @@ async def stop_training(
"""
try:
backend = get_training_backend()
if not backend.is_training_active():
trainer_thread = getattr(getattr(backend, "trainer", None), "training_thread", None)
thread_alive = bool(trainer_thread and trainer_thread.is_alive())
is_active = backend.is_training_active()
logger.info(
"Stop requested: save=%s is_active=%s thread_alive=%s should_stop=%s",
body.save,
is_active,
thread_alive,
getattr(getattr(backend, "trainer", None), "should_stop", None),
)
if not is_active and not thread_alive:
return TrainingStopResponse(
status="idle",
message="No training job is currently running"
)
# Call backend stop method
backend.stop_training(save=body.save)
return TrainingStopResponse(
status="stopped",
message="Training job stopped successfully"
message="Stop requested. Training will stop at the next safe step."
)
except Exception as e:
logger.error(f"Error stopping training: {e}", exc_info=True)
raise HTTPException(
@ -320,6 +330,23 @@ async def reset_training(
"""
try:
backend = get_training_backend()
trainer_thread = getattr(getattr(backend, "trainer", None), "training_thread", None)
thread_alive = bool(trainer_thread and trainer_thread.is_alive())
is_active = backend.is_training_active()
if is_active or thread_alive:
logger.warning(
"Rejected reset while training active: is_active=%s thread_alive=%s should_stop=%s",
is_active,
thread_alive,
getattr(getattr(backend, "trainer", None), "should_stop", None),
)
raise HTTPException(
status_code=409,
detail="Training is still running. Stop training and wait for it to finish before resetting.",
)
logger.info("Reset training state: clearing runtime + metric history")
backend.trainer.should_stop = False
backend.trainer.training_progress = backend.trainer.training_progress.__class__()
backend.loss_history = []
@ -328,6 +355,8 @@ async def reset_training(
backend.grad_norm_history = []
backend.grad_norm_step_history = []
return {"status": "ok"}
except HTTPException:
raise
except Exception as e:
logger.error(f"Error resetting training: {e}", exc_info=True)
raise HTTPException(

View file

@ -25,6 +25,7 @@ const STUDIO_TOUR_KEY = "tour:studio:v1";
export function StudioPage(): ReactElement {
useTrainingRuntimeLifecycle();
const showTrainingView = useTrainingRuntimeStore(shouldShowTrainingView);
const isTrainingRunning = useTrainingRuntimeStore((state) => state.isTrainingRunning);
const runtimeMessage = useTrainingRuntimeStore((state) => state.message);
const runtimePhase = useTrainingRuntimeStore((state) => state.phase);
const isHydratingRuntime = useTrainingRuntimeStore((state) => state.isHydrating);
@ -41,7 +42,11 @@ export function StudioPage(): ReactElement {
const dialogInitial = useDatasetPreviewDialogStore((s) => s.initialData);
const closeDialog = useDatasetPreviewDialogStore((s) => s.close);
const canGoBack = runtimePhase === "stopped" || runtimePhase === "error" || runtimePhase === "completed";
const canGoBack =
showTrainingView &&
!isTrainingRunning &&
!isHydratingRuntime &&
(runtimePhase === "stopped" || runtimePhase === "error" || runtimePhase === "completed" || runtimePhase === "idle");
const tourEnabled = hasHydratedRuntime && !isHydratingRuntime;
const isConfigTour = !showTrainingView;
const tourSteps = showTrainingView ? studioTrainingTourSteps : studioTourSteps;

View file

@ -8,6 +8,7 @@ import { useDatasetPreviewDialogStore } from "../stores/dataset-preview-dialog-s
import { useTrainingConfigStore } from "../stores/training-config-store";
import { useTrainingRuntimeStore } from "../stores/training-runtime-store";
import type { TrainingConfigState } from "../types/config";
import { toast } from "sonner";
export function useTrainingActions() {
const isStarting = useTrainingRuntimeStore((state) => state.isStarting);
@ -99,11 +100,18 @@ export function useTrainingActions() {
}, []);
const dismissTrainingRun = useCallback(async (): Promise<void> => {
useTrainingRuntimeStore.getState().resetRuntime();
try {
await resetTraining();
} catch {
// Frontend already reset; backend will catch up on next poll
useTrainingRuntimeStore.getState().resetRuntime();
} catch (error) {
const message =
error instanceof Error
? error.message
: "Stop training first, then return to configuration.";
toast.error("Training still active", {
description: message,
});
await syncTrainingRuntimeFromBackend();
}
}, []);

View file

@ -14,13 +14,7 @@ const METRICS_POLL_INTERVAL_MS = 5000;
const STREAM_RECONNECT_DELAY_MS = 1500;
function shouldUseLiveSync(state: TrainingRuntimeStore): boolean {
return (
state.isTrainingRunning ||
state.phase === "loading_model" ||
state.phase === "loading_dataset" ||
state.phase === "configuring" ||
state.phase === "training"
);
return state.isTrainingRunning || state.phase === "training";
}
export function useTrainingRuntimeLifecycle(): void {

View file

@ -128,14 +128,33 @@ export const useTrainingRuntimeStore = create<TrainingRuntimeStore>()((set) => (
})),
setStartQueued: (jobId, message) =>
set({
set((state) => ({
...state,
jobId,
message,
error: null,
startError: null,
phase: "configuring",
isStarting: false,
}),
sseConnected: false,
firstStepReceived: false,
lastEventId: null,
currentStep: 0,
totalSteps: 0,
currentEpoch: 0,
currentLoss: 0,
currentLearningRate: 0,
progressPercent: 0,
elapsedSeconds: null,
etaSeconds: null,
currentGradNorm: null,
currentNumTokens: null,
lossHistory: [],
lrHistory: [],
gradNormHistory: [],
evalLossHistory: [],
resetGeneration: state.resetGeneration + 1,
})),
setRuntimeError: (message) =>
set({