From fd7ca8bda8116805b2de18273f30adbb11dfcb84 Mon Sep 17 00:00:00 2001 From: Manan17 Date: Tue, 10 Mar 2026 02:35:32 +0000 Subject: [PATCH] distinguish cancel and stop for force terminate --- studio/backend/core/training/training.py | 4 ++++ studio/backend/routes/training.py | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.py index 47ed0c88cf..14ff57bbd7 100644 --- a/studio/backend/core/training/training.py +++ b/studio/backend/core/training/training.py @@ -71,6 +71,7 @@ class TrainingBackend: # Progress state (updated by pump thread from subprocess events) self._progress = TrainingProgress() self._should_stop = False + self._cancel_requested = False # True only for stop(save=False) # Training Metrics (consumed by routes for SSE and /metrics) self.loss_history: list = [] @@ -114,6 +115,7 @@ class TrainingBackend: # Reset state self._should_stop = False + self._cancel_requested = False self._progress = TrainingProgress(is_training=True, status_message="Initializing training...") self.loss_history.clear() self.lr_history.clear() @@ -213,6 +215,8 @@ class TrainingBackend: def stop_training(self, save: bool = True) -> bool: """Send stop signal to the training subprocess.""" self._should_stop = True + if not save: + self._cancel_requested = True with self._lock: if self._stop_queue is not None: try: diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py index 1590c0864d..6de29c4882 100644 --- a/studio/backend/routes/training.py +++ b/studio/backend/routes/training.py @@ -309,9 +309,9 @@ async def reset_training( is_active = backend.is_training_active() if is_active: - if backend._should_stop: - # Stop was already requested — force-terminate so we can reset immediately - logger.info("Force-terminating subprocess for immediate reset") + if backend._cancel_requested: + # Cancel (save=False) was requested — force-terminate so we can reset immediately + logger.info("Force-terminating subprocess for immediate reset (cancel path)") backend.force_terminate() else: logger.warning("Rejected reset while training active: is_active=%s", is_active)