ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

distinguish cancel and stop for force terminate

Browse source
This commit is contained in:
Manan17 2026-03-10 02:35:32 +00:00
commit fd7ca8bda8
2 changed files with 7 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

4
studio/backend/core/training/training.py
View file

@ -71,6 +71,7 @@ class TrainingBackend:
# Progress state (updated by pump thread from subprocess events)
self._progress = TrainingProgress()
self._should_stop = False
self._cancel_requested = False # True only for stop(save=False)
# Training Metrics (consumed by routes for SSE and /metrics)
self.loss_history: list = []
@ -114,6 +115,7 @@ class TrainingBackend:
# Reset state
self._should_stop = False
self._cancel_requested = False
self._progress = TrainingProgress(is_training=True, status_message="Initializing training...")
self.loss_history.clear()
self.lr_history.clear()
@ -213,6 +215,8 @@ class TrainingBackend:
def stop_training(self, save: bool = True) -> bool:
"""Send stop signal to the training subprocess."""
self._should_stop = True
if not save:
self._cancel_requested = True
with self._lock:
if self._stop_queue is not None:
try:

6
studio/backend/routes/training.py
View file

@ -309,9 +309,9 @@ async def reset_training(
is_active = backend.is_training_active()
if is_active:
if backend._should_stop:
# Stop was already requested — force-terminate so we can reset immediately
logger.info("Force-terminating subprocess for immediate reset")
if backend._cancel_requested:
# Cancel (save=False) was requested — force-terminate so we can reset immediately
logger.info("Force-terminating subprocess for immediate reset (cancel path)")
backend.force_terminate()
else:
logger.warning("Rejected reset while training active: is_active=%s", is_active)