diff --git a/studio/backend/routes/export.py b/studio/backend/routes/export.py index 6b335bc139..7f18f50eaa 100644 --- a/studio/backend/routes/export.py +++ b/studio/backend/routes/export.py @@ -86,6 +86,14 @@ async def load_checkpoint( if trn.is_training_active(): logger.info("Stopping active training to free GPU memory for export") trn.stop_training() + # Wait for training subprocess to actually exit before proceeding, + # otherwise it may still hold GPU memory when export tries to load. + for _ in range(60): # up to 30s + if not trn.is_training_active(): + break + import time; time.sleep(0.5) + else: + logger.warning("Training subprocess did not exit within 30s, proceeding anyway") except Exception as e: logger.warning("Could not stop training: %s", e) diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py index c0ee6deff7..84315291a9 100644 --- a/studio/backend/routes/training.py +++ b/studio/backend/routes/training.py @@ -304,6 +304,7 @@ async def reset_training( ) logger.info("Reset training state: clearing runtime + metric history") + backend._should_stop = False # Clear stop flag so status returns to idle backend.trainer._update_progress( is_training=False, is_completed=False, error=None, status_message="Ready to train", step=0, loss=0.0, epoch=0,