fix: wait for training shutdown before export load, clear stop flag on reset
1. Export route: stop_training() only signals the subprocess — wait up to 30s for it to actually exit before loading the export checkpoint, avoiding a GPU memory race. 2. Training reset: clear _should_stop so /api/train/status returns phase=idle instead of staying stuck on phase=stopped after a user-triggered stop. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
f45f1f74c0
commit
25b51fad3b
2 changed files with 9 additions and 0 deletions
|
|
@ -86,6 +86,14 @@ async def load_checkpoint(
|
|||
if trn.is_training_active():
|
||||
logger.info("Stopping active training to free GPU memory for export")
|
||||
trn.stop_training()
|
||||
# Wait for training subprocess to actually exit before proceeding,
|
||||
# otherwise it may still hold GPU memory when export tries to load.
|
||||
for _ in range(60): # up to 30s
|
||||
if not trn.is_training_active():
|
||||
break
|
||||
import time; time.sleep(0.5)
|
||||
else:
|
||||
logger.warning("Training subprocess did not exit within 30s, proceeding anyway")
|
||||
except Exception as e:
|
||||
logger.warning("Could not stop training: %s", e)
|
||||
|
||||
|
|
|
|||
|
|
@ -304,6 +304,7 @@ async def reset_training(
|
|||
)
|
||||
|
||||
logger.info("Reset training state: clearing runtime + metric history")
|
||||
backend._should_stop = False # Clear stop flag so status returns to idle
|
||||
backend.trainer._update_progress(
|
||||
is_training=False, is_completed=False, error=None,
|
||||
status_message="Ready to train", step=0, loss=0.0, epoch=0,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue