fixing cancel training
This commit is contained in:
parent
8c493bbd20
commit
9be55f0c1b
4 changed files with 36 additions and 7 deletions
|
|
@ -226,6 +226,20 @@ class TrainingBackend:
|
|||
)
|
||||
return True
|
||||
|
||||
def force_terminate(self) -> None:
|
||||
"""Force-kill the training subprocess so state can be reset immediately."""
|
||||
with self._lock:
|
||||
if self._proc is not None and self._proc.is_alive():
|
||||
logger.info("Force-terminating training subprocess (pid=%s)", self._proc.pid)
|
||||
self._proc.terminate()
|
||||
proc = self._proc
|
||||
|
||||
if proc is not None:
|
||||
proc.join(timeout=5.0)
|
||||
if proc.is_alive():
|
||||
proc.kill()
|
||||
proc.join(timeout=2.0)
|
||||
|
||||
def is_training_active(self) -> bool:
|
||||
"""Check if training is currently active."""
|
||||
with self._lock:
|
||||
|
|
|
|||
|
|
@ -309,11 +309,16 @@ async def reset_training(
|
|||
is_active = backend.is_training_active()
|
||||
|
||||
if is_active:
|
||||
logger.warning("Rejected reset while training active: is_active=%s", is_active)
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Training is still running. Stop training and wait for it to finish before resetting.",
|
||||
)
|
||||
if backend._should_stop:
|
||||
# Stop was already requested — force-terminate so we can reset immediately
|
||||
logger.info("Force-terminating subprocess for immediate reset")
|
||||
backend.force_terminate()
|
||||
else:
|
||||
logger.warning("Rejected reset while training active: is_active=%s", is_active)
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Training is still running. Stop training and wait for it to finish before resetting.",
|
||||
)
|
||||
|
||||
logger.info("Reset training state: clearing runtime + metric history")
|
||||
backend._should_stop = False # Clear stop flag so status returns to idle
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ export function TrainingStartOverlay({
|
|||
message,
|
||||
currentStep,
|
||||
}: TrainingStartOverlayProps): ReactElement {
|
||||
const { stopTrainingRun } = useTrainingActions();
|
||||
const { stopTrainingRun, dismissTrainingRun } = useTrainingActions();
|
||||
const isStarting = useTrainingRuntimeStore((s) => s.isStarting);
|
||||
const [cancelDialogOpen, setCancelDialogOpen] = useState(false);
|
||||
const [cancelRequested, setCancelRequested] = useState(false);
|
||||
|
|
@ -77,7 +77,11 @@ export function TrainingStartOverlay({
|
|||
setCancelDialogOpen(false);
|
||||
useTrainingRuntimeStore.getState().setStopRequested(true);
|
||||
void stopTrainingRun(false).then((ok) => {
|
||||
if (!ok) setCancelRequested(false);
|
||||
if (ok) {
|
||||
void dismissTrainingRun();
|
||||
} else {
|
||||
setCancelRequested(false);
|
||||
}
|
||||
});
|
||||
}}
|
||||
>
|
||||
|
|
|
|||
|
|
@ -105,6 +105,12 @@ export function useTrainingActions() {
|
|||
}
|
||||
}
|
||||
|
||||
// Abort if cancel was requested during dataset check
|
||||
if (useTrainingRuntimeStore.getState().stopRequested) {
|
||||
runtimeStore.setStarting(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Re-read config after potential store updates from dataset check
|
||||
const payload = buildTrainingStartPayload(useTrainingConfigStore.getState());
|
||||
const response = await startTraining(payload);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue