From f3aeceeb24796b32cb5fda7149540db80662f109 Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Sat, 7 Mar 2026 08:37:03 +0000 Subject: [PATCH] fix: join prior pump thread before starting new training job --- studio/backend/core/training/training.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.py index cf2238ba30..642b1e8e27 100644 --- a/studio/backend/core/training/training.py +++ b/studio/backend/core/training/training.py @@ -101,6 +101,14 @@ class TrainingBackend: logger.warning("Training subprocess already running") return False + # Join prior pump thread to prevent it from consuming events + # from the new job's queue (it reads self._event_queue dynamically). + if self._pump_thread is not None and self._pump_thread.is_alive(): + self._pump_thread.join(timeout=5.0) + if self._pump_thread.is_alive(): + logger.warning("Previous pump thread did not exit within 5s") + self._pump_thread = None + # Reset state self._should_stop = False self._progress = TrainingProgress(is_training=True, status_message="Initializing training...")