studio: skip training status/metrics polling when idle (#4988)
* fix(studio): skip training status/metrics polling when idle Add an early return in the status and metrics setInterval callbacks when the runtime store reports phase === "idle" and hasHydrated is true. Previously these polls fired unconditionally every 3s/5s, generating unnecessary network traffic and console errors when no training was running. * fix(studio): reduce idle polling to 30s instead of stopping entirely Review feedback (PR #4988): completely stopping polling when idle risks permanent UI desync if hydration fails, and misses out-of-band state changes from other clients. Add a 30s background poll that only fires when idle to recover gracefully. * fix: harden idle status polling around hydration and runtime reset --------- Co-authored-by: AdamPlatin123 <AdamPlatin123@users.noreply.github.com> Co-authored-by: Lee Jackson <130007945+Imagineer99@users.noreply.github.com> Co-authored-by: imagineer99 <samleejackson0@gmail.com>
This commit is contained in:
parent
9a261aec5f
commit
d2fc582840
2 changed files with 20 additions and 2 deletions
|
|
@ -14,6 +14,7 @@ import type { TrainingRuntimeStore } from "../types/runtime";
|
|||
|
||||
const STATUS_POLL_INTERVAL_MS = 3000;
|
||||
const METRICS_POLL_INTERVAL_MS = 5000;
|
||||
const IDLE_POLL_INTERVAL_MS = 30000;
|
||||
const STREAM_RECONNECT_DELAY_MS = 1500;
|
||||
|
||||
function shouldUseLiveSync(state: TrainingRuntimeStore): boolean {
|
||||
|
|
@ -164,21 +165,37 @@ export function useTrainingRuntimeLifecycle(): void {
|
|||
|
||||
void hydrate();
|
||||
|
||||
const isIdle = () => {
|
||||
const s = runtimeStore.getState();
|
||||
return s.phase === "idle" && !s.isTrainingRunning;
|
||||
};
|
||||
|
||||
const statusTimer = setInterval(() => {
|
||||
const s = runtimeStore.getState();
|
||||
if (isIdle() && s.hasHydrated) return;
|
||||
void pollStatus();
|
||||
}, STATUS_POLL_INTERVAL_MS);
|
||||
|
||||
const metricsTimer = setInterval(() => {
|
||||
const state = runtimeStore.getState();
|
||||
if (shouldUseLiveSync(state) || state.currentStep > 0) {
|
||||
if (isIdle()) return;
|
||||
const s = runtimeStore.getState();
|
||||
if (shouldUseLiveSync(s) || s.currentStep > 0) {
|
||||
void pollMetrics();
|
||||
}
|
||||
}, METRICS_POLL_INTERVAL_MS);
|
||||
|
||||
// Low-frequency background poll to recover from failed hydration or detect
|
||||
// out-of-band state changes (e.g. training started from another client).
|
||||
const idleTimer = setInterval(() => {
|
||||
if (!isIdle()) return;
|
||||
void pollStatus();
|
||||
}, IDLE_POLL_INTERVAL_MS);
|
||||
|
||||
return () => {
|
||||
disposed = true;
|
||||
clearInterval(statusTimer);
|
||||
clearInterval(metricsTimer);
|
||||
clearInterval(idleTimer);
|
||||
stopStream();
|
||||
};
|
||||
}, []);
|
||||
|
|
|
|||
|
|
@ -125,6 +125,7 @@ export const useTrainingRuntimeStore = create<TrainingRuntimeStore>()((set) => (
|
|||
resetRuntime: () =>
|
||||
set((state) => ({
|
||||
...initialState,
|
||||
hasHydrated: state.hasHydrated,
|
||||
lossHistory: [],
|
||||
lrHistory: [],
|
||||
gradNormHistory: [],
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue