From ace86ed0b8f6e5b6de65fc1822c3c966cdc4c652 Mon Sep 17 00:00:00 2001 From: Manan17 Date: Sun, 15 Feb 2026 05:38:06 +0000 Subject: [PATCH 1/2] Fixing stuck training processes --- studio/backend/core/training/trainer.py | 5 ++++- studio/backend/core/training/training.py | 26 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py index 013c7817b2..5a3d8683e2 100644 --- a/studio/backend/core/training/trainer.py +++ b/studio/backend/core/training/trainer.py @@ -2,13 +2,16 @@ Unsloth Training Backend Integrates Unsloth training capabilities with the FastAPI backend """ +import os +# Prevent tokenizer parallelism deadlocks when datasets uses multiprocessing fork +os.environ["TOKENIZERS_PARALLELISM"] = "false" + import torch from utils.hardware import clear_gpu_cache torch._dynamo.config.recompile_limit = 64 from unsloth import FastLanguageModel, FastVisionModel, is_bfloat16_supported from unsloth.chat_templates import get_chat_template -import os import json import threading import math diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.py index 62febadc13..62aa021136 100644 --- a/studio/backend/core/training/training.py +++ b/studio/backend/core/training/training.py @@ -6,6 +6,7 @@ from typing import Any, Generator, Tuple import logging from .trainer import get_trainer, TrainingProgress +from utils.hardware import clear_gpu_cache logger = logging.getLogger(__name__) @@ -101,6 +102,31 @@ class TrainingBackend: True if training started successfully, False otherwise. """ try: + # Wait for any previous training thread to finish + old_thread = getattr(self.trainer, "training_thread", None) + if old_thread and old_thread.is_alive(): + logger.info("Waiting for previous training thread to finish...") + old_thread.join(timeout=30) + + # Explicitly free old SFTTrainer and CUDA resources before loading new model. + # Without this, forked multiprocessing workers (num_proc tokenization) inherit + # stale CUDA state from the previous run, causing extreme slowdowns or crashes. + if self.trainer.trainer is not None: + logger.info("Cleaning up previous SFTTrainer...") + self.trainer.trainer = None + if self.trainer.model is not None: + self.trainer.model = None + if self.trainer.tokenizer is not None: + self.trainer.tokenizer = None + # Flush all pending async CUDA ops so forked tokenization processes + # don't inherit stale async state that causes pool join to hang. + import torch as _torch + if _torch.cuda.is_available(): + _torch.cuda.synchronize() + import gc + gc.collect() + clear_gpu_cache() + # Reset stop flag and clear history self.trainer.should_stop = False self.trainer.save_on_stop = True From 45618fb43cac0476a3f8287cdd2a6f8b175d39ba Mon Sep 17 00:00:00 2001 From: Manan17 Date: Sun, 15 Feb 2026 06:12:55 +0000 Subject: [PATCH 2/2] Adding hint for password length --- studio/frontend/src/features/auth/components/auth-form.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/studio/frontend/src/features/auth/components/auth-form.tsx b/studio/frontend/src/features/auth/components/auth-form.tsx index e8d531c0b4..f17d2af13b 100644 --- a/studio/frontend/src/features/auth/components/auth-form.tsx +++ b/studio/frontend/src/features/auth/components/auth-form.tsx @@ -199,6 +199,9 @@ export function AuthForm({ mode }: AuthFormProps): ReactElement | null { )} + {!isLoginMode && ( +

Must be at least 8 characters

+ )} {!isLoginMode && ( @@ -223,7 +226,7 @@ export function AuthForm({ mode }: AuthFormProps): ReactElement | null {