fixing the hangup of training after multiple back to back training processes

studio/backend/core/training/trainer.py

@@ -115,6 +115,14 @@ class UnslothTrainer:
                    is_dataset_multimodal: bool = False) -> bool:
         """Load model for training (supports both text and vision models)"""
         try:
+            if self.model is not None:
+                del self.model
+            if self.tokenizer is not None:
+                del self.tokenizer
+
+            if self.trainer is not None:
+                del self.trainer
+
             print("\nClearing GPU memory before training...")
             clear_gpu_cache()
 

studio/backend/utils/hardware/hardware.py

@@ -120,6 +120,7 @@ def clear_gpu_cache():
 
     if device == DeviceType.CUDA:
         import torch
+        torch.cuda.synchronize()
         torch.cuda.empty_cache()
         torch.cuda.ipc_collect()
     elif device == DeviceType.MLX: