Merge pull request #346 from unslothai/fix/eval-loss-worker-filtering

fix: eval loss broken after subprocess isolation refactor

studio/backend/core/training/trainer.py

@@ -1868,7 +1868,8 @@ class UnslothTrainer:
 
                 # Resolve eval split from a separate HF split (explicit or auto-detected)
                 if eval_enabled:
-                    if eval_split:
+                    effective_train = train_split or "train"
+                    if eval_split and eval_split != effective_train:
                         # Explicit eval split provided - load it directly
                         print(f"Loading explicit eval split: '{eval_split}'\n")
                         eval_load_kwargs = {"path": dataset_source, "split": eval_split}
@@ -1877,6 +1878,9 @@ class UnslothTrainer:
                         eval_dataset = load_dataset(**eval_load_kwargs)
                         has_separate_eval_source = True
                         print(f"Loaded eval split '{eval_split}' with {len(eval_dataset)} rows\n")
+                    elif eval_split and eval_split == effective_train:
+                        # Same split as training — will do 80/20 split after formatting
+                        print(f"Eval split '{eval_split}' is the same as train split — will split 80/20\n")
                     else:
                         # Auto-detect eval split from HF (returns a separate dataset, or None)
                         eval_dataset = self._auto_detect_eval_split_from_hf(

studio/backend/core/training/training.py

@@ -384,6 +384,9 @@ class TrainingBackend:
                     self.eval_step_history.append(step)
                     self.eval_enabled = True
 
+            elif etype == "eval_configured":
+                self.eval_enabled = True
+
             elif etype == "status":
                 self._progress.status_message = event.get("message", "")
                 self._progress.is_training = True

studio/backend/core/training/worker.py

@@ -135,7 +135,9 @@ def run_training_process(
 
     # Wire up progress callback → event_queue
     def _on_progress(progress: TrainingProgress):
-        if progress.step >= 0 and progress.loss > 0:
+        has_train_loss = progress.step >= 0 and progress.loss > 0
+        has_eval_loss = progress.eval_loss is not None
+        if has_train_loss or has_eval_loss:
             event_queue.put({
                 "type": "progress",
                 "step": progress.step,
@@ -267,6 +269,14 @@ def run_training_process(
         if eval_steps is not None and float(eval_steps) <= 0:
             eval_dataset = None
 
+        # Tell the parent process that eval is configured so the frontend
+        # shows "Waiting for first evaluation step..." instead of "not configured"
+        if eval_dataset is not None:
+            event_queue.put({
+                "type": "eval_configured",
+                "ts": time.time(),
+            })
+
         if dataset is None or trainer.should_stop:
             if trainer.should_stop:
                 event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})

studio/frontend/src/features/studio/sections/training-section.tsx

@@ -12,6 +12,7 @@ import {
   serializeConfigToYaml,
   useTrainingActions,
   useTrainingConfigStore,
+  validateTrainingConfig,
 } from "@/features/training";
 import {
   Archive04Icon,
@@ -43,7 +44,8 @@ export function TrainingSection() {
   const { isStarting, startError, startTrainingRun } = useTrainingActions();
   const isIncompatible =
     !store.isVisionModel && store.isDatasetImage === true;
-    const fileInputRef = useRef<HTMLInputElement>(null);
+  const configValidation = validateTrainingConfig(store);
+  const fileInputRef = useRef<HTMLInputElement>(null);
 
   const handleFileUpload = (e: React.ChangeEvent<HTMLInputElement>) => {
     const file = e.target.files?.[0];
@@ -150,7 +152,7 @@ export function TrainingSection() {
           data-tour="studio-start"
           className="w-full cursor-pointer bg-gradient-to-r from-emerald-500 to-teal-500 text-white hover:from-emerald-600 hover:to-teal-600"
           onClick={() => void startTrainingRun()}
-          disabled={isStarting || isIncompatible}
+          disabled={isStarting || isIncompatible || !configValidation.ok}
         >
           <HugeiconsIcon icon={Rocket01Icon} className="size-4" />
           {isStarting ? "Starting..." : "Start Training"}
@@ -163,6 +165,9 @@ export function TrainingSection() {
             Text model is not compatible with a multimodal dataset. Switch to a vision model or choose a text-only dataset.
           </p>
         )}
+        {!configValidation.ok && configValidation.message && !isIncompatible && (
+          <p className="text-xs text-red-500 leading-relaxed">{configValidation.message}</p>
+        )}
 
         {/* Upload / Save / Reset */}
         <p className="text-xs text-muted-foreground">Training Config</p>

studio/frontend/src/features/training/index.ts

@@ -11,3 +11,4 @@ export { listLocalModels } from "./api/models-api";
 export type { LocalModelInfo } from "./api/models-api";
 export type { TrainingPhase } from "./types/runtime";
 export { parseYamlConfig, serializeConfigToYaml } from "./lib/yaml-config";
+export { validateTrainingConfig } from "./lib/validation";

studio/frontend/src/features/training/lib/validation.ts

@@ -16,18 +16,22 @@ export function validateTrainingConfig(
     if (!config.dataset) {
       return { ok: false, message: "Select a Hugging Face dataset first." };
     }
-    return { ok: true, message: null };
-  }
-
-  if (config.datasetSource === "upload") {
+  } else if (config.datasetSource === "upload") {
     if (!config.uploadedFile) {
       return { ok: false, message: "Select a local dataset first." };
     }
-    return { ok: true, message: null };
+  } else {
+    return { ok: false, message: "Unsupported dataset source." };
   }
 
-  return {
-    ok: false,
-    message: "Unsupported dataset source.",
-  };
+  // Eval steps requires an eval split to be selected
+  if (config.evalSteps > 0 && !config.datasetEvalSplit) {
+    return {
+      ok: false,
+      message:
+        "Eval Steps is set but no Eval Split is selected. Choose an Eval Split or set Eval Steps to 0.",
+    };
+  }
+
+  return { ok: true, message: null };
 }