feat(recipe-studio, datasets): improve dataset handling and update metadata logic

setup.ps1

@@ -17,6 +17,7 @@ $PackageDir = Split-Path -Parent $ScriptDir
 
 # Detect if running from pip install (no studio/frontend/ dir in repo)
 $FrontendDir = Join-Path $ScriptDir "studio\frontend"
+$OxcValidatorDir = Join-Path $ScriptDir "studio\backend\core\data_recipe\oxc-validator"
 $IsPipInstall = -not (Test-Path $FrontendDir)
 
 # ─────────────────────────────────────────────
@@ -599,6 +600,23 @@ if ($IsPipInstall) {
     Write-Host "[OK] Frontend built to studio/frontend/dist" -ForegroundColor Green
 }
 
+if (Test-Path $OxcValidatorDir) {
+    Write-Host "Installing OXC validator runtime..." -ForegroundColor Cyan
+    $prevEAP_oxc = $ErrorActionPreference
+    $ErrorActionPreference = "Continue"
+    Push-Location $OxcValidatorDir
+    npm install 2>&1 | Out-Null
+    if ($LASTEXITCODE -ne 0) {
+        Pop-Location
+        $ErrorActionPreference = $prevEAP_oxc
+        Write-Host "[ERROR] OXC validator npm install failed (exit code $LASTEXITCODE)" -ForegroundColor Red
+        exit 1
+    }
+    Pop-Location
+    $ErrorActionPreference = $prevEAP_oxc
+    Write-Host "[OK] OXC validator runtime installed" -ForegroundColor Green
+}
+
 # ==========================================================================
 #  PHASE 3: Python environment + dependencies
 # ==========================================================================
@@ -992,4 +1010,4 @@ Write-Host "|  IMPORTANT: Open a NEW terminal, then run:    |" -ForegroundColor
 Write-Host "|                                               |" -ForegroundColor Green
 Write-Host "|    unsloth-studio -H 0.0.0.0 -p 8000         |" -ForegroundColor Green
 Write-Host "|                                               |" -ForegroundColor Green
-Write-Host "+===============================================+" -ForegroundColor Green
+Write-Host "+===============================================+" -ForegroundColor Green

studio/backend/core/data_recipe/jobs/worker.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 import logging
 import re
 import shutil
@@ -171,4 +172,40 @@ def _merge_batches_to_single_parquet(base_dataset_path: Path) -> None:
     dataframe = read_parquet_dataset(parquet_dir)
     shutil.rmtree(parquet_dir)
     parquet_dir.mkdir(parents=True, exist_ok=True)
-    dataframe.to_parquet(parquet_dir / "batch_00000.parquet", index=False)
+    merged_file = parquet_dir / "batch_00000.parquet"
+    dataframe.to_parquet(merged_file, index=False)
+    _rewrite_merged_metadata(
+        base_dataset_path=base_dataset_path,
+        parquet_file=merged_file,
+    )
+
+
+def _rewrite_merged_metadata(*, base_dataset_path: Path, parquet_file: Path) -> None:
+    metadata_path = base_dataset_path / "metadata.json"
+    if not metadata_path.exists():
+        return
+
+    try:
+        metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
+    except (OSError, TypeError, ValueError):
+        return
+
+    if not isinstance(metadata, dict):
+        return
+
+    relative_parquet_path = str(parquet_file.relative_to(base_dataset_path))
+    file_paths = metadata.get("file_paths")
+    if not isinstance(file_paths, dict):
+        file_paths = {}
+    file_paths["parquet-files"] = [relative_parquet_path]
+    metadata["file_paths"] = file_paths
+    metadata["total_num_batches"] = 1
+    metadata["num_completed_batches"] = 1
+
+    try:
+        metadata_path.write_text(
+            json.dumps(metadata, indent=2, sort_keys=True),
+            encoding="utf-8",
+        )
+    except OSError:
+        return

studio/backend/core/training/trainer.py

@@ -32,6 +32,9 @@ from trl import SFTTrainer, SFTConfig
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
+_BACKEND_ROOT = Path(__file__).resolve().parents[2]
+_ASSETS_DATASETS_ROOT = _BACKEND_ROOT / "assets" / "datasets"
+
 
 @dataclass
 class TrainingProgress:
@@ -1791,9 +1794,7 @@ class UnslothTrainer:
                         file_path = dataset_file
                     else:
                         # Fallback: try relative to assets/datasets
-                        script_dir = Path(__file__).parent.parent
-                        assets_datasets_dir = script_dir / "assets" / "datasets"
-                        file_path = assets_datasets_dir / dataset_file
+                        file_path = _ASSETS_DATASETS_ROOT / dataset_file
 
                     file_path_obj = Path(file_path)
                     file_path_str = str(file_path_obj)

studio/backend/routes/datasets.py

@@ -253,7 +253,9 @@ def _load_local_preview_slice(*, dataset_path: Path, train_split: str, preview_s
 
 
 @router.get("/local", response_model=LocalDatasetsResponse)
-def list_local_datasets() -> LocalDatasetsResponse:
+def list_local_datasets(
+    current_subject: str = Depends(get_current_subject),
+) -> LocalDatasetsResponse:
     return LocalDatasetsResponse(datasets=_build_local_dataset_items())
 
 

studio/backend/routes/training.py

@@ -111,6 +111,7 @@ async def start_training(
         # Validate dataset paths if provided
         if request.local_datasets:
             validated_datasets = []
+            missing_datasets = []
             # Get the backend directory (where this file is located)
             backend_dir = Path(__file__).parent.parent
             assets_datasets_dir = backend_dir / "assets" / "datasets"
@@ -131,12 +132,20 @@ async def start_training(
                     dataset_file = candidate
 
                 if not dataset_file.exists():
-                    logger.warning(
-                        f"Dataset file not found: {dataset_path} (resolved: {dataset_file})"
+                    missing_datasets.append(
+                        f"{dataset_path} (resolved: {dataset_file})"
                     )
-                else:
-                    logger.info(f"Found dataset file: {dataset_file}")
+                    continue
+
+                logger.info(f"Found dataset file: {dataset_file}")
                 validated_datasets.append(str(dataset_file))
+
+            if missing_datasets:
+                missing_detail = "; ".join(missing_datasets[:3])
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Local dataset not found: {missing_detail}",
+                )
             request.local_datasets = validated_datasets
 
         # Convert request to kwargs for backend

studio/frontend/src/features/data-recipes/learning-recipes/instruction-from-answer.json

@@ -72,7 +72,7 @@
         "width": 400,
         "node_type": "markdown_note",
         "name": "note_1",
-        "markdown": "#### Hugginface seed block\nThis recipe uses a ** HuggingFace dataset ** as seed data.\nYou select HuggingFace dataset, load columns, then generate new fields from seed columns. Each column in Hugginface dataset becomes a valid variable that you can reference eg. `{{ topic }}`\n\n##### Setup:\n\n1. Search for dataset and select on in the dropdown (example: `unsloth/alpaca-cleaned`)\n2. Add token only if dataset is gated/private\n3. Load columns + preview rows so variables are available in prompts\n\n##### Why this matters:\n- Seed columns can drive generation quality\n- You can reference seed values directly in prompts (for example `{{ output }}`)",
+        "markdown": "#### Hugging Face seed block\nThis recipe uses a ** HuggingFace dataset ** as seed data.\nYou select a Hugging Face dataset, load columns, then generate new fields from seed columns. Each column in the Hugging Face dataset becomes a valid variable that you can reference eg. `{{ topic }}`\n\n##### Setup:\n\n1. Search for a dataset and select one in the dropdown (example: `unsloth/alpaca-cleaned`)\n2. Add token only if dataset is gated/private\n3. Load columns + preview rows so variables are available in prompts\n\n##### Why this matters:\n- Seed columns can drive generation quality\n- You can reference seed values directly in prompts (for example `{{ output }}`)",
         "note_color": "#DCFCE7",
         "note_opacity": "35"
       },
@@ -145,4 +145,4 @@
     "unstructured_chunk_size": "1200",
     "unstructured_chunk_overlap": "200"
   }
-}
+}

studio/frontend/src/features/onboarding/components/steps/dataset-step.tsx

@@ -66,7 +66,8 @@ export function DatasetStep() {
     hfToken,
     setHfToken,
     datasetSource,
-    setDatasetSource,
+    selectHfDataset,
+    selectLocalDataset,
     datasetFormat,
     setDatasetFormat,
     dataset,
@@ -85,7 +86,8 @@ export function DatasetStep() {
       hfToken: s.hfToken,
       setHfToken: s.setHfToken,
       datasetSource: s.datasetSource,
-      setDatasetSource: s.setDatasetSource,
+      selectHfDataset: s.selectHfDataset,
+      selectLocalDataset: s.selectLocalDataset,
       datasetFormat: s.datasetFormat,
       setDatasetFormat: s.setDatasetFormat,
       dataset: s.dataset,
@@ -138,7 +140,9 @@ export function DatasetStep() {
         <div className="flex gap-2">
           <Button
             variant={datasetSource === "huggingface" ? "dark" : "outline"}
-            onClick={() => setDatasetSource("huggingface")}
+            onClick={() =>
+              selectHfDataset(datasetSource === "huggingface" ? dataset : null)
+            }
             className="flex-1"
           >
             <img
@@ -151,7 +155,11 @@ export function DatasetStep() {
           </Button>
           <Button
             variant={datasetSource === "upload" ? "dark" : "outline"}
-            onClick={() => setDatasetSource("upload")}
+            onClick={() =>
+              selectLocalDataset(
+                datasetSource === "upload" ? uploadedFile : null,
+              )
+            }
             className="flex-1"
           >
             <HugeiconsIcon icon={Upload04Icon} data-icon="inline-start" />

studio/frontend/src/features/recipe-studio/blocks/definitions.ts

@@ -138,7 +138,7 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [
   {
     kind: "seed",
     type: "seed_hf",
-    title: "Hugginface dataset",
+    title: "Hugging Face dataset",
     description: "Load real rows from HF and use them as generation context.",
     icon: Plant01Icon,
     dialogKey: "seed",

studio/frontend/src/features/recipe-studio/executions/tracker.ts

@@ -232,6 +232,12 @@ export async function trackRecipeExecution({
     const eventDataset = completedEventPayload
       ? completedEventPayload["dataset"]
       : null;
+    const eventProcessorArtifacts =
+      completedEventPayload &&
+      typeof completedEventPayload["processor_artifacts"] === "object" &&
+      completedEventPayload["processor_artifacts"] !== null
+        ? (completedEventPayload["processor_artifacts"] as Record<string, unknown>)
+        : null;
     const shouldFetchPreviewDataset = kind === "preview" && !Array.isArray(eventDataset);
     const shouldFetchAnalysis =
       !completedEventPayload ||
@@ -276,6 +282,7 @@ export async function trackRecipeExecution({
       datasetPage: 1,
       datasetPageSize: DATASET_PAGE_SIZE,
       error: null,
+      processor_artifacts: eventProcessorArtifacts ?? latestExecution.processor_artifacts,
       finishedAt: latestExecution.finishedAt ?? Date.now(),
     };
     onUpsert(latestExecution);

studio/frontend/src/features/studio/sections/dataset-section.tsx

@@ -168,6 +168,21 @@ export function DatasetSection() {
     void refreshLocalDatasets();
   }, [pickerTab, refreshLocalDatasets]);
 
+  useEffect(() => {
+    const handleRefresh = () => {
+      if (document.hidden) return;
+      if (pickerTab !== "local" && datasetSource !== "upload") return;
+      void refreshLocalDatasets();
+    };
+
+    window.addEventListener("focus", handleRefresh);
+    document.addEventListener("visibilitychange", handleRefresh);
+    return () => {
+      window.removeEventListener("focus", handleRefresh);
+      document.removeEventListener("visibilitychange", handleRefresh);
+    };
+  }, [datasetSource, pickerTab, refreshLocalDatasets]);
+
   function handleDatasetSelect(id: string | null) {
     selectingRef.current = true;
     pendingSourceTabRef.current = "huggingface";
@@ -266,6 +281,8 @@ export function DatasetSection() {
 
   useEffect(() => {
     if (!hasLoadedLocalDatasets) return;
+    if (localLoading) return;
+    if (localError) return;
     if (datasetSource !== "upload") return;
     if (!uploadedFile) return;
     if (selectedLocalDataset) return;
@@ -273,6 +290,8 @@ export function DatasetSection() {
   }, [
     datasetSource,
     hasLoadedLocalDatasets,
+    localError,
+    localLoading,
     uploadedFile,
     selectedLocalDataset,
     selectLocalDataset,
@@ -380,6 +399,9 @@ export function DatasetSection() {
                 value={comboboxValue}
                 onOpenChange={(open) => {
                   setSearchQuery("");
+                  if (open && (pickerTab === "local" || activeSourceTab === "local")) {
+                    void refreshLocalDatasets();
+                  }
                   if (!open) {
                     setPickerTab(pendingSourceTabRef.current ?? activeSourceTab);
                     pendingSourceTabRef.current = null;