diff --git a/setup.ps1 b/setup.ps1 index e46a7522d2..61ae40f3b2 100644 --- a/setup.ps1 +++ b/setup.ps1 @@ -17,6 +17,7 @@ $PackageDir = Split-Path -Parent $ScriptDir # Detect if running from pip install (no studio/frontend/ dir in repo) $FrontendDir = Join-Path $ScriptDir "studio\frontend" +$OxcValidatorDir = Join-Path $ScriptDir "studio\backend\core\data_recipe\oxc-validator" $IsPipInstall = -not (Test-Path $FrontendDir) # ───────────────────────────────────────────── @@ -599,6 +600,23 @@ if ($IsPipInstall) { Write-Host "[OK] Frontend built to studio/frontend/dist" -ForegroundColor Green } +if (Test-Path $OxcValidatorDir) { + Write-Host "Installing OXC validator runtime..." -ForegroundColor Cyan + $prevEAP_oxc = $ErrorActionPreference + $ErrorActionPreference = "Continue" + Push-Location $OxcValidatorDir + npm install 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0) { + Pop-Location + $ErrorActionPreference = $prevEAP_oxc + Write-Host "[ERROR] OXC validator npm install failed (exit code $LASTEXITCODE)" -ForegroundColor Red + exit 1 + } + Pop-Location + $ErrorActionPreference = $prevEAP_oxc + Write-Host "[OK] OXC validator runtime installed" -ForegroundColor Green +} + # ========================================================================== # PHASE 3: Python environment + dependencies # ========================================================================== @@ -992,4 +1010,4 @@ Write-Host "| IMPORTANT: Open a NEW terminal, then run: |" -ForegroundColor Write-Host "| |" -ForegroundColor Green Write-Host "| unsloth-studio -H 0.0.0.0 -p 8000 |" -ForegroundColor Green Write-Host "| |" -ForegroundColor Green -Write-Host "+===============================================+" -ForegroundColor Green \ No newline at end of file +Write-Host "+===============================================+" -ForegroundColor Green diff --git a/studio/backend/core/data_recipe/jobs/worker.py b/studio/backend/core/data_recipe/jobs/worker.py index 245484f2b6..9181fd8136 100644 --- a/studio/backend/core/data_recipe/jobs/worker.py +++ b/studio/backend/core/data_recipe/jobs/worker.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import logging import re import shutil @@ -171,4 +172,40 @@ def _merge_batches_to_single_parquet(base_dataset_path: Path) -> None: dataframe = read_parquet_dataset(parquet_dir) shutil.rmtree(parquet_dir) parquet_dir.mkdir(parents=True, exist_ok=True) - dataframe.to_parquet(parquet_dir / "batch_00000.parquet", index=False) + merged_file = parquet_dir / "batch_00000.parquet" + dataframe.to_parquet(merged_file, index=False) + _rewrite_merged_metadata( + base_dataset_path=base_dataset_path, + parquet_file=merged_file, + ) + + +def _rewrite_merged_metadata(*, base_dataset_path: Path, parquet_file: Path) -> None: + metadata_path = base_dataset_path / "metadata.json" + if not metadata_path.exists(): + return + + try: + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + except (OSError, TypeError, ValueError): + return + + if not isinstance(metadata, dict): + return + + relative_parquet_path = str(parquet_file.relative_to(base_dataset_path)) + file_paths = metadata.get("file_paths") + if not isinstance(file_paths, dict): + file_paths = {} + file_paths["parquet-files"] = [relative_parquet_path] + metadata["file_paths"] = file_paths + metadata["total_num_batches"] = 1 + metadata["num_completed_batches"] = 1 + + try: + metadata_path.write_text( + json.dumps(metadata, indent=2, sort_keys=True), + encoding="utf-8", + ) + except OSError: + return diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py index b51fe60491..7bfb8d4d7a 100644 --- a/studio/backend/core/training/trainer.py +++ b/studio/backend/core/training/trainer.py @@ -32,6 +32,9 @@ from trl import SFTTrainer, SFTConfig logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +_BACKEND_ROOT = Path(__file__).resolve().parents[2] +_ASSETS_DATASETS_ROOT = _BACKEND_ROOT / "assets" / "datasets" + @dataclass class TrainingProgress: @@ -1791,9 +1794,7 @@ class UnslothTrainer: file_path = dataset_file else: # Fallback: try relative to assets/datasets - script_dir = Path(__file__).parent.parent - assets_datasets_dir = script_dir / "assets" / "datasets" - file_path = assets_datasets_dir / dataset_file + file_path = _ASSETS_DATASETS_ROOT / dataset_file file_path_obj = Path(file_path) file_path_str = str(file_path_obj) diff --git a/studio/backend/routes/datasets.py b/studio/backend/routes/datasets.py index b7aa72727a..04345a944d 100644 --- a/studio/backend/routes/datasets.py +++ b/studio/backend/routes/datasets.py @@ -253,7 +253,9 @@ def _load_local_preview_slice(*, dataset_path: Path, train_split: str, preview_s @router.get("/local", response_model=LocalDatasetsResponse) -def list_local_datasets() -> LocalDatasetsResponse: +def list_local_datasets( + current_subject: str = Depends(get_current_subject), +) -> LocalDatasetsResponse: return LocalDatasetsResponse(datasets=_build_local_dataset_items()) diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py index d6726d69df..6c03cc13cd 100644 --- a/studio/backend/routes/training.py +++ b/studio/backend/routes/training.py @@ -111,6 +111,7 @@ async def start_training( # Validate dataset paths if provided if request.local_datasets: validated_datasets = [] + missing_datasets = [] # Get the backend directory (where this file is located) backend_dir = Path(__file__).parent.parent assets_datasets_dir = backend_dir / "assets" / "datasets" @@ -131,12 +132,20 @@ async def start_training( dataset_file = candidate if not dataset_file.exists(): - logger.warning( - f"Dataset file not found: {dataset_path} (resolved: {dataset_file})" + missing_datasets.append( + f"{dataset_path} (resolved: {dataset_file})" ) - else: - logger.info(f"Found dataset file: {dataset_file}") + continue + + logger.info(f"Found dataset file: {dataset_file}") validated_datasets.append(str(dataset_file)) + + if missing_datasets: + missing_detail = "; ".join(missing_datasets[:3]) + raise HTTPException( + status_code=400, + detail=f"Local dataset not found: {missing_detail}", + ) request.local_datasets = validated_datasets # Convert request to kwargs for backend diff --git a/studio/frontend/src/features/data-recipes/learning-recipes/instruction-from-answer.json b/studio/frontend/src/features/data-recipes/learning-recipes/instruction-from-answer.json index 1edb4f185e..51af88b268 100644 --- a/studio/frontend/src/features/data-recipes/learning-recipes/instruction-from-answer.json +++ b/studio/frontend/src/features/data-recipes/learning-recipes/instruction-from-answer.json @@ -72,7 +72,7 @@ "width": 400, "node_type": "markdown_note", "name": "note_1", - "markdown": "#### Hugginface seed block\nThis recipe uses a ** HuggingFace dataset ** as seed data.\nYou select HuggingFace dataset, load columns, then generate new fields from seed columns. Each column in Hugginface dataset becomes a valid variable that you can reference eg. `{{ topic }}`\n\n##### Setup:\n\n1. Search for dataset and select on in the dropdown (example: `unsloth/alpaca-cleaned`)\n2. Add token only if dataset is gated/private\n3. Load columns + preview rows so variables are available in prompts\n\n##### Why this matters:\n- Seed columns can drive generation quality\n- You can reference seed values directly in prompts (for example `{{ output }}`)", + "markdown": "#### Hugging Face seed block\nThis recipe uses a ** HuggingFace dataset ** as seed data.\nYou select a Hugging Face dataset, load columns, then generate new fields from seed columns. Each column in the Hugging Face dataset becomes a valid variable that you can reference eg. `{{ topic }}`\n\n##### Setup:\n\n1. Search for a dataset and select one in the dropdown (example: `unsloth/alpaca-cleaned`)\n2. Add token only if dataset is gated/private\n3. Load columns + preview rows so variables are available in prompts\n\n##### Why this matters:\n- Seed columns can drive generation quality\n- You can reference seed values directly in prompts (for example `{{ output }}`)", "note_color": "#DCFCE7", "note_opacity": "35" }, @@ -145,4 +145,4 @@ "unstructured_chunk_size": "1200", "unstructured_chunk_overlap": "200" } -} \ No newline at end of file +} diff --git a/studio/frontend/src/features/onboarding/components/steps/dataset-step.tsx b/studio/frontend/src/features/onboarding/components/steps/dataset-step.tsx index e037bb7f65..5e72e4b765 100644 --- a/studio/frontend/src/features/onboarding/components/steps/dataset-step.tsx +++ b/studio/frontend/src/features/onboarding/components/steps/dataset-step.tsx @@ -66,7 +66,8 @@ export function DatasetStep() { hfToken, setHfToken, datasetSource, - setDatasetSource, + selectHfDataset, + selectLocalDataset, datasetFormat, setDatasetFormat, dataset, @@ -85,7 +86,8 @@ export function DatasetStep() { hfToken: s.hfToken, setHfToken: s.setHfToken, datasetSource: s.datasetSource, - setDatasetSource: s.setDatasetSource, + selectHfDataset: s.selectHfDataset, + selectLocalDataset: s.selectLocalDataset, datasetFormat: s.datasetFormat, setDatasetFormat: s.setDatasetFormat, dataset: s.dataset, @@ -138,7 +140,9 @@ export function DatasetStep() {