diff --git a/.gitignore b/.gitignore index 904e15f974..d89fdb9693 100755 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ __pycache__/ *$py.class *.so .Python +*.egg-info/ # Virtual environments .venv/ @@ -23,6 +24,7 @@ unsloth_compiled_cache/ outputs/ exports/ /datasets/ +studio/backend/assets/datasets/ unsloth_training_checkpoints/ *.gguf *.safetensors @@ -53,6 +55,7 @@ firebase-debug.log # Other resources/ tmp/ +**/node_modules/ auth.db studio/frontend/package-lock.json diff --git a/install_python_stack.py b/install_python_stack.py index 77f1b1cea5..c6b4e556f4 100644 --- a/install_python_stack.py +++ b/install_python_stack.py @@ -22,6 +22,9 @@ SCRIPT_DIR = Path(__file__).resolve().parent REQ_ROOT = SCRIPT_DIR / "studio" / "backend" / "requirements" SINGLE_ENV = REQ_ROOT / "single-env" CONSTRAINTS = SINGLE_ENV / "constraints.txt" +LOCAL_DD_UNSTRUCTURED_PLUGIN = ( + SCRIPT_DIR / "studio" / "backend" / "plugins" / "data-designer-unstructured-seed" +) # ── Color support ────────────────────────────────────────────────────── @@ -235,13 +238,28 @@ def install_python_stack() -> int: req=SINGLE_ENV / "data-designer.txt", ) - # 11. Patch metadata for single-env compatibility + # 11. Local Data Designer seed plugin + if not LOCAL_DD_UNSTRUCTURED_PLUGIN.is_dir(): + print( + _red( + f"❌ Missing local plugin directory: {LOCAL_DD_UNSTRUCTURED_PLUGIN}", + ), + ) + return 1 + pip_install( + "Installing local data-designer unstructured plugin", + "--no-cache-dir", "--no-deps", + "-e", str(LOCAL_DD_UNSTRUCTURED_PLUGIN), + constrain=False, + ) + + # 12. Patch metadata for single-env compatibility run( "Patching single-env metadata", [sys.executable, str(SINGLE_ENV / "patch_metadata.py")], ) - # 12. Final check (silent — third-party conflicts are expected) + # 13. Final check (silent; third-party conflicts are expected) subprocess.run( [sys.executable, "-m", "pip", "check"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, diff --git a/setup.ps1 b/setup.ps1 index e46a7522d2..61ae40f3b2 100644 --- a/setup.ps1 +++ b/setup.ps1 @@ -17,6 +17,7 @@ $PackageDir = Split-Path -Parent $ScriptDir # Detect if running from pip install (no studio/frontend/ dir in repo) $FrontendDir = Join-Path $ScriptDir "studio\frontend" +$OxcValidatorDir = Join-Path $ScriptDir "studio\backend\core\data_recipe\oxc-validator" $IsPipInstall = -not (Test-Path $FrontendDir) # ───────────────────────────────────────────── @@ -599,6 +600,23 @@ if ($IsPipInstall) { Write-Host "[OK] Frontend built to studio/frontend/dist" -ForegroundColor Green } +if (Test-Path $OxcValidatorDir) { + Write-Host "Installing OXC validator runtime..." -ForegroundColor Cyan + $prevEAP_oxc = $ErrorActionPreference + $ErrorActionPreference = "Continue" + Push-Location $OxcValidatorDir + npm install 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0) { + Pop-Location + $ErrorActionPreference = $prevEAP_oxc + Write-Host "[ERROR] OXC validator npm install failed (exit code $LASTEXITCODE)" -ForegroundColor Red + exit 1 + } + Pop-Location + $ErrorActionPreference = $prevEAP_oxc + Write-Host "[OK] OXC validator runtime installed" -ForegroundColor Green +} + # ========================================================================== # PHASE 3: Python environment + dependencies # ========================================================================== @@ -992,4 +1010,4 @@ Write-Host "| IMPORTANT: Open a NEW terminal, then run: |" -ForegroundColor Write-Host "| |" -ForegroundColor Green Write-Host "| unsloth-studio -H 0.0.0.0 -p 8000 |" -ForegroundColor Green Write-Host "| |" -ForegroundColor Green -Write-Host "+===============================================+" -ForegroundColor Green \ No newline at end of file +Write-Host "+===============================================+" -ForegroundColor Green diff --git a/setup.sh b/setup.sh index 8e5ae2ac92..6edb470d8d 100755 --- a/setup.sh +++ b/setup.sh @@ -99,6 +99,8 @@ echo "Building frontend..." cd "$SCRIPT_DIR/studio/frontend" run_quiet "npm install" npm install run_quiet "npm run build" npm run build +cd "$SCRIPT_DIR/studio/backend/core/data_recipe/oxc-validator" +run_quiet "npm install (oxc validator runtime)" npm install cd "$SCRIPT_DIR" echo "✅ Frontend built to studio/frontend/dist" diff --git a/studio/backend/core/data_recipe/jobs/manager.py b/studio/backend/core/data_recipe/jobs/manager.py index dbaa620004..de367dcce6 100644 --- a/studio/backend/core/data_recipe/jobs/manager.py +++ b/studio/backend/core/data_recipe/jobs/manager.py @@ -13,7 +13,7 @@ from typing import Any import multiprocessing as mp -from ..jsonable import to_jsonable +from ..jsonable import to_preview_jsonable from .constants import ( EVENT_JOB_CANCELLING, EVENT_JOB_CANCELLED, @@ -138,6 +138,7 @@ class JobManager: "status": job.status, "stage": job.stage, "current_column": job.current_column, + "completed_columns": list(job.completed_columns), "batch": {"idx": job.batch.idx, "total": job.batch.total}, "progress": { "done": job.progress.done, @@ -298,7 +299,7 @@ class JobManager: dataframe = dataframe.drop(columns=[helper_col]) rows = dataframe.to_dict(orient="records") - return {"dataset": to_jsonable(rows), "total": total} + return {"dataset": to_preview_jsonable(rows), "total": total} @staticmethod def _load_dataset_page_with_data_designer( @@ -312,7 +313,7 @@ class JobManager: dataframe = read_parquet_dataset(parquet_dir) total = int(len(dataframe.index)) rows = dataframe.iloc[offset:offset + limit].to_dict(orient="records") - return {"dataset": to_jsonable(rows), "total": total} + return {"dataset": to_preview_jsonable(rows), "total": total} def subscribe(self, job_id: str, *, after_seq: int | None = None) -> Subscription | None: """SSE subscribe: get replay buffer + live events stream.""" diff --git a/studio/backend/core/data_recipe/jobs/parse.py b/studio/backend/core/data_recipe/jobs/parse.py index 6e2142adf2..858ddb7f03 100644 --- a/studio/backend/core/data_recipe/jobs/parse.py +++ b/studio/backend/core/data_recipe/jobs/parse.py @@ -151,6 +151,15 @@ def apply_update(job: Job, update: ParsedUpdate) -> None: job.cols = update.cols if update.progress is not None: job.column_progress = update.progress + if ( + job.current_column + and update.progress.done is not None + and update.progress.total is not None + and update.progress.total > 0 + and update.progress.done >= update.progress.total + and job.current_column not in job.completed_columns + ): + job.completed_columns.append(job.current_column) job.progress = _compute_overall_progress(job, update.progress) if update.batch_idx is not None: job.batch.idx = update.batch_idx @@ -240,5 +249,5 @@ def _compute_overall_progress(job: Job, column_progress: Progress) -> Progress: def coerce_event(obj: Any) -> dict: - # worker sends dict already + """Normalize worker payload into event dict.""" return obj if isinstance(obj, dict) else {"type": "log", "message": str(obj)} diff --git a/studio/backend/core/data_recipe/jobs/types.py b/studio/backend/core/data_recipe/jobs/types.py index 24a63d062c..80bd03cf77 100644 --- a/studio/backend/core/data_recipe/jobs/types.py +++ b/studio/backend/core/data_recipe/jobs/types.py @@ -66,6 +66,7 @@ class Job: processor_artifacts: dict[str, Any] | None = None model_usage: dict[str, ModelUsage] = field(default_factory=dict) progress_columns_total: int | None = None + completed_columns: list[str] = field(default_factory=list) _current_usage_model: str | None = None _in_usage_summary: bool = False _seen_generation_columns: list[str] = field(default_factory=list) diff --git a/studio/backend/core/data_recipe/jobs/worker.py b/studio/backend/core/data_recipe/jobs/worker.py index 8c0996b140..9181fd8136 100644 --- a/studio/backend/core/data_recipe/jobs/worker.py +++ b/studio/backend/core/data_recipe/jobs/worker.py @@ -1,13 +1,16 @@ from __future__ import annotations +import json import logging +import re import shutil import time import traceback +import unicodedata from pathlib import Path from typing import Any -from ..jsonable import to_jsonable +from ..jsonable import to_jsonable, to_preview_jsonable from .constants import EVENT_JOB_COMPLETED, EVENT_JOB_ERROR, EVENT_JOB_STARTED from ..service import build_config_builder, create_data_designer @@ -34,6 +37,27 @@ class _QueueLogHandler(logging.Handler): pass +def _slugify_run_name(value: str) -> str: + normalized = unicodedata.normalize("NFKD", value) + ascii_only = normalized.encode("ascii", "ignore").decode("ascii") + slug = re.sub(r"[^a-zA-Z0-9]+", "-", ascii_only).strip("-").lower() + if not slug: + return "" + return slug[:80].strip("-") + + +def _build_dataset_name(*, run_name: str | None, job_id: str, artifact_root: Path) -> str: + fallback = f"recipe_{job_id}" + slug = _slugify_run_name(run_name or "") + base_name = f"recipe_{slug}" if slug else fallback + candidate = base_name + suffix = 2 + while (artifact_root / candidate).exists(): + candidate = f"{base_name}_{suffix}" + suffix += 1 + return candidate + + def run_job_process( *, event_queue, @@ -53,7 +77,13 @@ def run_job_process( job_id = str(run.get("_job_id") or "").strip() if not job_id: job_id = f"{int(time.time())}" - dataset_name = f"recipe_{job_id}" + run_name_raw = run.get("run_name") + run_name = run_name_raw if isinstance(run_name_raw, str) else None + dataset_name = _build_dataset_name( + run_name=run_name, + job_id=job_id, + artifact_root=_ARTIFACT_ROOT, + ) merge_batches = bool(run.get("merge_batches")) _ARTIFACT_ROOT.mkdir(parents=True, exist_ok=True) run_config_raw = run.get("run_config") or {} @@ -84,7 +114,7 @@ def run_job_process( dataset = ( [] if results.dataset is None - else to_jsonable(results.dataset.to_dict(orient="records")) + else to_preview_jsonable(results.dataset.to_dict(orient="records")) ) processor_artifacts = ( None @@ -142,4 +172,40 @@ def _merge_batches_to_single_parquet(base_dataset_path: Path) -> None: dataframe = read_parquet_dataset(parquet_dir) shutil.rmtree(parquet_dir) parquet_dir.mkdir(parents=True, exist_ok=True) - dataframe.to_parquet(parquet_dir / "batch_00000.parquet", index=False) + merged_file = parquet_dir / "batch_00000.parquet" + dataframe.to_parquet(merged_file, index=False) + _rewrite_merged_metadata( + base_dataset_path=base_dataset_path, + parquet_file=merged_file, + ) + + +def _rewrite_merged_metadata(*, base_dataset_path: Path, parquet_file: Path) -> None: + metadata_path = base_dataset_path / "metadata.json" + if not metadata_path.exists(): + return + + try: + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + except (OSError, TypeError, ValueError): + return + + if not isinstance(metadata, dict): + return + + relative_parquet_path = str(parquet_file.relative_to(base_dataset_path)) + file_paths = metadata.get("file_paths") + if not isinstance(file_paths, dict): + file_paths = {} + file_paths["parquet-files"] = [relative_parquet_path] + metadata["file_paths"] = file_paths + metadata["total_num_batches"] = 1 + metadata["num_completed_batches"] = 1 + + try: + metadata_path.write_text( + json.dumps(metadata, indent=2, sort_keys=True), + encoding="utf-8", + ) + except OSError: + return diff --git a/studio/backend/core/data_recipe/jsonable.py b/studio/backend/core/data_recipe/jsonable.py index aa6e1d6b2e..b45a378028 100644 --- a/studio/backend/core/data_recipe/jsonable.py +++ b/studio/backend/core/data_recipe/jsonable.py @@ -1,8 +1,63 @@ from __future__ import annotations +import base64 +import io +from pathlib import Path from typing import Any +def _pil_to_preview_payload(image: Any) -> dict[str, Any]: + buffer = io.BytesIO() + image.convert("RGB").save(buffer, format="JPEG", quality=85) + return { + "type": "image", + "mime": "image/jpeg", + "width": image.width, + "height": image.height, + "data": base64.b64encode(buffer.getvalue()).decode("ascii"), + } + + +def _open_pil_image_from_bytes(raw_bytes: bytes): + from PIL import Image # type: ignore + + with Image.open(io.BytesIO(raw_bytes)) as image: + return image.copy() + + +def _to_pil_from_hf_image_dict(value: Any) -> Any | None: + if not isinstance(value, dict): + return None + + raw_bytes = value.get("bytes") + if isinstance(raw_bytes, (bytes, bytearray)) and len(raw_bytes) > 0: + try: + return _open_pil_image_from_bytes(bytes(raw_bytes)) + except (OSError, ValueError): + pass + if ( + isinstance(raw_bytes, list) + and len(raw_bytes) > 0 + and all(isinstance(item, int) and 0 <= item <= 255 for item in raw_bytes) + ): + try: + return _open_pil_image_from_bytes(bytes(raw_bytes)) + except (OSError, ValueError): + pass + + path_value = value.get("path") + if isinstance(path_value, str) and path_value.strip(): + try: + from PIL import Image # type: ignore + + with Image.open(Path(path_value)) as image: + return image.copy() + except (OSError, ValueError, TypeError): + return None + + return None + + def to_jsonable(value: Any) -> Any: """Convert numpy/pandas-ish values into plain JSON-safe values.""" try: @@ -28,3 +83,36 @@ def to_jsonable(value: Any) -> Any: return value return value + + +def _to_preview_image_payload(value: Any) -> dict[str, Any] | None: + try: + from PIL.Image import Image as PILImage # type: ignore + except ImportError: # pragma: no cover + return None + + if not isinstance(value, PILImage): + hf_image = _to_pil_from_hf_image_dict(value) + if hf_image is None: + return None + value = hf_image + + return _pil_to_preview_payload(value) + + +def to_preview_jsonable(value: Any) -> Any: + """Convert values into JSON-safe preview values, including PIL images.""" + image_payload = _to_preview_image_payload(value) + if image_payload is not None: + return image_payload + + converted = to_jsonable(value) + if converted is None or isinstance(converted, (str, int, float, bool)): + return converted + if isinstance(converted, dict): + return {str(k): to_preview_jsonable(v) for k, v in converted.items()} + if isinstance(converted, (list, tuple, set)): + return [to_preview_jsonable(v) for v in converted] + if isinstance(converted, (bytes, bytearray)): + return base64.b64encode(bytes(converted)).decode("ascii") + return str(converted) diff --git a/studio/backend/core/data_recipe/local_callable_validators.py b/studio/backend/core/data_recipe/local_callable_validators.py new file mode 100644 index 0000000000..3d95e68265 --- /dev/null +++ b/studio/backend/core/data_recipe/local_callable_validators.py @@ -0,0 +1,315 @@ +from __future__ import annotations + +import json +import logging +import subprocess +from copy import deepcopy +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +OXC_VALIDATION_FN_MARKER = "unsloth_oxc_validator" + +_OXC_LANG_TO_NODE_LANG = { + "javascript": "js", + "typescript": "ts", + "jsx": "jsx", + "tsx": "tsx", +} +_OXC_VALIDATION_MODES = {"syntax", "lint", "syntax+lint"} +_OXC_CODE_SHAPES = {"auto", "module", "snippet"} + +_OXC_TOOL_DIR = Path(__file__).resolve().parent / "oxc-validator" +_OXC_RUNNER_PATH = _OXC_TOOL_DIR / "validate.mjs" + + +@dataclass(frozen=True) +class OxcLocalCallableValidatorSpec: + name: str + drop: bool + target_columns: list[str] + batch_size: int + code_lang: str + validation_mode: str + code_shape: str + + +def split_oxc_local_callable_validators( + recipe_core: dict[str, Any], +) -> tuple[dict[str, Any], list[OxcLocalCallableValidatorSpec]]: + columns = recipe_core.get("columns") + if not isinstance(columns, list): + return recipe_core, [] + + sanitized = deepcopy(recipe_core) + sanitized_columns = sanitized.get("columns") + if not isinstance(sanitized_columns, list): + return sanitized, [] + + kept_columns: list[Any] = [] + oxc_specs: list[OxcLocalCallableValidatorSpec] = [] + + for column in sanitized_columns: + if not isinstance(column, dict): + kept_columns.append(column) + continue + + maybe_spec = _parse_oxc_spec(column=column) + if maybe_spec is None: + kept_columns.append(column) + continue + oxc_specs.append(maybe_spec) + + sanitized["columns"] = kept_columns + return sanitized, oxc_specs + + +def register_oxc_local_callable_validators( + *, + builder, + specs: list[OxcLocalCallableValidatorSpec], +) -> None: + if not specs: + return + + from data_designer.config.column_configs import ValidationColumnConfig + from data_designer.config.validator_params import ( + LocalCallableValidatorParams, + ValidatorType, + ) + + for spec in specs: + validation_function = _build_oxc_validation_function( + spec.code_lang, + spec.validation_mode, + spec.code_shape, + ) + builder.add_column( + ValidationColumnConfig( + name=spec.name, + drop=spec.drop, + target_columns=spec.target_columns, + validator_type=ValidatorType.LOCAL_CALLABLE, + validator_params=LocalCallableValidatorParams( + validation_function=validation_function, + ), + batch_size=spec.batch_size, + ) + ) + + +def _parse_oxc_spec( + *, + column: dict[str, Any], +) -> OxcLocalCallableValidatorSpec | None: + if str(column.get("column_type") or "").strip() != "validation": + return None + if str(column.get("validator_type") or "").strip() != "local_callable": + return None + + params = column.get("validator_params") + if not isinstance(params, dict): + return None + + fn_raw = params.get("validation_function") + fn_name = fn_raw.strip() if isinstance(fn_raw, str) else "" + if not fn_name.startswith(OXC_VALIDATION_FN_MARKER): + return None + + name = str(column.get("name") or "").strip() + if not name: + return None + + target_columns_raw = column.get("target_columns") + target_columns = ( + [value.strip() for value in target_columns_raw if isinstance(value, str) and value.strip()] + if isinstance(target_columns_raw, list) + else [] + ) + if not target_columns: + return None + + code_lang, validation_mode, code_shape = _parse_oxc_validation_marker(fn_name) + batch_size = _parse_batch_size(column.get("batch_size")) + drop = bool(column.get("drop") is True) + + return OxcLocalCallableValidatorSpec( + name=name, + drop=drop, + target_columns=target_columns, + batch_size=batch_size, + code_lang=code_lang, + validation_mode=validation_mode, + code_shape=code_shape, + ) + + +def _parse_batch_size(value: Any) -> int: + try: + parsed = int(value) + except (TypeError, ValueError): + return 10 + return parsed if parsed >= 1 else 10 + + +def _parse_oxc_validation_marker(fn_name: str) -> tuple[str, str, str]: + marker = f"{OXC_VALIDATION_FN_MARKER}:" + if not fn_name.startswith(marker): + return "javascript", "syntax", "auto" + suffix = fn_name[len(marker) :] + parts = [part.strip() for part in suffix.split(":") if part.strip()] + if len(parts) < 2: + return "javascript", "syntax", "auto" + code_lang = parts[0] if parts[0] in _OXC_LANG_TO_NODE_LANG else "javascript" + mode = parts[1] if parts[1] in _OXC_VALIDATION_MODES else "syntax" + code_shape = parts[2] if len(parts) >= 3 and parts[2] in _OXC_CODE_SHAPES else "auto" + return code_lang, mode, code_shape + + +@lru_cache(maxsize=8) +def _build_oxc_validation_function(lang: str, validation_mode: str, code_shape: str): + node_lang = _OXC_LANG_TO_NODE_LANG.get(lang, "js") + mode = ( + validation_mode + if validation_mode in _OXC_VALIDATION_MODES + else "syntax" + ) + normalized_code_shape = code_shape if code_shape in _OXC_CODE_SHAPES else "auto" + + def _validator(df): + import pandas as pd # imported lazily for local callable runtime + + row_count = int(len(df.index)) + if row_count == 0: + return pd.DataFrame({"is_valid": []}) + + code_column = str(df.columns[0]) if len(df.columns) > 0 else "" + code_values = ( + ["" for _ in range(row_count)] + if not code_column + else ["" if value is None else str(value) for value in df[code_column].tolist()] + ) + + results = _run_oxc_batch( + node_lang=node_lang, + validation_mode=mode, + code_shape=normalized_code_shape, + code_values=code_values, + ) + if len(results) != row_count: + results = _fallback_results( + row_count, + "OXC validator returned mismatched result size.", + ) + return pd.DataFrame(results) + + _validator.__name__ = ( + f"{OXC_VALIDATION_FN_MARKER}_{node_lang}_{mode.replace('+', '_')}_{normalized_code_shape}" + ) + return _validator + + +def _run_oxc_batch( + *, + node_lang: str, + validation_mode: str, + code_shape: str, + code_values: list[str], +) -> list[dict[str, Any]]: + if not _OXC_RUNNER_PATH.exists(): + return _fallback_results( + len(code_values), + f"OXC runner missing at {_OXC_RUNNER_PATH}", + ) + + payload = { + "lang": node_lang, + "mode": validation_mode, + "code_shape": code_shape, + "codes": code_values, + } + try: + proc = subprocess.run( + ["node", str(_OXC_RUNNER_PATH)], + cwd=str(_OXC_TOOL_DIR), + input=json.dumps(payload), + text=True, + capture_output=True, + check=False, + ) + except (OSError, ValueError) as exc: + logger.warning("OXC subprocess launch failed: %s", exc) + return _fallback_results(len(code_values), f"OXC launch failed: {exc}") + + if proc.returncode != 0: + message = (proc.stderr or proc.stdout or "unknown error").strip() + if len(message) > 300: + message = f"{message[:300]}..." + return _fallback_results(len(code_values), f"OXC failed: {message}") + + try: + raw = json.loads(proc.stdout) + except json.JSONDecodeError: + return _fallback_results(len(code_values), "OXC output parse failed.") + + if not isinstance(raw, list): + return _fallback_results(len(code_values), "OXC output must be an array.") + + out: list[dict[str, Any]] = [] + for item in raw: + if not isinstance(item, dict): + out.append( + { + "is_valid": False, + "error_count": 1, + "error_message": "Invalid OXC result entry.", + "severity": None, + "code": None, + "labels": [], + "codeframe": None, + "warning_count": 0, + } + ) + continue + is_valid_raw = item.get("is_valid") + error_count_raw = item.get("error_count") + message_raw = item.get("error_message") + severity_raw = item.get("severity") + code_raw = item.get("code") + labels_raw = item.get("labels") + codeframe_raw = item.get("codeframe") + warning_count_raw = item.get("warning_count") + out.append( + { + "is_valid": bool(is_valid_raw) if isinstance(is_valid_raw, bool) else False, + "error_count": int(error_count_raw) if isinstance(error_count_raw, int) else 0, + "error_message": str(message_raw or ""), + "severity": str(severity_raw) if isinstance(severity_raw, str) else None, + "code": str(code_raw) if isinstance(code_raw, str) else None, + "labels": labels_raw if isinstance(labels_raw, list) else [], + "codeframe": str(codeframe_raw) if isinstance(codeframe_raw, str) else None, + "warning_count": int(warning_count_raw) + if isinstance(warning_count_raw, int) + else 0, + } + ) + return out + + +def _fallback_results(row_count: int, message: str) -> list[dict[str, Any]]: + return [ + { + "is_valid": False, + "error_count": 1, + "error_message": message, + "severity": None, + "code": None, + "labels": [], + "codeframe": None, + "warning_count": 0, + } + for _ in range(row_count) + ] diff --git a/studio/backend/core/data_recipe/oxc-validator/package-lock.json b/studio/backend/core/data_recipe/oxc-validator/package-lock.json new file mode 100644 index 0000000000..0e16cc4187 --- /dev/null +++ b/studio/backend/core/data_recipe/oxc-validator/package-lock.json @@ -0,0 +1,794 @@ +{ + "name": "unsloth-oxc-validator-runtime", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "unsloth-oxc-validator-runtime", + "version": "0.0.1", + "dependencies": { + "oxc-parser": "^0.116.0", + "oxlint": "^1.51.0" + } + }, + "node_modules/@emnapi/core": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.8.1.tgz", + "integrity": "sha512-AvT9QFpxK0Zd8J0jopedNm+w/2fIzvtPKPjqyw9jwvBaReTTqPBk9Hixaz7KbjimP+QNz605/XnjFcDAL2pqBg==", + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.1.0", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.1.tgz", + "integrity": "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz", + "integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.1.tgz", + "integrity": "sha512-p64ah1M1ld8xjWv3qbvFwHiFVWrq1yFvV4f7w+mzaqiR4IlSgkqhcRdHwsGgomwzBH51sRY4NEowLxnaBjcW/A==", + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1", + "@tybys/wasm-util": "^0.10.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@oxc-parser/binding-android-arm-eabi": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm-eabi/-/binding-android-arm-eabi-0.116.0.tgz", + "integrity": "sha512-AOET7YIOU3+ANO/3xQeRVGN5Xx6+JGXaIwlqkcHSfxJ/zzw2B6jb0YaLhX45SeRluKVTU8rka4N/tHtNoJjoCg==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-android-arm64": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm64/-/binding-android-arm64-0.116.0.tgz", + "integrity": "sha512-yh0Zvth5cQ6XZkP3QF9MDrXf695zr5XxXq/wBQqpZb0uAgI9wpr98/Hx2RZITMfnNjkIq2VcyU44o3A0bdEmlQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-darwin-arm64": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-arm64/-/binding-darwin-arm64-0.116.0.tgz", + "integrity": "sha512-plcTd/Jska55dToZz6XdRBPRVsj+asjD8QCpQFvt3Wj8pY+10D1pE53Mei3POAS/wSRSy7HiQ2twrm7H2A0CjA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-darwin-x64": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-x64/-/binding-darwin-x64-0.116.0.tgz", + "integrity": "sha512-ahqcF3e3x5Z2ZepzXpZ8ugREdmxvBL+g1nQ0SxO11pIZfck6UtbOtwtdAAxnQXBHHtidu7lPcrBq1SEx26t1PQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-freebsd-x64": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-freebsd-x64/-/binding-freebsd-x64-0.116.0.tgz", + "integrity": "sha512-yo2/LaSXtlzKBurvNbwun/sN/RJwW3XhbMr069FwNVtft7GBnaLLdPIz/sf47icxw/BPViEX6wFvzeD12mtrAg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-arm-gnueabihf": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-0.116.0.tgz", + "integrity": "sha512-EiZeliIPPdFsuaPx8PzDMVijD/4YaUxO46/eYPk5raRocJqjjxOG6GAacQ8UrG3fbrgYjaEChfYL1e8DyE445A==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-arm-musleabihf": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-0.116.0.tgz", + "integrity": "sha512-Nf7hnKRYRSIgglQcLAqE2St4b/Yr6dh+Z7in8mxol065Knevw71XZAiV1fmPSojq6uKPLV9eoH/wFrgr4TnZXw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-arm64-gnu": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-0.116.0.tgz", + "integrity": "sha512-9SJI0S4Qggn3QHpT8Y1jtZceA0m4BlpvO3ne2Wxd33UdTHMmelAnrXryjWutHWQtjCzOwSnFBEoQAdNNyt1u3A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-arm64-musl": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-musl/-/binding-linux-arm64-musl-0.116.0.tgz", + "integrity": "sha512-wMZ6//GI+q1JwO7G2OR51+eA5P8Gr3BobU8RAzCGJptvyGMkWb7KQ1E8s8naVZRr6bSGWAL2p3mCzKOxmEPmrA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-ppc64-gnu": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-0.116.0.tgz", + "integrity": "sha512-5BO0KCzTG2HZTnp3r6SCAOeCs/GwFBQJ1WAOG/ROfDf1fVVEy6hrtLKTLCuUMaamH38v+1+RVEmzRkzBj+rMDQ==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-riscv64-gnu": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-0.116.0.tgz", + "integrity": "sha512-M24gYb/ocVMnLwnH2wY5sLt4sRBkAUHDmfiYtyUYdKTkfPOKtpopd5otsL/BPLnIhpMD8zby4uXVvw7BU0UIlw==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-riscv64-musl": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-0.116.0.tgz", + "integrity": "sha512-LHLXTHCH0bdvGjlitwr1ngeh32GAgq9HYzQ5VAgt0k0UT84AS8AkXj9Spoa9l20fXkVgSvAKcCEkydi4Ol23Dw==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-s390x-gnu": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-0.116.0.tgz", + "integrity": "sha512-VE+XsztuE5jdHvLIDIQMuyDpz5NJGq1Vx/8EXYF0sS/gehlv9GhDpGVWU0SCZ/LjzIy4io/Z0W84UudqufvP3g==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-x64-gnu": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-gnu/-/binding-linux-x64-gnu-0.116.0.tgz", + "integrity": "sha512-rxUkauyjjCmgA7BoR63ogRGEtgubROnCm8AXE9ydg+p42jCGLLqG05mFcS2eC+FYyAU58ZFJNXXeqFW1iCyTGQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-x64-musl": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-musl/-/binding-linux-x64-musl-0.116.0.tgz", + "integrity": "sha512-0zoZlk9MmXe6oTgSh5lT1D51SDC1bfwC96JmE1amMFAPdEbJk5MFRisfTN9TFBpBigQua65842tjaxqMiorAYw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-openharmony-arm64": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-openharmony-arm64/-/binding-openharmony-arm64-0.116.0.tgz", + "integrity": "sha512-PGS7Xqik77U9WMyW626gAD5A2rSN629UvyYJKAl/tgpT+KqZI4+56pJfExhv8IW/PpSHjYHwjmakwobLikz8ww==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-wasm32-wasi": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-wasm32-wasi/-/binding-wasm32-wasi-0.116.0.tgz", + "integrity": "sha512-lGNf/9PU8XxB4Gt1Gr1AKwSrjxGYa6os0PlrT4bpoQsfE3gaZonQTKwJyKhiQdgy7pBCI+ed1LB1NNib1FYULw==", + "cpu": [ + "wasm32" + ], + "license": "MIT", + "optional": true, + "dependencies": { + "@napi-rs/wasm-runtime": "^1.1.1" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@oxc-parser/binding-win32-arm64-msvc": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-0.116.0.tgz", + "integrity": "sha512-tcsOHE31duBSRQXZ7NfdtjmMKZwQYlS00PwAMJ4w5oXs3iPCvisUuIXP7Ko4FzeOBTRvkd64btxtQ6cRM0Kwlw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-win32-ia32-msvc": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-0.116.0.tgz", + "integrity": "sha512-higCz/x+dOQ264YEk22hnu4RDqvjhfehjFORpxoh42QyUxsP6eIembYesBUu5ilALWo0HvRD+m89az2BSTwqpQ==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-win32-x64-msvc": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.116.0.tgz", + "integrity": "sha512-Lg2SRmVHpGG85knDVLbv44r1bYn0OpIV0vg9jVmoEIpDj3Q4kwXuQ6MWVtuslwHR8o2CSiqdBeEn1n1URrs6Eg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.116.0.tgz", + "integrity": "sha512-uOT8S1tlPmDckNxMNtIudN/yXpLdnhlJMX2oLS7cxCd7L0sUF09A/EbSVMWT3Y/iT44IwXCJSJfgfSxXAqWf9Q==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/@oxlint/binding-android-arm-eabi": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm-eabi/-/binding-android-arm-eabi-1.51.0.tgz", + "integrity": "sha512-jJYIqbx4sX+suIxWstc4P7SzhEwb4ArWA2KVrmEuu9vH2i0qM6QIHz/ehmbGE4/2fZbpuMuBzTl7UkfNoqiSgw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-android-arm64": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm64/-/binding-android-arm64-1.51.0.tgz", + "integrity": "sha512-GtXyBCcH4ti98YdiMNCrpBNGitx87EjEWxevnyhcBK12k/Vu4EzSB45rzSC4fGFUD6sQgeaxItRCEEWeVwPafw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-darwin-arm64": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-arm64/-/binding-darwin-arm64-1.51.0.tgz", + "integrity": "sha512-3QJbeYaMHn6Bh2XeBXuITSsbnIctyTjvHf5nRjKYrT9pPeErNIpp5VDEeAXC0CZSwSVTsc8WOSDwgrAI24JolQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-darwin-x64": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-x64/-/binding-darwin-x64-1.51.0.tgz", + "integrity": "sha512-NzErhMaTEN1cY0E8C5APy74lw5VwsNfJfVPBMWPVQLqAbO0k4FFLjvHURvkUL+Y18Wu+8Vs1kbqPh2hjXYA4pg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-freebsd-x64": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-freebsd-x64/-/binding-freebsd-x64-1.51.0.tgz", + "integrity": "sha512-msAIh3vPAoKoHlOE/oe6Q5C/n9umypv/k81lED82ibrJotn+3YG2Qp1kiR8o/Dg5iOEU97c6tl0utxcyFenpFw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm-gnueabihf": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.51.0.tgz", + "integrity": "sha512-CqQPcvqYyMe9ZBot2stjGogEzk1z8gGAngIX7srSzrzexmXixwVxBdFZyxTVM0CjGfDeV+Ru0w25/WNjlMM2Hw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm-musleabihf": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-1.51.0.tgz", + "integrity": "sha512-dstrlYQgZMnyOssxSbolGCge/sDbko12N/35RBNuqLpoPbft2aeBidBAb0dvQlyBd9RJ6u8D4o4Eh8Un6iTgyQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm64-gnu": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.51.0.tgz", + "integrity": "sha512-QEjUpXO7d35rP1/raLGGbAsBLLGZIzV3ZbeSjqWlD3oRnxpRIZ6iL4o51XQHkconn3uKssc+1VKdtHJ81BBhDA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm64-musl": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.51.0.tgz", + "integrity": "sha512-YSJua5irtG4DoMAjUapDTPhkQLHhBIY0G9JqlZS6/SZPzqDkPku/1GdWs0D6h/wyx0Iz31lNCfIaWKBQhzP0wQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-ppc64-gnu": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.51.0.tgz", + "integrity": "sha512-7L4Wj2IEUNDETKssB9IDYt16T6WlF+X2jgC/hBq3diGHda9vJLpAgb09+D3quFq7TdkFtI7hwz/jmuQmQFPc1Q==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-riscv64-gnu": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-1.51.0.tgz", + "integrity": "sha512-cBUHqtOXy76G41lOB401qpFoKx1xq17qYkhWrLSM7eEjiHM9sOtYqpr6ZdqCnN9s6ZpzudX4EkeHOFH2E9q0vA==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-riscv64-musl": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-1.51.0.tgz", + "integrity": "sha512-WKbg8CysgZcHfZX0ixQFBRSBvFZUHa3SBnEjHY2FVYt2nbNJEjzTxA3ZR5wMU0NOCNKIAFUFvAh5/XJKPRJuJg==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-s390x-gnu": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.51.0.tgz", + "integrity": "sha512-N1QRUvJTxqXNSu35YOufdjsAVmKVx5bkrggOWAhTWBc3J4qjcBwr1IfyLh/6YCg8sYRSR1GraldS9jUgJL/U4A==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-x64-gnu": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.51.0.tgz", + "integrity": "sha512-e0Mz0DizsCoqNIjeOg6OUKe8JKJWZ5zZlwsd05Bmr51Jo3AOL4UJnPvwKumr4BBtBrDZkCmOLhCvDGm95nJM2g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-x64-musl": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-musl/-/binding-linux-x64-musl-1.51.0.tgz", + "integrity": "sha512-wD8HGTWhYBKXvRDvoBVB1y+fEYV01samhWQSy1Zkxq2vpezvMnjaFKRuiP6tBNITLGuffbNDEXOwcAhJ3gI5Ug==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-openharmony-arm64": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-openharmony-arm64/-/binding-openharmony-arm64-1.51.0.tgz", + "integrity": "sha512-5NSwQ2hDEJ0GPXqikjWtwzgAQCsS7P9aLMNenjjKa+gknN3lTCwwwERsT6lKXSirfU3jLjexA2XQvQALh5h27w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-arm64-msvc": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.51.0.tgz", + "integrity": "sha512-JEZyah1M0RHMw8d+jjSSJmSmO8sABA1J1RtrHYujGPeCkYg1NeH0TGuClpe2h5QtioRTaF57y/TZfn/2IFV6fA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-ia32-msvc": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-1.51.0.tgz", + "integrity": "sha512-q3cEoKH6kwjz/WRyHwSf0nlD2F5Qw536kCXvmlSu+kaShzgrA0ojmh45CA81qL+7udfCaZL2SdKCZlLiGBVFlg==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-x64-msvc": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.51.0.tgz", + "integrity": "sha512-Q14+fOGb9T28nWF/0EUsYqERiRA7cl1oy4TJrGmLaqhm+aO2cV+JttboHI3CbdeMCAyDI1+NoSlrM7Melhp/cw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", + "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/oxc-parser": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/oxc-parser/-/oxc-parser-0.116.0.tgz", + "integrity": "sha512-ugEo6wwqaqCGcpi7GsLCwSkoD7gIXzvtdaTxE+mbrXFYazU5Q9YdpZdAj9z2b79i/xlv+uW2aAvyzGAlpUzhKQ==", + "license": "MIT", + "dependencies": { + "@oxc-project/types": "^0.116.0" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/sponsors/Boshen" + }, + "optionalDependencies": { + "@oxc-parser/binding-android-arm-eabi": "0.116.0", + "@oxc-parser/binding-android-arm64": "0.116.0", + "@oxc-parser/binding-darwin-arm64": "0.116.0", + "@oxc-parser/binding-darwin-x64": "0.116.0", + "@oxc-parser/binding-freebsd-x64": "0.116.0", + "@oxc-parser/binding-linux-arm-gnueabihf": "0.116.0", + "@oxc-parser/binding-linux-arm-musleabihf": "0.116.0", + "@oxc-parser/binding-linux-arm64-gnu": "0.116.0", + "@oxc-parser/binding-linux-arm64-musl": "0.116.0", + "@oxc-parser/binding-linux-ppc64-gnu": "0.116.0", + "@oxc-parser/binding-linux-riscv64-gnu": "0.116.0", + "@oxc-parser/binding-linux-riscv64-musl": "0.116.0", + "@oxc-parser/binding-linux-s390x-gnu": "0.116.0", + "@oxc-parser/binding-linux-x64-gnu": "0.116.0", + "@oxc-parser/binding-linux-x64-musl": "0.116.0", + "@oxc-parser/binding-openharmony-arm64": "0.116.0", + "@oxc-parser/binding-wasm32-wasi": "0.116.0", + "@oxc-parser/binding-win32-arm64-msvc": "0.116.0", + "@oxc-parser/binding-win32-ia32-msvc": "0.116.0", + "@oxc-parser/binding-win32-x64-msvc": "0.116.0" + } + }, + "node_modules/oxlint": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/oxlint/-/oxlint-1.51.0.tgz", + "integrity": "sha512-g6DNPaV9/WI9MoX2XllafxQuxwY1TV++j7hP8fTJByVBuCoVtm3dy9f/2vtH/HU40JztcgWF4G7ua+gkainklQ==", + "license": "MIT", + "bin": { + "oxlint": "bin/oxlint" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/sponsors/Boshen" + }, + "optionalDependencies": { + "@oxlint/binding-android-arm-eabi": "1.51.0", + "@oxlint/binding-android-arm64": "1.51.0", + "@oxlint/binding-darwin-arm64": "1.51.0", + "@oxlint/binding-darwin-x64": "1.51.0", + "@oxlint/binding-freebsd-x64": "1.51.0", + "@oxlint/binding-linux-arm-gnueabihf": "1.51.0", + "@oxlint/binding-linux-arm-musleabihf": "1.51.0", + "@oxlint/binding-linux-arm64-gnu": "1.51.0", + "@oxlint/binding-linux-arm64-musl": "1.51.0", + "@oxlint/binding-linux-ppc64-gnu": "1.51.0", + "@oxlint/binding-linux-riscv64-gnu": "1.51.0", + "@oxlint/binding-linux-riscv64-musl": "1.51.0", + "@oxlint/binding-linux-s390x-gnu": "1.51.0", + "@oxlint/binding-linux-x64-gnu": "1.51.0", + "@oxlint/binding-linux-x64-musl": "1.51.0", + "@oxlint/binding-openharmony-arm64": "1.51.0", + "@oxlint/binding-win32-arm64-msvc": "1.51.0", + "@oxlint/binding-win32-ia32-msvc": "1.51.0", + "@oxlint/binding-win32-x64-msvc": "1.51.0" + }, + "peerDependencies": { + "oxlint-tsgolint": ">=0.15.0" + }, + "peerDependenciesMeta": { + "oxlint-tsgolint": { + "optional": true + } + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD", + "optional": true + } + } +} diff --git a/studio/backend/core/data_recipe/oxc-validator/package.json b/studio/backend/core/data_recipe/oxc-validator/package.json new file mode 100644 index 0000000000..a47c0ea521 --- /dev/null +++ b/studio/backend/core/data_recipe/oxc-validator/package.json @@ -0,0 +1,10 @@ +{ + "name": "unsloth-oxc-validator-runtime", + "private": true, + "version": "0.0.1", + "type": "module", + "dependencies": { + "oxc-parser": "^0.116.0", + "oxlint": "^1.51.0" + } +} diff --git a/studio/backend/core/data_recipe/oxc-validator/validate.mjs b/studio/backend/core/data_recipe/oxc-validator/validate.mjs new file mode 100644 index 0000000000..7d2f206ce0 --- /dev/null +++ b/studio/backend/core/data_recipe/oxc-validator/validate.mjs @@ -0,0 +1,573 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseSync } from "oxc-parser"; + +const LANG_TO_EXT = { + js: "js", + jsx: "jsx", + ts: "ts", + tsx: "tsx", +}; + +const VALIDATION_MODES = new Set(["syntax", "lint", "syntax+lint"]); +const CODE_SHAPES = new Set(["auto", "module", "snippet"]); +const SNIPPET_PREFIX = "(() => {\n"; +const SNIPPET_SUFFIX = "\n})();\nexport {};\n"; +const OXLINT_SUPPRESSED_RULES = ["no-unused-vars", "no-new-array"]; +const TOOL_DIR = dirname(fileURLToPath(import.meta.url)); + +function mapLang(value) { + const normalized = String(value || "").trim().toLowerCase(); + if (normalized === "javascript" || normalized === "js") { + return "js"; + } + if (normalized === "typescript" || normalized === "ts") { + return "ts"; + } + if (normalized === "jsx") { + return "jsx"; + } + if (normalized === "tsx") { + return "tsx"; + } + return "js"; +} + +function mapMode(value) { + const normalized = String(value || "").trim().toLowerCase(); + if (VALIDATION_MODES.has(normalized)) { + return normalized; + } + return "syntax"; +} + +function mapCodeShape(value) { + const normalized = String(value || "").trim().toLowerCase(); + if (CODE_SHAPES.has(normalized)) { + return normalized; + } + return "auto"; +} + +function parseFileIndex(filePath) { + if (typeof filePath !== "string") { + return null; + } + const match = basename(filePath).match(/^snippet_(\d+)\./); + if (!match) { + return null; + } + const parsed = Number.parseInt(match[1], 10); + return Number.isFinite(parsed) ? parsed : null; +} + +function toCodeString(code) { + return typeof code === "string" ? code : String(code ?? ""); +} + +function makeValidationEntry({ code, index, lang, codeShape }) { + const source = toCodeString(code); + if (codeShape === "snippet") { + return { + index, + lang, + code: `${SNIPPET_PREFIX}${source}${SNIPPET_SUFFIX}`, + offset: SNIPPET_PREFIX.length, + }; + } + return { + index, + lang, + code: source, + offset: 0, + }; +} + +function shiftOffset(value, offset) { + if (!Number.isInteger(value)) { + return null; + } + const shifted = value - offset; + return shifted >= 0 ? shifted : null; +} + +function remapDiagnosticOffsets(diagnostic, offset) { + if (!diagnostic || typeof diagnostic !== "object" || offset <= 0) { + return diagnostic; + } + return { + ...diagnostic, + labels: Array.isArray(diagnostic.labels) + ? diagnostic.labels.map((label) => ({ + ...label, + start: shiftOffset(label.start, offset), + end: shiftOffset(label.end, offset), + })) + : [], + }; +} + +function normalizeParserError(error) { + if (typeof error === "string") { + return { + code: null, + message: error.trim() || "Unknown parser error", + severity: null, + labels: [], + codeframe: null, + }; + } + if (!error || typeof error !== "object") { + return { + code: null, + message: "Unknown parser error", + severity: null, + labels: [], + codeframe: null, + }; + } + const code = typeof error.code === "string" ? error.code : null; + const message = String(error.message || error.reason || "").trim() || "Unknown parser error"; + const severity = typeof error.severity === "string" ? error.severity : null; + const labels = Array.isArray(error.labels) + ? error.labels.map((label) => ({ + message: + label && typeof label === "object" && typeof label.message === "string" + ? label.message + : null, + start: + label && typeof label === "object" && Number.isInteger(label.start) + ? label.start + : null, + end: + label && typeof label === "object" && Number.isInteger(label.end) + ? label.end + : null, + })) + : []; + const codeframe = typeof error.codeframe === "string" ? error.codeframe : null; + return { + code, + message, + severity, + labels, + codeframe, + }; +} + +function normalizeLintDiagnostic(diagnostic) { + if (!diagnostic || typeof diagnostic !== "object") { + return null; + } + + const readString = (value) => + typeof value === "string" ? value : null; + const readInt = (value) => + Number.isInteger(value) ? value : null; + const asObject = (value) => + value && typeof value === "object" ? value : null; + + const message = String(diagnostic.message || "").trim(); + if (!message) { + return null; + } + + const severityRaw = String(diagnostic.severity || "").trim().toLowerCase(); + const severity = severityRaw === "error" ? "error" : "warning"; + + const labels = []; + if (Array.isArray(diagnostic.labels)) { + for (const label of diagnostic.labels) { + const labelObj = asObject(label); + const span = asObject(labelObj?.span); + const start = readInt(span?.offset); + const length = readInt(span?.length); + labels.push({ + message: readString(labelObj?.label), + start, + end: start !== null && length !== null ? start + length : null, + }); + } + } + + const code = typeof diagnostic.code === "string" ? diagnostic.code : null; + return { + code, + message: code ? `${code}: ${message}` : message, + severity, + labels, + codeframe: null, + }; +} + +function makeResult({ + isValid, + errorCount, + warningCount = 0, + message = "", + severity = null, + code = null, + labels = [], + codeframe = null, +}) { + return { + is_valid: Boolean(isValid), + error_count: Number.isInteger(errorCount) ? errorCount : 0, + warning_count: Number.isInteger(warningCount) ? warningCount : 0, + error_message: String(message || ""), + severity: typeof severity === "string" ? severity : null, + code: typeof code === "string" ? code : null, + labels: Array.isArray(labels) ? labels : [], + codeframe: typeof codeframe === "string" ? codeframe : null, + }; +} + +function syntaxResultFromErrors(errors) { + const first = errors[0] ?? null; + return makeResult({ + isValid: errors.length === 0, + errorCount: errors.length, + warningCount: 0, + message: errors.slice(0, 3).map((error) => error.message).join(" | "), + severity: first ? first.severity : null, + code: first ? first.code : null, + labels: first ? first.labels : [], + codeframe: first ? first.codeframe : null, + }); +} + +function runSyntaxParse(entry) { + const ext = LANG_TO_EXT[entry.lang] ?? "js"; + const filename = `snippet_${entry.index}.${ext}`; + try { + const parsed = parseSync(filename, entry.code, { + lang: entry.lang, + sourceType: "module", + showSemanticErrors: true, + }); + const errors = Array.isArray(parsed?.errors) + ? parsed.errors + .map(normalizeParserError) + .filter(Boolean) + .map((error) => remapDiagnosticOffsets(error, entry.offset)) + : []; + return errors; + } catch (error) { + return [ + remapDiagnosticOffsets( + normalizeParserError(error), + entry.offset, + ), + ]; + } +} + +function pickPreferredErrorList(firstErrors, secondErrors) { + if (secondErrors.length < firstErrors.length) { + return secondErrors; + } + return firstErrors; +} + +function validateSyntaxOne({ code, lang, index, codeShape }) { + if (codeShape !== "auto") { + const lintEntry = makeValidationEntry({ + code, + index, + lang, + codeShape, + }); + const errors = runSyntaxParse(lintEntry); + return { + result: syntaxResultFromErrors(errors), + lintEntry, + }; + } + + const moduleEntry = makeValidationEntry({ + code, + index, + lang, + codeShape: "module", + }); + const moduleErrors = runSyntaxParse(moduleEntry); + if (moduleErrors.length === 0) { + return { + result: syntaxResultFromErrors(moduleErrors), + lintEntry: moduleEntry, + }; + } + + const snippetEntry = makeValidationEntry({ + code, + index, + lang, + codeShape: "snippet", + }); + const snippetErrors = runSyntaxParse(snippetEntry); + if (snippetErrors.length === 0) { + return { + result: syntaxResultFromErrors(snippetErrors), + lintEntry: snippetEntry, + }; + } + + const chosenErrors = pickPreferredErrorList(moduleErrors, snippetErrors); + const lintEntry = chosenErrors === snippetErrors ? snippetEntry : moduleEntry; + return { + result: syntaxResultFromErrors(chosenErrors), + lintEntry, + }; +} + +function resolveLintEntry({ code, lang, index, codeShape }) { + if (codeShape !== "auto") { + return makeValidationEntry({ + code, + index, + lang, + codeShape, + }); + } + + const moduleEntry = makeValidationEntry({ + code, + index, + lang, + codeShape: "module", + }); + if (runSyntaxParse(moduleEntry).length === 0) { + return moduleEntry; + } + + const snippetEntry = makeValidationEntry({ + code, + index, + lang, + codeShape: "snippet", + }); + if (runSyntaxParse(snippetEntry).length === 0) { + return snippetEntry; + } + + return moduleEntry; +} + +function fallbackLintResults(entries, message) { + return new Map( + entries.map((entry) => [ + entry.index, + makeResult({ + isValid: false, + errorCount: 1, + warningCount: 0, + message, + severity: "error", + }), + ]), + ); +} + +function runLintBatch(entries) { + if (entries.length === 0) { + return new Map(); + } + + const entryByIndex = new Map(entries.map((entry) => [entry.index, entry])); + const tempDir = mkdtempSync(join(tmpdir(), "oxlint-")); + try { + for (const entry of entries) { + const ext = LANG_TO_EXT[entry.lang] ?? "js"; + const filePath = join(tempDir, `snippet_${entry.index}.${ext}`); + writeFileSync(filePath, entry.code, "utf8"); + } + + const oxlintBin = join(TOOL_DIR, "node_modules", ".bin", "oxlint"); + const oxlintArgs = [ + ...OXLINT_SUPPRESSED_RULES.flatMap((rule) => ["-A", rule]), + "--format", + "json", + tempDir, + ]; + const exec = spawnSync(oxlintBin, oxlintArgs, { + encoding: "utf8", + cwd: TOOL_DIR, + }); + if (exec.error) { + return fallbackLintResults( + entries, + `oxlint execution failed: ${exec.error.message}`, + ); + } + const stdout = String(exec.stdout || "").trim(); + if (!stdout) { + const stderr = String(exec.stderr || "").trim(); + return fallbackLintResults( + entries, + stderr || "oxlint returned empty output", + ); + } + + let parsed; + try { + parsed = JSON.parse(stdout); + } catch { + return fallbackLintResults(entries, "oxlint JSON parse failed"); + } + + const rawDiagnostics = Array.isArray(parsed?.diagnostics) + ? parsed.diagnostics + : []; + const byIndex = new Map(); + + for (const diag of rawDiagnostics) { + const filenameRaw = + typeof diag?.filename === "string" ? diag.filename : ""; + const filename = filenameRaw.startsWith("file://") + ? filenameRaw.replace("file://", "") + : filenameRaw; + const index = parseFileIndex(filename); + if (index === null) { + continue; + } + const normalized = normalizeLintDiagnostic(diag); + if (!normalized) { + continue; + } + const entry = entryByIndex.get(index); + const remapped = remapDiagnosticOffsets(normalized, entry?.offset ?? 0); + const list = byIndex.get(index) ?? []; + list.push(remapped); + byIndex.set(index, list); + } + + const results = new Map(); + for (const entry of entries) { + const diagnostics = byIndex.get(entry.index) ?? []; + const errorDiagnostics = diagnostics.filter( + (diag) => diag.severity === "error", + ); + const warningDiagnostics = diagnostics.filter( + (diag) => diag.severity !== "error", + ); + const top = errorDiagnostics[0] ?? warningDiagnostics[0] ?? null; + const messageSource = + errorDiagnostics.length > 0 ? errorDiagnostics : warningDiagnostics; + results.set( + entry.index, + makeResult({ + isValid: errorDiagnostics.length === 0, + errorCount: errorDiagnostics.length, + warningCount: warningDiagnostics.length, + message: messageSource + .slice(0, 3) + .map((diag) => diag.message) + .join(" | "), + severity: top ? top.severity : null, + code: top ? top.code : null, + labels: top ? top.labels : [], + codeframe: top ? top.codeframe : null, + }), + ); + } + return results; + } catch (error) { + return fallbackLintResults(entries, `oxlint execution failed: ${error}`); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } +} + +function readStdin() { + return new Promise((resolve, reject) => { + let data = ""; + process.stdin.setEncoding("utf8"); + process.stdin.on("data", (chunk) => { + data += chunk; + }); + process.stdin.on("end", () => resolve(data)); + process.stdin.on("error", (error) => reject(error)); + }); +} + +function runValidation({ codes, lang, mode, codeShape }) { + if (mode === "syntax") { + return codes.map((code, index) => + validateSyntaxOne({ code, lang, index, codeShape }).result, + ); + } + + if (mode === "lint") { + const entries = codes.map((code, index) => + resolveLintEntry({ code, lang, index, codeShape }), + ); + const lintMap = runLintBatch(entries); + return entries.map( + (entry) => + lintMap.get(entry.index) ?? + makeResult({ + isValid: true, + errorCount: 0, + warningCount: 0, + }), + ); + } + + const syntaxRuns = codes.map((code, index) => + validateSyntaxOne({ code, lang, index, codeShape }), + ); + const lintTargets = syntaxRuns + .filter((run) => run.result.is_valid === true) + .map((run) => run.lintEntry); + const lintMap = runLintBatch(lintTargets); + + return syntaxRuns.map((run) => { + if (run.result.is_valid !== true) { + return run.result; + } + return ( + lintMap.get(run.lintEntry.index) ?? + makeResult({ + isValid: true, + errorCount: 0, + warningCount: 0, + }) + ); + }); +} + +async function main() { + const raw = await readStdin(); + let payload; + try { + payload = JSON.parse(raw || "{}"); + } catch { + process.stdout.write( + JSON.stringify([ + makeResult({ + isValid: false, + errorCount: 1, + warningCount: 0, + message: "Invalid JSON payload", + severity: "error", + }), + ]), + ); + return; + } + + const lang = mapLang(payload?.lang); + const mode = mapMode(payload?.mode); + const codeShape = mapCodeShape(payload?.code_shape); + const codes = Array.isArray(payload?.codes) ? payload.codes : []; + const out = runValidation({ codes, lang, mode, codeShape }); + process.stdout.write(JSON.stringify(out)); +} + +main().catch((error) => { + process.stderr.write(String(error?.stack || error)); + process.exit(1); +}); diff --git a/studio/backend/core/data_recipe/service.py b/studio/backend/core/data_recipe/service.py index 2d11cb2845..f2c25e62f8 100644 --- a/studio/backend/core/data_recipe/service.py +++ b/studio/backend/core/data_recipe/service.py @@ -1,9 +1,125 @@ from __future__ import annotations +import base64 +import io import os +from pathlib import Path from typing import Any from .jsonable import to_jsonable +from .local_callable_validators import ( + register_oxc_local_callable_validators, + split_oxc_local_callable_validators, +) +_IMAGE_CONTEXT_PATCHED = False + + +def _encode_bytes_to_base64(value: bytes | bytearray) -> str: + return base64.b64encode(bytes(value)).decode("utf-8") + + +def _load_image_file_to_base64(path_value: str, *, base_path: str | None = None) -> str | None: + try: + path = Path(path_value) + candidates: list[Path] = [] + if path.is_absolute(): + candidates.append(path) + else: + if base_path: + candidates.append(Path(base_path) / path) + candidates.append(Path.cwd() / path) + + for candidate in candidates: + if not candidate.exists() or not candidate.is_file(): + continue + with candidate.open("rb") as f: + return _encode_bytes_to_base64(f.read()) + except (OSError, TypeError, ValueError): + return None + return None + + +def _pil_image_to_base64(value: Any) -> str | None: + try: + from PIL.Image import Image as PILImage # type: ignore + except ImportError: + return None + if not isinstance(value, PILImage): + return None + buffer = io.BytesIO() + image_format = str(getattr(value, "format", "") or "").upper() + if image_format not in {"PNG", "JPEG", "JPG", "WEBP", "GIF"}: + image_format = "PNG" + value.save(buffer, format=image_format) + return _encode_bytes_to_base64(buffer.getvalue()) + + +def _normalize_image_context_value(value: Any, *, base_path: str | None = None) -> Any: + if isinstance(value, str): + return value + + if isinstance(value, (bytes, bytearray)): + return _encode_bytes_to_base64(value) + + pil_base64 = _pil_image_to_base64(value) + if pil_base64 is not None: + return pil_base64 + + if isinstance(value, dict): + url = value.get("url") + if isinstance(url, str): + return url + + image_url = value.get("image_url") + if isinstance(image_url, str): + return image_url + if isinstance(image_url, dict): + nested_url = image_url.get("url") + if isinstance(nested_url, str): + return nested_url + + inline_data = value.get("data") + if isinstance(inline_data, str): + return inline_data + + raw_bytes = value.get("bytes") + if isinstance(raw_bytes, (bytes, bytearray)): + return _encode_bytes_to_base64(raw_bytes) + if isinstance(raw_bytes, str) and raw_bytes.strip(): + return raw_bytes + + path_value = value.get("path") + if isinstance(path_value, str) and path_value.strip(): + if as_base64 := _load_image_file_to_base64(path_value, base_path=base_path): + return as_base64 + return path_value + + return value + + +def _apply_data_designer_image_context_patch() -> None: + global _IMAGE_CONTEXT_PATCHED + if _IMAGE_CONTEXT_PATCHED: + return + + try: + from data_designer.config.models import ImageContext + except ImportError: + return + + if getattr(ImageContext, "_unsloth_image_context_patch_applied", False): + _IMAGE_CONTEXT_PATCHED = True + return + + original_auto_resolve = ImageContext._auto_resolve_context_value + + def _patched_auto_resolve(self: Any, context_value: Any, base_path: str | None) -> Any: + normalized = _normalize_image_context_value(context_value, base_path=base_path) + return original_auto_resolve(self, normalized, base_path) + + ImageContext._auto_resolve_context_value = _patched_auto_resolve + setattr(ImageContext, "_unsloth_image_context_patch_applied", True) + _IMAGE_CONTEXT_PATCHED = True def build_model_providers(recipe: dict[str, Any]): @@ -75,6 +191,7 @@ def build_mcp_providers( def build_config_builder(recipe: dict[str, Any]): + _apply_data_designer_image_context_patch() from data_designer.config import DataDesignerConfigBuilder from data_designer.config.processors import ProcessorType @@ -83,7 +200,14 @@ def build_config_builder(recipe: dict[str, Any]): for key, value in recipe.items() if key not in {"model_providers", "mcp_providers"} } + recipe_core, oxc_local_callable_specs = split_oxc_local_callable_validators( + recipe_core + ) builder = DataDesignerConfigBuilder.from_config({"data_designer": recipe_core}) + register_oxc_local_callable_validators( + builder=builder, + specs=oxc_local_callable_specs, + ) # DataDesignerConfigBuilder.from_config currently skips processors. # Re-attach explicitly so drop_columns/schema_transform survive API payload. @@ -107,6 +231,7 @@ def create_data_designer( *, artifact_path: str | None = None, ): + _apply_data_designer_image_context_patch() from data_designer.interface.data_designer import DataDesigner return DataDesigner( diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py index 40d653b69a..22c0f34075 100644 --- a/studio/backend/core/training/trainer.py +++ b/studio/backend/core/training/trainer.py @@ -18,6 +18,7 @@ import threading import math import logging import time +from pathlib import Path from typing import Optional, Callable from dataclasses import dataclass import pandas as pd @@ -31,6 +32,9 @@ from trl import SFTTrainer, SFTConfig logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +_BACKEND_ROOT = Path(__file__).resolve().parents[2] +_ASSETS_DATASETS_ROOT = _BACKEND_ROOT / "assets" / "datasets" + @dataclass class TrainingProgress: @@ -1803,20 +1807,38 @@ class UnslothTrainer: file_path = dataset_file else: # Fallback: try relative to assets/datasets - script_dir = Path(__file__).parent.parent - assets_datasets_dir = script_dir / "assets" / "datasets" - file_path = assets_datasets_dir / dataset_file - - if str(file_path).endswith('.json'): - with open(file_path, 'r', encoding='utf-8') as f: + file_path = _ASSETS_DATASETS_ROOT / dataset_file + + file_path_obj = Path(file_path) + file_path_str = str(file_path_obj) + + if file_path_obj.is_dir(): + parquet_dir = ( + file_path_obj / "parquet-files" + if (file_path_obj / "parquet-files").exists() + else file_path_obj + ) + parquet_files = sorted(parquet_dir.glob("*.parquet")) + if parquet_files: + for parquet_file in parquet_files: + df = pd.read_parquet(parquet_file) + all_data.extend(df.to_dict("records")) + continue + + if file_path_str.endswith('.json'): + with open(file_path_obj, 'r', encoding='utf-8') as f: data = json.load(f) if isinstance(data, list): all_data.extend(data) else: all_data.append(data) - elif str(file_path).endswith('.csv'): - df = pd.read_csv(file_path) + elif file_path_str.endswith('.csv'): + df = pd.read_csv(file_path_obj) all_data.extend(df.to_dict('records')) + elif file_path_str.endswith('.parquet'): + df = pd.read_parquet(file_path_obj) + all_data.extend(df.to_dict('records')) + continue if all_data: dataset = Dataset.from_list(all_data) diff --git a/studio/backend/models/data_recipe.py b/studio/backend/models/data_recipe.py index 9e501c15e2..01e7c8cd1f 100644 --- a/studio/backend/models/data_recipe.py +++ b/studio/backend/models/data_recipe.py @@ -49,6 +49,9 @@ class SeedInspectUploadRequest(BaseModel): filename: str = Field(min_length=1) content_base64: str = Field(min_length=1) preview_size: int = Field(default=10, ge=1, le=50) + seed_source_type: str | None = None + unstructured_chunk_size: int | None = Field(default=None, ge=1, le=20000) + unstructured_chunk_overlap: int | None = Field(default=None, ge=0, le=20000) class SeedInspectResponse(BaseModel): diff --git a/studio/backend/models/datasets.py b/studio/backend/models/datasets.py index 0fd3db7955..19197d99ad 100644 --- a/studio/backend/models/datasets.py +++ b/studio/backend/models/datasets.py @@ -1,8 +1,9 @@ """ Dataset-related Pydantic models for API requests and responses. """ -from pydantic import BaseModel, model_validator -from typing import Any, Optional, Dict, List +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, Field, model_validator class CheckFormatRequest(BaseModel): @@ -38,3 +39,23 @@ class CheckFormatResponse(BaseModel): preview_samples: Optional[List[Dict]] = None total_rows: Optional[int] = None warning: Optional[str] = None + + +class LocalDatasetItem(BaseModel): + class Metadata(BaseModel): + actual_num_records: Optional[int] = None + target_num_records: Optional[int] = None + total_num_batches: Optional[int] = None + num_completed_batches: Optional[int] = None + columns: Optional[List[str]] = None + + id: str + label: str + path: str + rows: Optional[int] = None + updated_at: Optional[float] = None + metadata: Optional[Metadata] = None + + +class LocalDatasetsResponse(BaseModel): + datasets: List[LocalDatasetItem] = Field(default_factory=list) diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index ab5c63d8c9..d285d99507 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -29,6 +29,34 @@ class UnloadRequest(BaseModel): model_path: str = Field(..., description="Model identifier to unload") +class ValidateModelRequest(BaseModel): + """ + Lightweight validation request to check whether a model identifier + *can be resolved* into a ModelConfig. + + This does NOT actually load weights into GPU memory. + """ + model_path: str = Field(..., description="Model identifier or local path") + hf_token: Optional[str] = Field(None, description="HuggingFace token for gated models") + gguf_variant: Optional[str] = Field(None, description="GGUF quantization variant (e.g. 'Q4_K_M')") + + +class ValidateModelResponse(BaseModel): + """ + Result of model validation. + + valid == True means ModelConfig.from_identifier() succeeded and basic + introspection (GGUF / LoRA / vision flags) is available. + """ + valid: bool = Field(..., description="Whether the model identifier looks valid") + message: str = Field(..., description="Human-readable validation message") + identifier: Optional[str] = Field(None, description="Resolved model identifier") + display_name: Optional[str] = Field(None, description="Display name derived from identifier") + is_gguf: bool = Field(False, description="Whether this is a GGUF model (llama.cpp)") + is_lora: bool = Field(False, description="Whether this is a LoRA adapter") + is_vision: bool = Field(False, description="Whether this is a vision-capable model") + + class GenerateRequest(BaseModel): """Request for text generation (legacy /generate/stream endpoint)""" messages: List[dict] = Field(..., description="Chat messages in OpenAI format") diff --git a/studio/backend/plugins/data-designer-unstructured-seed/pyproject.toml b/studio/backend/plugins/data-designer-unstructured-seed/pyproject.toml new file mode 100644 index 0000000000..5b83e5468a --- /dev/null +++ b/studio/backend/plugins/data-designer-unstructured-seed/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "data-designer-unstructured-seed" +version = "0.1.0" +description = "Local Data Designer unstructured seed reader plugin" +requires-python = ">=3.11" +dependencies = [ + "data-designer-engine>=0.5.1,<0.6", + "pandas>=2,<3", +] + +[project.entry-points."data_designer.plugins"] +unstructured = "data_designer_unstructured_seed.plugin:unstructured_seed_plugin" + +[tool.setuptools] +package-dir = {"" = "src"} + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/__init__.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/__init__.py new file mode 100644 index 0000000000..0d9b5ce7eb --- /dev/null +++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/__init__.py @@ -0,0 +1,21 @@ +from .chunking import ( + DEFAULT_CHUNK_OVERLAP, + DEFAULT_CHUNK_SIZE, + build_unstructured_preview_rows, + materialize_unstructured_seed_dataset, + resolve_chunking, +) +from .config import UnstructuredSeedSource +from .impl import UnstructuredSeedReader +from .plugin import unstructured_seed_plugin + +__all__ = [ + "DEFAULT_CHUNK_OVERLAP", + "DEFAULT_CHUNK_SIZE", + "build_unstructured_preview_rows", + "materialize_unstructured_seed_dataset", + "resolve_chunking", + "UnstructuredSeedSource", + "UnstructuredSeedReader", + "unstructured_seed_plugin", +] diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py new file mode 100644 index 0000000000..4ce7d360bc --- /dev/null +++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +import hashlib +import re +from pathlib import Path +from typing import Any + +DEFAULT_CHUNK_SIZE = 1200 +DEFAULT_CHUNK_OVERLAP = 200 +MAX_CHUNK_SIZE = 20000 +_MIN_BREAK_RATIO = 0.6 +_CACHE_DIR = Path.home() / ".cache" / "unsloth" / "data-recipe" / "unstructured-seed-cache" + + +def resolve_chunking( + chunk_size: Any, + chunk_overlap: Any, +) -> tuple[int, int]: + size = _to_int(chunk_size, DEFAULT_CHUNK_SIZE) + size = max(1, min(size, MAX_CHUNK_SIZE)) + overlap = _to_int(chunk_overlap, DEFAULT_CHUNK_OVERLAP) + overlap = max(0, min(overlap, max(0, size - 1))) + return size, overlap + + +def build_unstructured_preview_rows( + *, + source_path: Path, + preview_size: int, + chunk_size: Any, + chunk_overlap: Any, +) -> list[dict[str, str]]: + parquet_path, rows = materialize_unstructured_seed_dataset( + source_path=source_path, + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + ) + count = max(0, int(preview_size)) + if rows: + return rows[:count] + + try: + import pandas as pd + except ImportError as exc: # pragma: no cover + raise RuntimeError(f"pandas is required for unstructured seed processing: {exc}") from exc + + dataframe = pd.read_parquet(parquet_path).head(count) + return [ + {"chunk_text": str(value.get("chunk_text", "")).strip()} + for value in dataframe.to_dict(orient="records") + if str(value.get("chunk_text", "")).strip() + ] + + +def materialize_unstructured_seed_dataset( + *, + source_path: Path, + chunk_size: Any, + chunk_overlap: Any, +) -> tuple[Path, list[dict[str, str]]]: + resolved = source_path.expanduser().resolve() + if not resolved.is_file(): + raise FileNotFoundError(f"unstructured seed file not found: {resolved}") + + size, overlap = resolve_chunking(chunk_size, chunk_overlap) + key = _compute_cache_key( + source_path=resolved, + chunk_size=size, + chunk_overlap=overlap, + ) + parquet_path = _CACHE_DIR / f"{key}.parquet" + if parquet_path.exists(): + return parquet_path, [] + + text = load_unstructured_text_file(resolved) + chunks = split_text_into_chunks( + text=text, + chunk_size=size, + chunk_overlap=overlap, + ) + if not chunks: + raise ValueError("No text found in unstructured seed source.") + + rows = [{"chunk_text": chunk} for chunk in chunks] + _CACHE_DIR.mkdir(parents=True, exist_ok=True) + try: + import pandas as pd + except ImportError as exc: # pragma: no cover + raise RuntimeError(f"pandas is required for unstructured seed processing: {exc}") from exc + + tmp_path = _CACHE_DIR / f"{key}.tmp.parquet" + pd.DataFrame(rows).to_parquet(tmp_path, index=False) + tmp_path.replace(parquet_path) + return parquet_path, rows + + +def load_unstructured_text_file(path: Path) -> str: + ext = path.suffix.lower() + if ext not in {".txt", ".md"}: + raise ValueError(f"Unsupported unstructured seed file type: {ext}") + + raw = path.read_text(encoding="utf-8", errors="ignore") + return normalize_unstructured_text(raw) + + +def normalize_unstructured_text(text: str) -> str: + normalized = text.replace("\r\n", "\n").replace("\r", "\n") + return re.sub(r"\n{3,}", "\n\n", normalized).strip() + + +def split_text_into_chunks( + *, + text: str, + chunk_size: int, + chunk_overlap: int, +) -> list[str]: + if not text: + return [] + if chunk_size <= 0: + return [text] + + chunks: list[str] = [] + start = 0 + min_break_index = int(chunk_size * _MIN_BREAK_RATIO) + text_len = len(text) + while start < text_len: + end = min(text_len, start + chunk_size) + if end < text_len: + window = text[start:end] + cut = _find_break_index(window, min_break_index) + if cut is not None and cut > 0: + end = start + cut + + if end <= start: + end = min(text_len, start + chunk_size) + + chunk = text[start:end].strip() + if chunk: + chunks.append(chunk) + if end >= text_len: + break + + next_start = end - chunk_overlap + if next_start <= start: + next_start = end + start = max(0, next_start) + + return chunks + + +def _find_break_index(window: str, min_index: int) -> int | None: + breakpoints = ["\n\n", "\n", " "] + for token in breakpoints: + idx = window.rfind(token) + if idx >= min_index: + return idx + len(token) + return None + + +def _to_int(value: Any, fallback: int) -> int: + if isinstance(value, bool): + return fallback + try: + parsed = int(str(value).strip()) + except (TypeError, ValueError): + return fallback + return parsed + + +def _compute_cache_key( + *, + source_path: Path, + chunk_size: int, + chunk_overlap: int, +) -> str: + stat = source_path.stat() + payload = "|".join( + [ + str(source_path), + str(stat.st_size), + str(stat.st_mtime_ns), + str(chunk_size), + str(chunk_overlap), + ] + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/config.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/config.py new file mode 100644 index 0000000000..487579fda1 --- /dev/null +++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/config.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Literal + +from pydantic import Field, field_validator + +from data_designer.config.seed_source import SeedSource + +from .chunking import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, resolve_chunking + + +class UnstructuredSeedSource(SeedSource): + seed_type: Literal["unstructured"] = "unstructured" + path: str = Field(..., min_length=1) + chunk_size: int = DEFAULT_CHUNK_SIZE + chunk_overlap: int = DEFAULT_CHUNK_OVERLAP + + @field_validator("path", mode="after") + @classmethod + def _validate_path(cls, value: str) -> str: + path = Path(value).expanduser() + if not path.is_file(): + raise ValueError(f"Unstructured seed path is not a file: {path}") + return value + + @field_validator("chunk_size", mode="after") + @classmethod + def _validate_chunk_size(cls, value: int) -> int: + size, _ = resolve_chunking(value, 0) + return size + + @field_validator("chunk_overlap", mode="after") + @classmethod + def _validate_chunk_overlap(cls, value: int, info) -> int: + size = info.data.get("chunk_size", cls.model_fields["chunk_size"].default) + _, overlap = resolve_chunking(size, value) + return overlap diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py new file mode 100644 index 0000000000..0c81f636bf --- /dev/null +++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from pathlib import Path + +import data_designer.lazy_heavy_imports as lazy +from data_designer.engine.resources.seed_reader import SeedReader + +from .chunking import materialize_unstructured_seed_dataset +from .config import UnstructuredSeedSource + + +class UnstructuredSeedReader(SeedReader[UnstructuredSeedSource]): + def create_duckdb_connection(self): + return lazy.duckdb.connect() + + def get_dataset_uri(self) -> str: + path, _ = materialize_unstructured_seed_dataset( + source_path=Path(self.source.path), + chunk_size=self.source.chunk_size, + chunk_overlap=self.source.chunk_overlap, + ) + return str(path) diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/plugin.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/plugin.py new file mode 100644 index 0000000000..e96d1d2cd6 --- /dev/null +++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/plugin.py @@ -0,0 +1,7 @@ +from data_designer.plugins.plugin import Plugin, PluginType + +unstructured_seed_plugin = Plugin( + impl_qualified_name="data_designer_unstructured_seed.impl.UnstructuredSeedReader", + config_qualified_name="data_designer_unstructured_seed.config.UnstructuredSeedSource", + plugin_type=PluginType.SEED_READER, +) diff --git a/studio/backend/routes/data_recipe/__init__.py b/studio/backend/routes/data_recipe/__init__.py index dc0301d1c9..0d3f4febf9 100644 --- a/studio/backend/routes/data_recipe/__init__.py +++ b/studio/backend/routes/data_recipe/__init__.py @@ -5,7 +5,9 @@ from __future__ import annotations import sys from pathlib import Path -from fastapi import APIRouter +from fastapi import APIRouter, Depends + +from auth.authentication import get_current_subject backend_path = Path(__file__).parent.parent.parent if str(backend_path) not in sys.path: @@ -15,7 +17,7 @@ from .jobs import router as jobs_router from .seed import router as seed_router from .validate import router as validate_router -router = APIRouter() +router = APIRouter(dependencies=[Depends(get_current_subject)]) router.include_router(seed_router) router.include_router(validate_router) router.include_router(jobs_router) diff --git a/studio/backend/routes/data_recipe/jobs.py b/studio/backend/routes/data_recipe/jobs.py index ffbded9474..67a64ec784 100644 --- a/studio/backend/routes/data_recipe/jobs.py +++ b/studio/backend/routes/data_recipe/jobs.py @@ -14,6 +14,17 @@ from models.data_recipe import JobCreateResponse, RecipePayload router = APIRouter() +def _normalize_run_name(value: Any) -> str | None: + if value is None: + return None + if not isinstance(value, str): + raise HTTPException(status_code=400, detail="invalid run_name: must be a string") + trimmed = value.strip() + if not trimmed: + return None + return trimmed[:120] + + @router.post("/jobs", response_class=JSONResponse, response_model=JobCreateResponse) def create_job(payload: RecipePayload): recipe = payload.recipe @@ -27,6 +38,7 @@ def create_job(payload: RecipePayload): if execution_type not in {"preview", "full"}: raise HTTPException(status_code=400, detail="invalid execution_type: must be 'preview' or 'full'") run["execution_type"] = execution_type + run["run_name"] = _normalize_run_name(run.get("run_name")) run_config_raw = run.get("run_config") if run_config_raw is not None: try: diff --git a/studio/backend/routes/data_recipe/seed.py b/studio/backend/routes/data_recipe/seed.py index eb02ab2bbf..ce6073d902 100644 --- a/studio/backend/routes/data_recipe/seed.py +++ b/studio/backend/routes/data_recipe/seed.py @@ -10,6 +10,11 @@ from typing import Any from uuid import uuid4 from fastapi import APIRouter, HTTPException +from data_designer_unstructured_seed.chunking import ( + build_unstructured_preview_rows, + resolve_chunking, +) +from core.data_recipe.jsonable import to_preview_jsonable from models.data_recipe import ( SeedInspectRequest, @@ -22,17 +27,12 @@ router = APIRouter() DATA_EXTS = (".parquet", ".jsonl", ".json", ".csv") DEFAULT_SPLIT = "train" LOCAL_UPLOAD_EXTS = {".csv", ".json", ".jsonl"} +UNSTRUCTURED_UPLOAD_EXTS = {".txt", ".md"} SEED_UPLOAD_DIR = Path.home() / ".cache" / "unsloth" / "data-recipe" / "seed-uploads" def _serialize_preview_value(value: Any) -> Any: - if value is None or isinstance(value, (str, int, float, bool)): - return value - if isinstance(value, dict): - return {str(key): _serialize_preview_value(item) for key, item in value.items()} - if isinstance(value, (list, tuple)): - return [_serialize_preview_value(item) for item in value] - return str(value) + return to_preview_jsonable(value) def _serialize_preview_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: @@ -63,10 +63,10 @@ def _list_hf_data_files(*, dataset_name: str, token: str | None) -> list[str]: return [] -def _select_best_file(data_files: list[str]) -> str | None: +def _select_best_file(data_files: list[str], split: str = DEFAULT_SPLIT) -> str | None: if not data_files: return None - split_lower = DEFAULT_SPLIT + split_lower = split.lower() def score(path: str) -> tuple[int, int]: name = path.lower() @@ -85,8 +85,8 @@ def _select_best_file(data_files: list[str]) -> str | None: return sorted(data_files, key=score)[0] -def _resolve_seed_hf_path(dataset_name: str, data_files: list[str]) -> str | None: - selected = _select_best_file(data_files) +def _resolve_seed_hf_path(dataset_name: str, data_files: list[str], split: str = DEFAULT_SPLIT) -> str | None: + selected = _select_best_file(data_files, split) if not selected: return None @@ -185,6 +185,26 @@ def _read_preview_rows_from_local_file(path: Path, preview_size: int) -> list[di return _serialize_preview_rows(rows) +def _read_preview_rows_from_unstructured_file( + *, + path: Path, + preview_size: int, + chunk_size: int | None, + chunk_overlap: int | None, +) -> list[dict[str, Any]]: + size, overlap = resolve_chunking(chunk_size, chunk_overlap) + try: + rows = build_unstructured_preview_rows( + source_path=path, + preview_size=preview_size, + chunk_size=size, + chunk_overlap=overlap, + ) + except (FileNotFoundError, RuntimeError, ValueError, OSError) as exc: + raise HTTPException(status_code=422, detail=f"seed inspect failed: {exc}") from exc + return _serialize_preview_rows(rows) + + @router.post("/seed/inspect", response_model=SeedInspectResponse) def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse: dataset_name = payload.dataset_name.strip() @@ -196,7 +216,7 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse: except ImportError as exc: raise HTTPException(status_code=500, detail=f"seed inspect dependencies unavailable: {exc}") from exc - split = DEFAULT_SPLIT + split = _normalize_optional_text(payload.split) or DEFAULT_SPLIT subset = _normalize_optional_text(payload.subset) token = _normalize_optional_text(payload.hf_token) preview_size = int(payload.preview_size) @@ -204,12 +224,12 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse: preview_rows: list[dict[str, Any]] = [] data_files = _list_hf_data_files(dataset_name=dataset_name, token=token) - selected_file = _select_best_file(data_files) + selected_file = _select_best_file(data_files, split) if selected_file: try: single_file_kwargs = _build_stream_load_kwargs( dataset_name=dataset_name, - split=DEFAULT_SPLIT, + split=split, subset=subset, token=token, data_file=selected_file, @@ -246,7 +266,7 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse: if not data_files: resolved_path = f"datasets/{dataset_name}/**/*.parquet" else: - resolved_path = _resolve_seed_hf_path(dataset_name, data_files) + resolved_path = _resolve_seed_hf_path(dataset_name, data_files, split) if not resolved_path: raise HTTPException(status_code=422, detail="unable to resolve seed dataset path") @@ -255,18 +275,24 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse: resolved_path=resolved_path, columns=columns, preview_rows=preview_rows, - split=None, + split=split, subset=subset, ) @router.post("/seed/inspect-upload", response_model=SeedInspectResponse) def inspect_seed_upload(payload: SeedInspectUploadRequest) -> SeedInspectResponse: + seed_source_type = _normalize_optional_text(payload.seed_source_type) or "local" filename = _sanitize_filename(payload.filename) ext = Path(filename).suffix.lower() - if ext not in LOCAL_UPLOAD_EXTS: - allowed = ", ".join(sorted(LOCAL_UPLOAD_EXTS)) - raise HTTPException(status_code=400, detail=f"unsupported file type: {ext}. allowed: {allowed}") + if seed_source_type == "unstructured": + if ext not in UNSTRUCTURED_UPLOAD_EXTS: + allowed = ", ".join(sorted(UNSTRUCTURED_UPLOAD_EXTS)) + raise HTTPException(status_code=400, detail=f"unsupported file type: {ext}. allowed: {allowed}") + else: + if ext not in LOCAL_UPLOAD_EXTS: + allowed = ", ".join(sorted(LOCAL_UPLOAD_EXTS)) + raise HTTPException(status_code=400, detail=f"unsupported file type: {ext}. allowed: {allowed}") file_bytes = _decode_base64_payload(payload.content_base64) if not file_bytes: @@ -280,10 +306,18 @@ def inspect_seed_upload(payload: SeedInspectUploadRequest) -> SeedInspectRespons stored_path = SEED_UPLOAD_DIR / stored_name stored_path.write_bytes(file_bytes) - preview_rows = _read_preview_rows_from_local_file( - stored_path, - int(payload.preview_size), - ) + if seed_source_type == "unstructured": + preview_rows = _read_preview_rows_from_unstructured_file( + path=stored_path, + preview_size=int(payload.preview_size), + chunk_size=payload.unstructured_chunk_size, + chunk_overlap=payload.unstructured_chunk_overlap, + ) + else: + preview_rows = _read_preview_rows_from_local_file( + stored_path, + int(payload.preview_size), + ) if not preview_rows: raise HTTPException(status_code=422, detail="dataset appears empty or unreadable") columns = _extract_columns(preview_rows) diff --git a/studio/backend/routes/datasets.py b/studio/backend/routes/datasets.py index 4669f05a93..04345a944d 100644 --- a/studio/backend/routes/datasets.py +++ b/studio/backend/routes/datasets.py @@ -3,6 +3,7 @@ Datasets API routes """ import base64 import io +import json import sys from pathlib import Path from fastapi import APIRouter, Depends, HTTPException @@ -30,7 +31,12 @@ if not logger.handlers: logger.setLevel(logging.INFO) -from models.datasets import CheckFormatRequest, CheckFormatResponse +from models.datasets import ( + CheckFormatRequest, + CheckFormatResponse, + LocalDatasetItem, + LocalDatasetsResponse, +) def _serialize_preview_value(value): @@ -82,6 +88,175 @@ DATA_EXTS = ( '.gz', '.zst', '.zip', ) +LOCAL_FILE_EXTS = ('.json', '.jsonl', '.csv', '.parquet') +BACKEND_ROOT = Path(__file__).resolve().parents[1] +LOCAL_DATASETS_ROOT = BACKEND_ROOT / "assets" / "datasets" + + +def _safe_read_metadata(path: Path) -> dict | None: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError, TypeError): + return None + if not isinstance(payload, dict): + return None + return payload + + +def _safe_read_rows_from_metadata(payload: dict | None) -> int | None: + if not payload: + return None + for key in ("actual_num_records", "target_num_records"): + value = payload.get(key) + if isinstance(value, int): + return value + return None + + +def _safe_read_metadata_summary(payload: dict | None) -> dict | None: + if not payload: + return None + + actual_num_records = ( + payload.get("actual_num_records") + if isinstance(payload.get("actual_num_records"), int) + else None + ) + target_num_records = ( + payload.get("target_num_records") + if isinstance(payload.get("target_num_records"), int) + else actual_num_records + ) + + columns: list[str] | None = None + schema = payload.get("schema") + if isinstance(schema, dict): + columns = [str(key) for key in schema.keys()] + if not columns: + stats = payload.get("column_statistics") + if isinstance(stats, list): + derived = [ + str(item.get("column_name")) + for item in stats + if isinstance(item, dict) and item.get("column_name") + ] + columns = derived or None + + parquet_files_count = None + file_paths = payload.get("file_paths") + if isinstance(file_paths, dict): + parquet_files = file_paths.get("parquet-files") + if isinstance(parquet_files, list): + parquet_files_count = len(parquet_files) + + total_num_batches = ( + payload.get("total_num_batches") + if isinstance(payload.get("total_num_batches"), int) + else parquet_files_count + ) + num_completed_batches = ( + payload.get("num_completed_batches") + if isinstance(payload.get("num_completed_batches"), int) + else total_num_batches + ) + + return { + "actual_num_records": actual_num_records, + "target_num_records": target_num_records, + "total_num_batches": total_num_batches, + "num_completed_batches": num_completed_batches, + "columns": columns, + } + + +def _build_local_dataset_items() -> list[LocalDatasetItem]: + if not LOCAL_DATASETS_ROOT.exists(): + return [] + + items: list[LocalDatasetItem] = [] + for entry in LOCAL_DATASETS_ROOT.iterdir(): + if not entry.is_dir() or not entry.name.startswith("recipe_"): + continue + parquet_dir = entry / "parquet-files" + if not parquet_dir.exists() or not any(parquet_dir.glob("*.parquet")): + continue + + rows = None + metadata_summary = None + metadata_path = entry / "metadata.json" + if metadata_path.exists(): + metadata_payload = _safe_read_metadata(metadata_path) + rows = _safe_read_rows_from_metadata(metadata_payload) + metadata_summary = _safe_read_metadata_summary(metadata_payload) + + try: + updated_at = entry.stat().st_mtime + except OSError: + updated_at = None + + items.append( + LocalDatasetItem( + id=entry.name, + label=entry.name, + path=str(parquet_dir.resolve()), + rows=rows, + updated_at=updated_at, + metadata=metadata_summary, + ) + ) + + items.sort(key=lambda item: item.updated_at or 0, reverse=True) + return items + + +def _load_local_preview_slice(*, dataset_path: Path, train_split: str, preview_size: int): + from datasets import load_dataset + + if dataset_path.is_dir(): + parquet_dir = dataset_path / "parquet-files" if (dataset_path / "parquet-files").exists() else dataset_path + parquet_files = sorted(parquet_dir.glob("*.parquet")) + if parquet_files: + dataset = load_dataset( + "parquet", + data_files=[str(path) for path in parquet_files], + split=train_split, + ) + total_rows = len(dataset) + preview_slice = dataset.select(range(min(preview_size, total_rows))) + return preview_slice, total_rows + else: + candidate_files: list[Path] = [] + for ext in LOCAL_FILE_EXTS: + candidate_files.extend(sorted(dataset_path.glob(f"*{ext}"))) + if not candidate_files: + raise HTTPException( + status_code=400, + detail="Unsupported local dataset directory (expected parquet/json/jsonl/csv files)", + ) + dataset_path = candidate_files[0] + + if dataset_path.suffix in ['.json', '.jsonl']: + dataset = load_dataset('json', data_files=str(dataset_path), split=train_split) + elif dataset_path.suffix == '.csv': + dataset = load_dataset('csv', data_files=str(dataset_path), split=train_split) + elif dataset_path.suffix == '.parquet': + dataset = load_dataset('parquet', data_files=str(dataset_path), split=train_split) + else: + raise HTTPException( + status_code=400, + detail=f"Unsupported file format: {dataset_path.suffix}" + ) + + total_rows = len(dataset) + preview_slice = dataset.select(range(min(preview_size, total_rows))) + return preview_slice, total_rows + + +@router.get("/local", response_model=LocalDatasetsResponse) +def list_local_datasets( + current_subject: str = Depends(get_current_subject), +) -> LocalDatasetsResponse: + return LocalDatasetsResponse(datasets=_build_local_dataset_items()) @router.post("/check-format", response_model=CheckFormatResponse) @@ -116,19 +291,12 @@ def check_format( if dataset_path.exists(): # ── Local file ────────────────────────────────────────── - if dataset_path.suffix in ['.json', '.jsonl']: - dataset = load_dataset('json', data_files=str(dataset_path), split=request.train_split) - elif dataset_path.suffix == '.csv': - dataset = load_dataset('csv', data_files=str(dataset_path), split=request.train_split) - elif dataset_path.suffix == '.parquet': - dataset = load_dataset('parquet', data_files=str(dataset_path), split=request.train_split) - else: - raise HTTPException( - status_code=400, - detail=f"Unsupported file format: {dataset_path.suffix}" - ) - total_rows = len(dataset) - preview_slice = dataset.select(range(min(PREVIEW_SIZE, total_rows))) + train_split = request.train_split or "train" + preview_slice, total_rows = _load_local_preview_slice( + dataset_path=dataset_path, + train_split=train_split, + preview_size=PREVIEW_SIZE, + ) else: # ── HuggingFace dataset ───────────────────────────────── # Tier 1: list_repo_files → load only the first data file diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 263700a68c..0b3dc2a2f8 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -51,6 +51,8 @@ from models.inference import ( ChoiceDelta, CompletionChoice, CompletionMessage, + ValidateModelRequest, + ValidateModelResponse, ) from auth.authentication import get_current_subject @@ -277,6 +279,50 @@ async def load_model( ) +@router.post("/validate", response_model=ValidateModelResponse) +async def validate_model( + request: ValidateModelRequest, + current_subject: str = Depends(get_current_subject), +): + """ + Lightweight validation endpoint for model identifiers. + + This checks that ModelConfig.from_identifier() can resolve the given + model_path, but it does NOT actually load model weights into GPU memory. + """ + try: + config = ModelConfig.from_identifier( + model_id=request.model_path, + hf_token=request.hf_token, + gguf_variant=request.gguf_variant, + ) + + if not config: + raise HTTPException( + status_code=400, + detail=f"Invalid model identifier: {request.model_path}", + ) + + return ValidateModelResponse( + valid=True, + message="Model identifier is valid.", + identifier=config.identifier, + display_name=getattr(config, "display_name", config.identifier), + is_gguf=getattr(config, "is_gguf", False), + is_lora=getattr(config, "is_lora", False), + is_vision=getattr(config, "is_vision", False), + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error validating model identifier '{request.model_path}': {e}", exc_info=True) + raise HTTPException( + status_code=400, + detail=f"Invalid model: {str(e)}", + ) + + @router.post("/unload", response_model=UnloadResponse) async def unload_model( request: UnloadRequest, diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py index c0e78d2b24..545ef18747 100644 --- a/studio/backend/routes/training.py +++ b/studio/backend/routes/training.py @@ -113,6 +113,7 @@ async def start_training( # Validate dataset paths if provided if request.local_datasets: validated_datasets = [] + missing_datasets = [] # Get the backend directory (where this file is located) backend_dir = Path(__file__).parent.parent assets_datasets_dir = backend_dir / "assets" / "datasets" @@ -133,12 +134,20 @@ async def start_training( dataset_file = candidate if not dataset_file.exists(): - logger.warning( - f"Dataset file not found: {dataset_path} (resolved: {dataset_file})" + missing_datasets.append( + f"{dataset_path} (resolved: {dataset_file})" ) - else: - logger.info(f"Found dataset file: {dataset_file}") + continue + + logger.info(f"Found dataset file: {dataset_file}") validated_datasets.append(str(dataset_file)) + + if missing_datasets: + missing_detail = "; ".join(missing_datasets[:3]) + raise HTTPException( + status_code=400, + detail=f"Local dataset not found: {missing_detail}", + ) request.local_datasets = validated_datasets # Convert request to kwargs for backend diff --git a/studio/frontend/bun.lock b/studio/frontend/bun.lock index f200156928..e096f34cd7 100644 --- a/studio/frontend/bun.lock +++ b/studio/frontend/bun.lock @@ -18,7 +18,6 @@ "@hugeicons/react": "^1.1.5", "@huggingface/hub": "^2.9.0", "@langchain/core": "^1.1.27", - "@langchain/textsplitters": "^1.0.1", "@radix-ui/react-checkbox": "^1.3.3", "@radix-ui/react-label": "^2.1.8", "@radix-ui/react-select": "^2.2.6", @@ -389,8 +388,6 @@ "@langchain/core": ["@langchain/core@1.1.28", "", { "dependencies": { "@cfworker/json-schema": "^4.0.2", "ansi-styles": "^5.0.0", "camelcase": "6", "decamelize": "1.2.0", "js-tiktoken": "^1.0.12", "langsmith": ">=0.5.0 <1.0.0", "mustache": "^4.2.0", "p-queue": "^6.6.2", "uuid": "^10.0.0", "zod": "^3.25.76 || ^4" } }, "sha512-6FAGdezEp8zHY92LtnsAiv54KaG41nBdsuukk+R+1484edV20cVOyIc36ANuGKPx0pmYFCBWhCUdO0jxB/zn2Q=="], - "@langchain/textsplitters": ["@langchain/textsplitters@1.0.1", "", { "dependencies": { "js-tiktoken": "^1.0.12" }, "peerDependencies": { "@langchain/core": "^1.0.0" } }, "sha512-rheJlB01iVtrOUzttscutRgLybPH9qR79EyzBEbf1u97ljWyuxQfCwIWK+SjoQTM9O8M7GGLLRBSYE26Jmcoww=="], - "@mermaid-js/parser": ["@mermaid-js/parser@1.0.0", "", { "dependencies": { "langium": "^4.0.0" } }, "sha512-vvK0Hi/VWndxoh03Mmz6wa1KDriSPjS2XMZL/1l19HFwygiObEEoEwSDxOqyLzzAI6J2PU3261JjTMTO7x+BPw=="], "@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.27.1", "", { "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-sr6GbP+4edBwFndLbM60gf07z0FQ79gaExpnsjMGePXqFcSSb7t6iscpjk9DhFhwd+mTEQrzNafGP8/iGGFYaA=="], diff --git a/studio/frontend/index.html b/studio/frontend/index.html index bf6610cdff..c55ae77f62 100644 --- a/studio/frontend/index.html +++ b/studio/frontend/index.html @@ -2,9 +2,9 @@ - + - vite-app + Unsloth Studio
diff --git a/studio/frontend/package.json b/studio/frontend/package.json index e24843a301..9a607f1c1f 100644 --- a/studio/frontend/package.json +++ b/studio/frontend/package.json @@ -26,7 +26,6 @@ "@hugeicons/react": "^1.1.5", "@huggingface/hub": "^2.9.0", "@langchain/core": "^1.1.27", - "@langchain/textsplitters": "^1.0.1", "@radix-ui/react-checkbox": "^1.3.3", "@radix-ui/react-label": "^2.1.8", "@radix-ui/react-select": "^2.2.6", diff --git a/studio/frontend/public/favicon.png b/studio/frontend/public/favicon.png new file mode 100644 index 0000000000..86fbdf1fbc Binary files /dev/null and b/studio/frontend/public/favicon.png differ diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts index 3ac20221bc..f7a47c6a19 100644 --- a/studio/frontend/src/features/chat/api/chat-api.ts +++ b/studio/frontend/src/features/chat/api/chat-api.ts @@ -10,6 +10,7 @@ import type { OpenAIChatChunk, OpenAIChatCompletionsRequest, UnloadModelRequest, + ValidateModelResponse, } from "../types/api"; function parseErrorText(status: number, body: unknown): string { @@ -67,6 +68,21 @@ export async function loadModel( return parseJsonOrThrow(response); } +export async function validateModel( + payload: LoadModelRequest, +): Promise { + const response = await authFetch("/api/inference/validate", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + model_path: payload.model_path, + hf_token: payload.hf_token, + gguf_variant: payload.gguf_variant ?? null, + }), + }); + return parseJsonOrThrow(response); +} + export async function unloadModel(payload: UnloadModelRequest): Promise { const response = await authFetch("/api/inference/unload", { method: "POST", diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index 27fc8f99e6..303030372d 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -6,6 +6,7 @@ import { listModels, loadModel, unloadModel, + validateModel, } from "../api/chat-api"; import { useChatRuntimeStore } from "../stores/chat-runtime-store"; import type { LoadModelResponse } from "../types/api"; @@ -192,6 +193,17 @@ export function useChatModelRuntime() { const displayName = model?.name || lora?.name || modelId; const currentCheckpoint = useChatRuntimeStore.getState().params.checkpoint; + const previousCheckpoint = currentCheckpoint; + const previousVariant = + useChatRuntimeStore.getState().activeGgufVariant ?? null; + const previousModel = previousCheckpoint + ? models.find((entry) => entry.id === previousCheckpoint) + : undefined; + const previousLora = previousCheckpoint + ? loras.find((entry) => entry.id === previousCheckpoint) + : undefined; + const previousIsLora = + previousModel?.isLora ?? (previousLora ? true : false); const loadingDescription = [ currentCheckpoint ? "Unloading previous model first." : null, extraLoadingDescription ?? null, @@ -204,26 +216,61 @@ export function useChatModelRuntime() { setLoadingModel({ id: modelId, displayName }); try { async function performLoad(): Promise { + let previousWasUnloaded = false; const currentCheckpoint = useChatRuntimeStore.getState().params.checkpoint; - if (currentCheckpoint) { - await unloadModel({ model_path: currentCheckpoint }); + try { + // Lightweight pre-flight validation: avoid unloading a working model + // if the new identifier is clearly invalid (e.g. bad HF id / path). + await validateModel({ + model_path: modelId, + hf_token: null, + max_seq_length: DEFAULT_MODEL_MAX_SEQ_LENGTH, + load_in_4bit: true, + is_lora: isLora, + gguf_variant: ggufVariant ?? null, + }); + + if (currentCheckpoint) { + await unloadModel({ model_path: currentCheckpoint }); + previousWasUnloaded = true; + } + + const paramsBeforeLoad = useChatRuntimeStore.getState().params; + const loadResponse = await loadModel({ + model_path: modelId, + hf_token: null, + max_seq_length: DEFAULT_MODEL_MAX_SEQ_LENGTH, + load_in_4bit: true, + is_lora: isLora, + gguf_variant: ggufVariant ?? null, + trust_remote_code: paramsBeforeLoad.trustRemoteCode ?? false, + }); + + const currentParams = useChatRuntimeStore.getState().params; + setParams( + mergeRecommendedInference(currentParams, loadResponse, modelId), + ); + await refresh(); + } catch (error) { + // If we unloaded a previous model and the new load failed, attempt a rollback. + if (previousWasUnloaded && previousCheckpoint) { + try { + await loadModel({ + model_path: previousCheckpoint, + hf_token: null, + max_seq_length: DEFAULT_MODEL_MAX_SEQ_LENGTH, + load_in_4bit: true, + is_lora: previousIsLora, + gguf_variant: previousVariant, + }); + await refresh(); + } catch { + // If rollback also fails, surface the original error. + } + } + throw error; } - - const currentParams = useChatRuntimeStore.getState().params; - const loadResponse = await loadModel({ - model_path: modelId, - hf_token: null, - max_seq_length: DEFAULT_MODEL_MAX_SEQ_LENGTH, - load_in_4bit: true, - is_lora: isLora, - gguf_variant: ggufVariant ?? null, - trust_remote_code: currentParams.trustRemoteCode ?? false, - }); - - const paramsAfterLoad = useChatRuntimeStore.getState().params; - setParams(mergeRecommendedInference(paramsAfterLoad, loadResponse, modelId)); - await refresh(); } const loadPromise = performLoad().finally(() => { diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index 08bdc20435..602574f00a 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -38,6 +38,16 @@ export interface LoadModelRequest { trust_remote_code?: boolean; } +export interface ValidateModelResponse { + valid: boolean; + message: string; + identifier?: string | null; + display_name?: string | null; + is_gguf?: boolean; + is_lora?: boolean; + is_vision?: boolean; +} + export interface GgufVariantDetail { filename: string; quant: string; diff --git a/studio/frontend/src/features/data-recipes/learning-recipes/index.ts b/studio/frontend/src/features/data-recipes/learning-recipes/index.ts index 6c898e8228..98e20a954a 100644 --- a/studio/frontend/src/features/data-recipes/learning-recipes/index.ts +++ b/studio/frontend/src/features/data-recipes/learning-recipes/index.ts @@ -12,7 +12,10 @@ const instructionFromAnswerUrl = new URL( ).href; const textToPythonUrl = new URL("./text-to-python.json", import.meta.url).href; const textToSqlUrl = new URL("./text-to-sql.json", import.meta.url).href; -const conversationUrl = new URL("./conversation.json", import.meta.url).href; +const ocrDocumentExtractionUrl = new URL( + "./ocr-document-extraction.json", + import.meta.url, +).href; function isRecord(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); @@ -125,10 +128,10 @@ export const LEARNING_RECIPES: LearningRecipeDef[] = [ loadPayload: () => loadPayloadFromUrl(textToSqlUrl), }, { - id: "conversation", - title: "Multi-Turn Chat", + id: "ocr-document-extraction", + title: "OCR Document Extraction", description: - "Generate realistic user-assistant conversations with structured message output.", - loadPayload: () => loadPayloadFromUrl(conversationUrl), + "Use image context to generate OCR-style document extraction output.", + loadPayload: () => loadPayloadFromUrl(ocrDocumentExtractionUrl), }, ]; diff --git a/studio/frontend/src/features/data-recipes/learning-recipes/instruction-from-answer.json b/studio/frontend/src/features/data-recipes/learning-recipes/instruction-from-answer.json index 845fdc4f17..51af88b268 100644 --- a/studio/frontend/src/features/data-recipes/learning-recipes/instruction-from-answer.json +++ b/studio/frontend/src/features/data-recipes/learning-recipes/instruction-from-answer.json @@ -3,7 +3,7 @@ "model_providers": [ { "name": "openai_provider", - "endpoint": "https://openrouter.ai/api/v1", + "endpoint": "", "provider_type": "openai", "extra_headers": {}, "extra_body": {} @@ -13,19 +13,18 @@ "model_configs": [ { "alias": "ministral", - "model": "mistralai/ministral-8b-2512", + "model": "", "provider": "openai_provider", "inference_parameters": { "temperature": 0.7, - "max_tokens": 1024 + "max_tokens": 2048 } } ], "seed_config": { "source": { "seed_type": "hf", - "path": "datasets/unsloth/alpaca-cleaned/**/*.json", - "token": null, + "path": "unsloth/alpaca-cleaned", "endpoint": "https://huggingface.co" }, "sampling_strategy": "ordered", @@ -42,10 +41,20 @@ "drop": false, "model_alias": "ministral", "prompt": "Based on this target answer:\n{{ output }}\n\nWrite one high-quality plain text short and brief user instruction that this answer would satisfy.\nReturn only the instruction.", - "with_trace": "none" + "with_trace": "none", + "extract_reasoning_content": false } ], - "processors": [] + "processors": [ + { + "processor_type": "drop_columns", + "name": "drop_seed_columns", + "column_names": [ + "input", + "instruction" + ] + } + ] }, "run": { "rows": 5, @@ -63,7 +72,7 @@ "width": 400, "node_type": "markdown_note", "name": "note_1", - "markdown": "#### Hugginface seed block\nThis recipe uses a ** HuggingFace dataset ** as seed data.\nYou provide dataset identity, load columns, then generate new fields from seed columns.\n\n##### Setup:\n\n1. Paste dataset id as `org/repo` (example: `unsloth/alpaca-cleaned`)\n2. Add token only if dataset is gated/private\n3. Load columns + preview rows so variables are available in prompts\n\n##### Why this matters:\n- Seed columns can drive generation quality\n- You can reference seed values directly in prompts (for example `{{ output }}`)", + "markdown": "#### Hugging Face seed block\nThis recipe uses a ** HuggingFace dataset ** as seed data.\nYou select a Hugging Face dataset, load columns, then generate new fields from seed columns. Each column in the Hugging Face dataset becomes a valid variable that you can reference eg. `{{ topic }}`\n\n##### Setup:\n\n1. Search for a dataset and select one in the dropdown (example: `unsloth/alpaca-cleaned`)\n2. Add token only if dataset is gated/private\n3. Load columns + preview rows so variables are available in prompts\n\n##### Why this matters:\n- Seed columns can drive generation quality\n- You can reference seed values directly in prompts (for example `{{ output }}`)", "note_color": "#DCFCE7", "note_opacity": "35" }, @@ -74,7 +83,7 @@ "width": 400, "node_type": "markdown_note", "name": "note_2", - "markdown": "##### Drop columns behavior:\n\n- You can mark specific seed columns to **drop from final output**\n- Those columns are still used during generation\n- They are removed only from exported final dataset\n\n##### Example:\n- Keep `generated_instruction` from llm-text block\n- Drop original `instruction`, `input`, `output` from the hugginface dataset from final artifact\n- Result: clean training output while still using source columns as generation context\n", + "markdown": "##### Drop columns behavior:\n\n- You can mark specific seed columns to **drop from final output**\n- Those columns are still used during generation\n- They are removed only from exported final dataset\n\n##### Example:\n- Keep `generated_instruction` from llm-text block\n- Drop original `instruction`, `input`, `output` from the hugginface dataset from final dataset\n- Result: clean training output while still using source columns as generation context\n", "note_color": "#DCFCE7", "note_opacity": "35" }, @@ -128,64 +137,9 @@ ], "layout_direction": "LR", "seed_source_type": "hf", - "seed_columns": [ - "instruction", - "input", - "output" - ], + "seed_columns": [], "seed_drop_columns": [], - "seed_preview_rows": [ - { - "instruction": "Give three tips for staying healthy.", - "input": "", - "output": "1. Eat a balanced and nutritious diet: Make sure your meals are inclusive of a variety of fruits and vegetables, lean protein, whole grains, and healthy fats. This helps to provide your body with the essential nutrients to function at its best and can help prevent chronic diseases.\n\n2. Engage in regular physical activity: Exercise is crucial for maintaining strong bones, muscles, and cardiovascular health. Aim for at least 150 minutes of moderate aerobic exercise or 75 minutes of vigorous exercise each week.\n\n3. Get enough sleep: Getting enough quality sleep is crucial for physical and mental well-being. It helps to regulate mood, improve cognitive function, and supports healthy growth and immune function. Aim for 7-9 hours of sleep each night." - }, - { - "instruction": "What are the three primary colors?", - "input": "", - "output": "The three primary colors are red, blue, and yellow. These colors are called primary because they cannot be created by mixing other colors and all other colors can be made by combining them in various proportions. In the additive color system, used for light, the primary colors are red, green, and blue (RGB)." - }, - { - "instruction": "Describe the structure of an atom.", - "input": "", - "output": "An atom is the basic building block of all matter and is made up of three types of particles: protons, neutrons, and electrons. The structure of an atom can be described as a nucleus at the center surrounded by a cloud of electrons.\n\nThe nucleus of an atom is made up of protons and neutrons. Protons are positively charged particles and neutrons are neutral particles with no charge. Both of these particles are located in the nucleus of the atom, which is at the center of the atom and contains most of the atom's mass.\n\nSurrounding the nucleus of the atom is a cloud of electrons. Electrons are negatively charged particles that are in constant motion around the nucleus. The electron cloud is divided into shells or orbitals, and each shell can hold a certain number of electrons. The number of electrons in the outermost shell, called the valence shell, determines the chemical properties of the atom. \n\nIn a neutral atom, the number of protons in the nucleus is equal to the number of electrons in the electron cloud, so the positive and negative charges balance out and the atom has no overall charge. The number of protons, also called the atomic number, determines what element the atom is." - }, - { - "instruction": "How can we reduce air pollution?", - "input": "", - "output": "There are several ways to reduce air pollution, including:\n\n1. Reduce energy consumption: By conserving energy, we reduce the amount of pollution emitted from power plants. Some ways to achieve this include: using energy-efficient appliances and lighting, insulating homes, and shutting off lights and electronics when not in use.\n\n2. Use public transportation or carpool: Cars and trucks are major sources of pollution. By using public transportation, carpooling or biking, you can reduce the number of cars on the road, which in turn reduces emissions.\n\n3. Be responsible with waste: When waste is burned or decomposed, they can release harmful gases into the air. Dispose of hazardous waste correctly and recycle materials like plastics, paper, and glass when possible.\n\n4. Support clean energy sources: Renewable energy sources like solar and wind power generate little or no pollution. Support clean energy companies and projects, and consider installing renewable energy systems in your home.\n\n5. Drive efficiently: If you need to drive, make sure your vehicle is well maintained and drive efficiently by avoiding rapid acceleration and braking, and reducing idling.\n\n6. Be mindful of product choices: Some products and practices release more pollution than others. Choose products with eco-friendly packaging and try to minimize your use of aerosol sprays, which can release harmful chemicals into the air." - }, - { - "instruction": "Pretend you are a project manager of a construction company. Describe a time when you had to make a difficult decision.", - "input": "", - "output": "I had to make a difficult decision when I was working as a project manager at a construction company. I was in charge of a project that needed to be completed by a certain date in order to meet the client’s expectations. However, due to unexpected delays, we were not able to meet the deadline and so I had to make a difficult decision. I decided to extend the deadline, but I had to stretch the team’s resources even further and increase the budget. Although it was a risky decision, I ultimately decided to go ahead with it to ensure that the project was completed on time and that the client’s expectations were met. The project was eventually successfully completed and this was seen as a testament to my leadership and decision-making abilities." - }, - { - "instruction": "Write a concise summary of the following:\n\"Commodore 64 (commonly known as the C64 or CBM 64) was manufactured by Commodore Business Machine (CBM) in August 1982 with a starting price of $595. It was an 8-bit home computer with remarkable market success. Between 1983-1986, C64 sales amounted to about 17 million units sold, making them the best-selling single personal computer model of all time in 1983-1986. \n\nAdditionally, the Commodore 64 dominated the market with between 30% and 40% share and 2 million units sold per year, outselling the IBM PC clones, Apple Computers, and Atari computers. Adding to their success, Sam Tramiel (former Atari president), during an interview in 1989, said they were building 400,000 C64s a month for a couple of years. \"", - "input": "", - "output": "The Commodore 64 was a highly successful 8-bit home computer manufactured by Commodore Business Machine (CBM) in 1982, with sales amounting to approximately 17 million units sold between 1983-1986. It dominated the market with between 30% and 40% share and outsold its competitors, including IBM PC clones, Apple Computers, and Atari computers. At its peak, CBM was building 400,000 C64s a month for a couple of years." - }, - { - "instruction": "Explain why the following fraction is equivalent to 1/4", - "input": "4/16", - "output": "The fraction 4/16 is equivalent to 1/4 because both fractions represent the same value. A fraction can be simplified by dividing both the numerator and the denominator by a common factor. In this case, 4 is a common factor of both the numerator and the denominator of 4/16. When we divide both by 4, we get 4/4 = 1 and 16/4 = 4, so the simplified fraction is 1/4. Alternatively, we can think of this in terms of multiplication. For example, if we multiply the numerator and denominator of the fraction 1/4 by 4, we get (1x4)/(4x4), or 4/16. Since both fractions can be derived from the other through multiplication or division by the same number, they represent the same value and are equivalent." - }, - { - "instruction": "Write a short story in third person narration about a protagonist who has to make an important career decision.", - "input": "", - "output": "Sophie sat at her desk, staring blankly at the computer screen. Her mind was racing as she weighed the options in front of her. She had been offered a promotion to a senior management position within her company, but it meant moving across the country and leaving her friends and family behind. At the same time, her dream job at a rival company had finally opened up, and she had been invited for an interview. \n\nThe thought of leaving her comfortable life behind, to take a chance on something new and uncertain, made Sophie feel anxious, but also excited. She had always been one to take risks, and it had paid off in the past. But now, with such a big decision to make, Sophie couldn't help but feel overwhelmed.\n\nAfter much deliberation, Sophie made her choice. She packed her bags, said goodbye to her loved ones, and moved across the country for the new job. It was a difficult first few months, adjusting to a new city and a new company culture. But as time went by, Sophie realized that she had made the best decision for her career. The job was challenging, but also rewarding, and she was surrounded by passionate, driven colleagues who inspired her to grow and develop.\n\nIn the end, Sophie's bold move paid off. She was thriving in her new role, and had made new friends and connections in her new city. Looking back, Sophie was grateful for the opportunity to take a risk and make a change, and was confident that she had made the right decision for her career." - }, - { - "instruction": "Evaluate this sentence for spelling and grammar mistakes", - "input": "He finnished his meal and left the resturant", - "output": "There are two spelling errors in the sentence. The corrected sentence should be: \"He finished his meal and left the restaurant.\"" - }, - { - "instruction": "How did Julius Caesar die?", - "input": "", - "output": "Julius Caesar, the Roman Military general, and statesman was assassinated on March 15, 44 BC. He was stabbed to death at a meeting of the Senate in the Theatre of Pompey by a group of senators who called themselves the Liberatores. Led by Gaius Cassius Longinus, and Marcus Junius Brutus, they believed that Caesar's dictatorship was subverting the Roman Republic and sought to restore the traditional republican government. The exact number of assailants is not known, but it is believed that a group of as many as 60 conspirators participated in the assassination, and each of them took turns in stabbing Caesar. The day of his death is still known as the Ides of March." - } - ], + "seed_preview_rows": [], "local_file_name": "", "unstructured_file_name": "", "unstructured_chunk_size": "1200", diff --git a/studio/frontend/src/features/data-recipes/learning-recipes/ocr-document-extraction.json b/studio/frontend/src/features/data-recipes/learning-recipes/ocr-document-extraction.json new file mode 100644 index 0000000000..84ba03a0c2 --- /dev/null +++ b/studio/frontend/src/features/data-recipes/learning-recipes/ocr-document-extraction.json @@ -0,0 +1,148 @@ +{ + "recipe": { + "model_providers": [ + { + "name": "provider_1", + "endpoint": "https://openrouter.ai/api/v1", + "provider_type": "openai", + "extra_headers": {}, + "extra_body": {} + } + ], + "mcp_providers": [], + "model_configs": [ + { + "alias": "provider_column", + "model": "google/gemini-2.0-flash-001", + "provider": "provider_1", + "inference_parameters": { + "temperature": 0.2, + "max_tokens": 4096 + } + } + ], + "seed_config": { + "source": { + "seed_type": "hf", + "path": "datasets/ylecun/mnist/mnist/**/*.parquet" + }, + "sampling_strategy": "ordered", + "selection_strategy": null + }, + "tool_configs": [], + "columns": [ + { + "column_type": "llm-text", + "name": "ocr_text", + "drop": false, + "model_alias": "provider_column", + "prompt": "Transcribe all text from this document image.", + "multi_modal_context": [ + { + "modality": "image", + "column_name": "image" + } + ] + } + ], + "processors": [] + }, + "run": { + "rows": 5, + "preview": true, + "output_formats": ["jsonl"] + }, + "ui": { + "nodes": [ + { + "id": "note_1", + "x": -180, + "y": 43, + "width": 400, + "node_type": "markdown_note", + "name": "note_1", + "markdown": "This recipe uses **Gemini 2.0 Flash** via OpenRouter to transcribe document images into clean text.\n\nThe Seed block is prefilled with `ylecun/mnist` so you can run immediately. You can swap to any Hugging Face dataset that includes an `image` column.\n\nOutput: `ocr_text` column with the raw transcribed text per image.", + "note_color": "#DCFCE7", + "note_opacity": "35" + }, + { + "id": "note_2", + "x": 283, + "y": -333, + "width": 400, + "node_type": "markdown_note", + "name": "note_2", + "markdown": "##### Setup\n\nAdd your OpenRouter API key to the **Model Provider** block — same as every other recipe.\n\nGemini 2.0 Flash is well-suited for OCR: fast, cheap, and strong on tables, receipts, forms, and multi-column layouts.\n\nWant a purpose-built OCR model? Swap the endpoint to a local vLLM server running `lightonai/LightOnOCR-2-1B` for maximum throughput.", + "note_color": "#DCFCE7", + "note_opacity": "35" + }, + { + "id": "note_3", + "x": 303, + "y": 299, + "width": 400, + "node_type": "markdown_note", + "name": "note_3", + "markdown": "##### Seed: HF dataset with image column\n\nThis template starts with `ylecun/mnist` so first run works without seed setup.\n\nTo use your own data: open Seed → keep **HF dataset** selected → choose a dataset that contains an `image` column → click **Load**.\n\nThen open the LLM Text block and set **Image Context** to the `image` column so each row image is sent with the prompt.\n\nTip: datasets with embedded image columns are more reliable than URL-only image fields.", + "note_color": "#DCFCE7", + "note_opacity": "35" + }, + { + "id": "seed", + "x": 295, + "y": 108, + "width": 400 + }, + { + "id": "provider_1", + "x": 960, + "y": -465, + "width": 400 + }, + { + "id": "provider_column", + "x": 959, + "y": -180, + "width": 400 + }, + { + "id": "ocr_text", + "x": 960, + "y": 108, + "width": 400 + } + ], + "edges": [ + { + "from": "seed", + "to": "ocr_text", + "type": "canvas", + "source_handle": "data-out", + "target_handle": "data-in" + }, + { + "from": "provider_1", + "to": "provider_column", + "type": "semantic", + "source_handle": "semantic-out-bottom", + "target_handle": "semantic-in-top" + }, + { + "from": "provider_column", + "to": "ocr_text", + "type": "semantic", + "source_handle": "semantic-out-bottom", + "target_handle": "data-in-top" + } + ], + "layout_direction": "LR", + "seed_source_type": "hf", + "seed_columns": [], + "seed_drop_columns": [], + "seed_preview_rows": [], + "local_file_name": "", + "unstructured_file_name": "", + "unstructured_chunk_size": "900", + "unstructured_chunk_overlap": "150" + } +} diff --git a/studio/frontend/src/features/data-recipes/learning-recipes/pdf-grounded-qa.json b/studio/frontend/src/features/data-recipes/learning-recipes/pdf-grounded-qa.json index 3f2fb95743..bd999b9779 100644 --- a/studio/frontend/src/features/data-recipes/learning-recipes/pdf-grounded-qa.json +++ b/studio/frontend/src/features/data-recipes/learning-recipes/pdf-grounded-qa.json @@ -3,7 +3,7 @@ "model_providers": [ { "name": "provider_1", - "endpoint": "https://openrouter.ai/api/v1", + "endpoint": "", "provider_type": "openai", "extra_headers": {}, "extra_body": {} @@ -13,18 +13,19 @@ "model_configs": [ { "alias": "provider_column", - "model": "mistralai/ministral-8b-2512", + "model": "", "provider": "provider_1", "inference_parameters": { - "temperature": 0.7, - "max_tokens": 256 + "temperature": 0.7 } } ], "seed_config": { "source": { - "seed_type": "local", - "path": "/home/wasim/.cache/unsloth/data-recipe/seed-uploads/dc8ef7c169ed43399b7d37164495150f_50 page sample PDF.indd.jsonl" + "seed_type": "unstructured", + "path": "", + "chunk_size": 1200, + "chunk_overlap": 200 }, "sampling_strategy": "ordered", "selection_strategy": null @@ -37,6 +38,8 @@ "drop": false, "model_alias": "provider_column", "prompt": "Given ONLY this chunk: {{ chunk_text }} generate one answerable question, answer, and exact supporting quote from chunk. If not answerable, skip.", + "with_trace": "none", + "extract_reasoning_content": false, "output_format": { "type": "object", "additionalProperties": false, @@ -72,8 +75,8 @@ "nodes": [ { "id": "note_1", - "x": -180.01113025994076, - "y": 43.27382247773167, + "x": 474.6120044693708, + "y": 1229.5810476890458, "width": 400, "node_type": "markdown_note", "name": "note_1", @@ -83,8 +86,8 @@ }, { "id": "note_2", - "x": 283.8131688769869, - "y": -333.10847089567505, + "x": 26.758540311329455, + "y": 963.3465578835235, "width": 400, "node_type": "markdown_note", "name": "note_2", @@ -94,8 +97,8 @@ }, { "id": "note_3", - "x": 303.52241671566657, - "y": 299.62272507131615, + "x": 473.62551435180245, + "y": 741.5352258256931, "width": 400, "node_type": "markdown_note", "name": "note_3", @@ -105,37 +108,30 @@ }, { "id": "seed", - "x": 295.56977201312833, - "y": 108.19964868337735, + "x": 484.36210245413577, + "y": 1059.99180558796, "width": 400 }, { "id": "provider_1", - "x": 960.0115722892822, - "y": -465.06410256410254, + "x": 960, + "y": 622, "width": 400 }, { "id": "provider_column", - "x": 959.90231990232, - "y": -180.56654456654456, + "x": 960, + "y": 816, "width": 400 }, { "id": "llm_structured_1", "x": 960, - "y": 108.25, + "y": 1077, "width": 400 } ], "edges": [ - { - "from": "seed", - "to": "llm_structured_1", - "type": "canvas", - "source_handle": "data-out", - "target_handle": "data-in" - }, { "from": "provider_1", "to": "provider_column", @@ -149,49 +145,23 @@ "type": "semantic", "source_handle": "semantic-out-bottom", "target_handle": "data-in-top" + }, + { + "from": "llm_structured_1", + "to": "seed", + "type": "canvas", + "source_handle": "data-out-left", + "target_handle": "data-in-right" } ], "layout_direction": "LR", "seed_source_type": "unstructured", - "seed_columns": [ - "chunk_text" - ], + "seed_columns": [], "seed_drop_columns": [], - "seed_preview_rows": [ - { - "chunk_text": "[Citation Needed] The Best of Wikipedia’s Worst Writing Conor Lastowka and Josh Fruhlinger Boring Legal Fine Print Each entry in this book contains material from Wikipedia, although the text we use may not represent the current version of any article. The URL at the bottom of each page will direct you to the source Wikipedia article; use the article’s History tab to find a list of contributors. All material in this book that is taken from Wikipedia is licensed under the Creative Commons-Attribution Share Alike 3.0 license. Here’s a quick human-readable summary of your rights to use this content: You are free: to Share—to copy, distribute and transmit the work, and to Remix—to adapt the work Under the following conditions: Attribution—You must attribute the work in the manner specified by the author or licensor (but not in any way that suggests that they endorse you or your use of the work.) Share Alike—If you alter, transform, or build upon this work, you may distribute the resulting work only under the same, similar or a compatible license. With the understanding that: Waiver—Any of the above conditions can be waived if you get permission from the copyright holder. Other Rights—In" - }, - { - "chunk_text": "only under the same, similar or a compatible license. With the understanding that: Waiver—Any of the above conditions can be waived if you get permission from the copyright holder. Other Rights—In no way are any of the following rights affected by the license: your fair dealing or fair use rights; the author’s moral rights; and rights other persons may have either in the work itself or in how the work is used, such as publicity or privacy rights. Notice—For any reuse or distribution, you must make clear to others the license terms of this work. The best way to do this is with a link to: http://creativecommons.org/licenses/by-sa/3.0/ Italicized material beneath each Wikipedia entry is © 2011 Conor Lastowka and Josh Fruhlinger. Copy-edited by Lauren Lastowka Cover design by Jaime Robinson ISBN # 978-1466346987 This book is dedicated to every person who wrote an entry that appears in it. May your citations always be needed. 6 Introduction Wikipedia. Whether you’ve used it to settle an argument, plagiarized a history report from it, or simply replaced the entire text of the biography of a respected humanitarian with the single word “dogballs,” it’s an inescapable part of the Internet" - }, - { - "chunk_text": "plagiarized a history report from it, or simply replaced the entire text of the biography of a respected humanitarian with the single word “dogballs,” it’s an inescapable part of the Internet experience. Since its launch in 2001, it has rapidly risen to become the seventh most popular website, with over 365 million readers (Source: Wikipedia). If you’re like us, when you want to know the name of the kangaroo on Shirt Tales or just want to confirm that Mother Teresa was a dogballs who helped the farts (Source: Wikipedia), The Encyclopedia That Anyone Can Edit will probably be the first place you check. But here’s the thing about letting anybody edit your encyclopedia: it means that anybody can edit your encyclopedia. And while in theory this means that one day Stephen Hawking might decide to weigh in on the entry for string theory, in reality it means that somebody who deeply cares about pro wrestling is going to call someone else a Nazi when they revert his edits about Wrestlemania XI on Razor Ramon’s page. And so we arrive at a cosmic intersection, where an obscure topic of dubious relevance is written about by the type of weirdo who logs on to Wikipedia to write about obscure" - }, - { - "chunk_text": "XI on Razor Ramon’s page. And so we arrive at a cosmic intersection, where an obscure topic of dubious relevance is written about by the type of weirdo who logs on to Wikipedia to write about obscure topics of dubious relevance. Were these authors re-watching their video of Wrestlemania XI instead of completing basic 8th grade English assignments? It’s very likely. Does this 7 stop them from attempting to emulate the academic tone of the great encyclopedias of the past as they describe a large mammalian species from the Star Wars universe that shares a common ancestor with the Wookies? It does not. The result? Some really terrible Wikipedia writing. For the past two years, we have collected this writing on our blog, [Citation Needed]. Fascinated and delighted by the brilliantly bad writing we encountered in our Wikipedia browsing, we set out to curate The Best of Wikipedia’s Worst Writing. Starting the blog was a no-brainer; our only concern was whether, after a few months of our daily mining, the well of awful Wikipedia writing would eventually run dry. By the time you read this, we will have published our thousandth entry. We started a podcast. Instead of drying up, the ocean of" - }, - { - "chunk_text": "mining, the well of awful Wikipedia writing would eventually run dry. By the time you read this, we will have published our thousandth entry. We started a podcast. Instead of drying up, the ocean of ineptitude has proven far more vast than we ever could have imagined. Through our own browsing, and with the help of a dedicated group of readers who are exploring the topics they submit for God knows what reason, we’ve continually lowered and re-lowered the bar for bad Wikipedia writing. Now, let’s get one thing straight: we love each and every entry written in this book. If you are one of the authors who have chosen to use your valuable time on this planet to write straight-faced exegeses on the subject of forgotten action figures from the seventies, we hope you don’t take offense. And if you do, we have an acceptable retort prepared for you: “You guys ran a blog about Wikipedia for two years, who the hell are you to talk?” Feel free to use it! Others may criticize us for not doing our part to help Wikipedia become “better” by revising these passages. Nothing that does not involve electrodes near our genitals would make us more miserable. In our opinion, many of the passages in this" - }, - { - "chunk_text": "to help Wikipedia become “better” by revising these passages. Nothing that does not involve electrodes near our genitals would make us more miserable. In our opinion, many of the passages in this book stand alone as works of art. Think of us as photographers preserving the memory of the great street art of the world before the joyless police come and whitewash over it. (Is that an official police responsibility? It seems beneath them. If it’s not, but they’re still forced to do it, that might explain the joylessness.) The point is, if you’re moved to correct these entries, we’re powerless to stop you. They’ve already given us joy, and we’re just happy to have encountered them. Enough introduction. Here are over two hundred of our favorite bad Wikipedia articles of all time. Comments in italics are ours. Everything else is a faithful reproduction of the way the entry stood at the moment we or our informants encountered it. We hope you will laugh, cry, maybe even learn something, and always remember to dogballs. —Conor Lastowka & Josh Fruhlinger citationneeded.tumblr.com 8 9 In barely one decade, Jimmy Wales has succeeded in establishing a worldwide network of knowledge. Wikipedia," - }, - { - "chunk_text": "remember to dogballs. —Conor Lastowka & Josh Fruhlinger citationneeded.tumblr.com 8 9 In barely one decade, Jimmy Wales has succeeded in establishing a worldwide network of knowledge. Wikipedia, his online encyclopaedia, accessible on the Internet for free, has become a symbol of a radical change in the media economy. Moreover, it revolutionized the access to knowledge as man’s most important resource and thus contributed to democratizing knowledge. The Gottlieb Duttweiler Institute, awarding the 2011 Gottlieb Duttweiler Prize to Wikipedia founder Jimmy Wales I saw the Beavis and Butt-Head episode that had Hogan’s “Real American” music on there. I don’t quite remembering it being critiqued by Beavis and Butt-Head. They sounded more like they liked the music and I don’t really remember any criticism of it (except for when it was going, when Butt-Head said “homework sucks”, but I’m not quite sure if he was referring to music or not). Wikipedia discussion page for Hulk Hogan 10 11 http://en.wikipedia.org/wiki/Polybius_(video_game) Want a Citation for this one? Please see the following 206 pages. The Roach story contained a number of inconsistencies: some of it seems to be directly" - }, - { - "chunk_text": "Want a Citation for this one? Please see the following 206 pages. The Roach story contained a number of inconsistencies: some of it seems to be directly sourced from Wikipedia- all in all, an entirely untrustworthy source. Polybius (video game) 12 http://en.wikipedia.org/wiki/General_Mills_monster-themed_breakfast_cereals You can imagine the marketing team having their first meeting after the cereal’s release. “We have good news and bad news. The good news is, your latest cereal is very, very popular. The bad news is, it’s not in any way due to the character you came up with, the box design you slaved over, the costly ad campaign, or the hours you put in coming up with free toy ideas. Gentlemen, you should probably sit down....” Franken Berry was very popular when first introduced possibly because the initial batches of the cereal used a dye that didn’t break down in the body, causing many children’s feces to be bright pink, a symptom sometimes referred to as “Frankenberry Stool.” General Mills monster-themed breakfast cereals 13 http://en.wikipedia.org/wiki/Skiffle Because if your encyclopedia can’t provide you with an unsourced claim that it admits is only one of several" - }, - { - "chunk_text": "General Mills monster-themed breakfast cereals 13 http://en.wikipedia.org/wiki/Skiffle Because if your encyclopedia can’t provide you with an unsourced claim that it admits is only one of several theories put forth about the subject, and then go on to inform you in the very same sentence that other unidentified parties disagree with that claim, then what the hell good is it? Skiffle is often said to have developed from New Orleans jazz, but this has been disputed. Skiffle 14 http://en.wikipedia.org/wiki/House_Party_%28film%29 “HP4: Coda or Mistake?” was by far the most contentious panel at HoPaCon 2009, with impassioned arguments echoing through the halls of the Kansas City International Airport Days Inn. In 2001, Immature (now going by IMx) starred in a direct-to-video sequel, House Party 4: Down to the Last Minute, which does not feature Kid or Play. The film is not considered a part of the House Party canon amongst fans. [citation needed] House Party (film) 15 http://en.wikipedia.org/wiki/Inglewood,_California So it would appear that, due to the presence of the anti- drug organization D.A.R.E. in Inglewood, there are in fact parts of Inglewood that are attempting to do good." - }, - { - "chunk_text": "So it would appear that, due to the presence of the anti- drug organization D.A.R.E. in Inglewood, there are in fact parts of Inglewood that are attempting to do good. Thus the claim that Inglewood is “always” up to no good can be assumed to be false, or at the very least a gross exaggeration. Also, many have speculated that the so-called “Doctor” Dre never actually received a PhD. D.A.R.E. America has its headquarters in Inglewood. Despite this, in the 1996 rap hit “California Love”, Dr. Dre remarks that Inglewood is “always up to no good”. Inglewood, California 16 http://en.wikipedia.org/wiki/Tyler_Perry People who feel that Tyler Perry’s 6’5” stature is severely diminished by his wearing a wig? The line in the sand has been drawn. Another comical aspect is provided by Perry’s 6’- 5” stature, which is in no way diminished by his wearing a wig. Tyler Perry 17 http://en.wikipedia.org/wiki/Bondage_bed Do not attempt to affix your bondage partner to this question mark using chains and shackles! It is purely metaphorical! It is possible to buy inflatable bondage beds; however, a question mark must remain over how effective they are. Bondage bed 18" - } - ], + "seed_preview_rows": [], "local_file_name": "", - "unstructured_file_name": "50 page sample PDF.indd.pdf", + "unstructured_file_name": "", "unstructured_chunk_size": "1200", "unstructured_chunk_overlap": "200" } -} +} \ No newline at end of file diff --git a/studio/frontend/src/features/data-recipes/learning-recipes/structured-outputs-jinja.json b/studio/frontend/src/features/data-recipes/learning-recipes/structured-outputs-jinja.json index f6f11bbc71..d7ccabaca3 100644 --- a/studio/frontend/src/features/data-recipes/learning-recipes/structured-outputs-jinja.json +++ b/studio/frontend/src/features/data-recipes/learning-recipes/structured-outputs-jinja.json @@ -3,7 +3,7 @@ "model_providers": [ { "name": "provider_column", - "endpoint": "https://openrouter.ai/api/v1", + "endpoint": "", "provider_type": "openai", "extra_headers": {}, "extra_body": {} @@ -13,11 +13,10 @@ "model_configs": [ { "alias": "ministral", - "model": "mistralai/ministral-8b-2512", + "model": "", "provider": "provider_column", "inference_parameters": { - "temperature": 0.7, - "max_tokens": 256 + "temperature": 0.7 } } ], @@ -76,6 +75,8 @@ "drop": false, "model_alias": "ministral", "prompt": "Create a realistic support ticket from {{ user_full_name }} using the {{ platform }} platform. Impact scope is {{ impact_scope }}.\n", + "with_trace": "none", + "extract_reasoning_content": false, "output_format": { "type": "object", "additionalProperties": false, @@ -129,6 +130,8 @@ "drop": false, "model_alias": "ministral", "prompt": "Write a concise support reply for ticket '{{ ticket.issue_title }}'. Category: {{ ticket.category }}. Priority: {{ ticket.priority }}. SLA target: {{ sla_target }}. {% if ticket.priority == 'P1' %}Tone must be urgent and action-first.{% else %}Tone must be calm and instructional.{% endif %}", + "with_trace": "none", + "extract_reasoning_content": false, "output_format": { "type": "object", "additionalProperties": false, @@ -168,8 +171,8 @@ "nodes": [ { "id": "note_1", - "x": 1084.767431711644, - "y": -293.4482850247655, + "x": 990.3973509933774, + "y": 1487.5768211920529, "width": 782, "node_type": "markdown_note", "name": "note_1", @@ -179,8 +182,8 @@ }, { "id": "note_2", - "x": 1944, - "y": 760.9999999999999, + "x": 3217.6543046357615, + "y": 2081.596026490066, "width": 400, "node_type": "markdown_note", "name": "note_2", @@ -190,8 +193,8 @@ }, { "id": "note_3", - "x": 2381.178207301403, - "y": 790.2196835690842, + "x": 2294.4516556291387, + "y": 2399.6099337748346, "width": 638, "node_type": "markdown_note", "name": "note_3", @@ -201,8 +204,8 @@ }, { "id": "note_4", - "x": 2405.796928768747, - "y": -362.26583299682716, + "x": 2544.684105960265, + "y": 1126.5490066225163, "width": 399, "node_type": "markdown_note", "name": "note_4", @@ -212,62 +215,62 @@ }, { "id": "provider_column", - "x": 1947.2039072039072, - "y": 32.08363858363858, + "x": 2542, + "y": 1696, "width": 400 }, { "id": "ministral", - "x": 1947.0573870573871, - "y": 271.94139194139194, + "x": 2542, + "y": 1890, "width": 400 }, { "id": "user", - "x": 0, - "y": 656.5, + "x": 191, + "y": 2423, "width": 400 }, { "id": "platform", - "x": 480, - "y": 656.5, + "x": 858.0384105960266, + "y": 2286.5, "width": 400 }, { "id": "impact_scope", - "x": 960, - "y": 656.5, + "x": 1342, + "y": 2286.5, "width": 400 }, { "id": "user_first_name", - "x": 1440, - "y": 895, + "x": 1822, + "y": 2505, "width": 400 }, { "id": "user_full_name", - "x": 1440, - "y": 269, + "x": 1822, + "y": 1959, "width": 400 }, { "id": "ticket", - "x": 1946.9108669108673, - "y": 657.2161172161173, + "x": 2302, + "y": 2286.5, "width": 400 }, { "id": "sla_target", - "x": 1440, - "y": 582, + "x": 1822, + "y": 2232, "width": 400 }, { "id": "agent_reply", - "x": 2384.5665445665445, - "y": 657.449938949939, + "x": 2782, + "y": 2151, "width": 400 } ], @@ -353,10 +356,10 @@ "from": "ministral", "to": "agent_reply", "type": "semantic", - "source_handle": "semantic-out", + "source_handle": "semantic-out-bottom", "target_handle": "data-in-top" } ], "layout_direction": "LR" } -} +} \ No newline at end of file diff --git a/studio/frontend/src/features/data-recipes/learning-recipes/text-to-python.json b/studio/frontend/src/features/data-recipes/learning-recipes/text-to-python.json index 1208e4a959..cd8118baa6 100644 --- a/studio/frontend/src/features/data-recipes/learning-recipes/text-to-python.json +++ b/studio/frontend/src/features/data-recipes/learning-recipes/text-to-python.json @@ -2,8 +2,8 @@ "recipe": { "model_providers": [ { - "name": "provider_1", - "endpoint": "https://openrouter.ai/api/v1", + "name": "openai-compatible", + "endpoint": "", "provider_type": "openai", "extra_headers": {}, "extra_body": {} @@ -12,12 +12,11 @@ "mcp_providers": [], "model_configs": [ { - "alias": "model_1", - "model": "mistralai/ministral-8b-2512", - "provider": "provider_1", + "alias": "coding-model", + "model": "", + "provider": "openai-compatible", "inference_parameters": { - "temperature": 0.7, - "max_tokens": 2048 + "temperature": 0.7 } } ], @@ -66,24 +65,29 @@ "column_type": "llm-text", "name": "instruction", "drop": false, - "model_alias": "model_1", + "model_alias": "coding-model", "prompt": "Write one clear Python coding instruction.\nDomain: {{ domain }}\nTask type: {{ task_type }}\n\nKeep it practical and specific.\nReturn only the instruction without any code.", - "with_trace": "none" + "with_trace": "none", + "extract_reasoning_content": false }, { "column_type": "llm-code", "name": "code_implementation", "drop": false, - "model_alias": "model_1", + "model_alias": "coding-model", "prompt": "Write Python code for:\n{{ instruction }}\n\nRequirements:\n- runnable script or function\n- include needed imports\n- short comments only where useful\n- no markdown fences", + "with_trace": "none", + "extract_reasoning_content": false, "code_lang": "python" }, { "column_type": "llm-judge", "name": "code_judge_result", "drop": false, - "model_alias": "model_1", + "model_alias": "coding-model", "prompt": "Evaluate generated Python code against the instruction.\n\nInstruction:\n{{ instruction }}\n\nCode:\n{{ code_implementation }}", + "with_trace": "none", + "extract_reasoning_content": false, "scores": [ { "name": "Correctness", @@ -109,52 +113,10 @@ }, "ui": { "nodes": [ - { - "id": "provider_1", - "x": 1032.6798211423347, - "y": -450.4885376732656, - "width": 400 - }, - { - "id": "model_1", - "x": 1538.0273166472973, - "y": -483.2003290046642, - "width": 400 - }, - { - "id": "domain", - "x": 0, - "y": 24, - "width": 400 - }, - { - "id": "task_type", - "x": 480, - "y": 24, - "width": 400 - }, - { - "id": "instruction", - "x": 958.8989453654599, - "y": -9.971266983459952, - "width": 400 - }, - { - "id": "code_implementation", - "x": 1538.788058529745, - "y": -45.56493974435071, - "width": 400 - }, - { - "id": "code_judge_result", - "x": 2040.9251520522098, - "y": -13.362336454344792, - "width": 400 - }, { "id": "note_1", - "x": 1482.1328175027095, - "y": 242.4370179053253, + "x": 1526, + "y": 1790.75, "width": 568, "node_type": "markdown_note", "name": "note_1", @@ -164,14 +126,56 @@ }, { "id": "note_2", - "x": 2513.2527820497985, - "y": -235.2544980991115, + "x": 2597.376821192053, + "y": 1233.2039735099338, "width": 471, "node_type": "markdown_note", "name": "note_2", "markdown": "The **LLM Judge** block evaluates generated outputs with rubric-style scores.\n\n##### Important:\n\n- A judge can have **one or many scores**\n- Each score has:\n - a name (for example: `Correctness`)\n - a description\n - options (value + meaning)\n\n##### Example multi-score setup:\n\n- Correctness\n- Readability\n- Efficiency\n\n##### Why use multiple scores:\n\n- You get richer quality signals than a single pass/fail\n- Easier filtering and weighting later in training data prep\n\n##### Practical pattern:\n\n1. Generate code with LLM Code\n2. Judge with 2-4 focused scores\n3. Keep high-quality rows based on score thresholds\n", "note_color": "#FEF3C7", "note_opacity": "35" + }, + { + "id": "openai-compatible", + "x": 1627.1046357615896, + "y": 921.0301324503313, + "width": 400 + }, + { + "id": "coding-model", + "x": 1627.1046357615894, + "y": 1138.910927152318, + "width": 400 + }, + { + "id": "domain", + "x": 84, + "y": 1600.5, + "width": 400 + }, + { + "id": "task_type", + "x": 648, + "y": 1600.5, + "width": 400 + }, + { + "id": "instruction", + "x": 1128, + "y": 1567, + "width": 400 + }, + { + "id": "code_implementation", + "x": 1627.1046357615894, + "y": 1531.4728476821192, + "width": 400 + }, + { + "id": "code_judge_result", + "x": 2124.617218543046, + "y": 1567.076490066225, + "width": 400 } ], "edges": [ @@ -190,11 +194,11 @@ "target_handle": "data-in" }, { - "from": "provider_1", - "to": "model_1", + "from": "openai-compatible", + "to": "coding-model", "type": "semantic", - "source_handle": "semantic-out", - "target_handle": "semantic-in" + "source_handle": "semantic-out-bottom", + "target_handle": "semantic-in-top" }, { "from": "instruction", @@ -204,14 +208,14 @@ "target_handle": "data-in" }, { - "from": "model_1", + "from": "coding-model", "to": "instruction", "type": "semantic", "source_handle": "semantic-out-bottom", "target_handle": "data-in-top" }, { - "from": "model_1", + "from": "coding-model", "to": "code_implementation", "type": "semantic", "source_handle": "semantic-out-bottom", @@ -225,7 +229,7 @@ "target_handle": "data-in" }, { - "from": "model_1", + "from": "coding-model", "to": "code_judge_result", "type": "semantic", "source_handle": "semantic-out-bottom", @@ -234,4 +238,4 @@ ], "layout_direction": "LR" } -} +} \ No newline at end of file diff --git a/studio/frontend/src/features/data-recipes/learning-recipes/text-to-sql.json b/studio/frontend/src/features/data-recipes/learning-recipes/text-to-sql.json index 60957bcf4a..d2d9abbd41 100644 --- a/studio/frontend/src/features/data-recipes/learning-recipes/text-to-sql.json +++ b/studio/frontend/src/features/data-recipes/learning-recipes/text-to-sql.json @@ -2,8 +2,8 @@ "recipe": { "model_providers": [ { - "name": "provider_1", - "endpoint": "https://openrouter.ai/api/v1", + "name": "vllm", + "endpoint": "", "provider_type": "openai", "extra_headers": {}, "extra_body": {} @@ -12,12 +12,11 @@ "mcp_providers": [], "model_configs": [ { - "alias": "model_1", - "model": "mistralai/ministral-8b-2512", - "provider": "provider_1", + "alias": "sql-pro", + "model": "", + "provider": "vllm", "inference_parameters": { - "temperature": 0.7, - "max_tokens": 2048 + "temperature": 0.7 } } ], @@ -93,19 +92,35 @@ "column_type": "llm-text", "name": "sql_prompt", "drop": false, - "model_alias": "model_1", + "model_alias": "sql-pro", "prompt": "Generate one natural-language SQL task.\nContext:\n- Domain: {{ domain }}\n- Topic: {{ topic }}\n- Task type: {{ sql_task_type }}\nRules:\n- Must start exactly with: \"{{ instruction_phrase }}\"\n- Make it specific and practical.\n- Mention expected business outcome.\n- Keep it 1-2 sentences.\n- Do not include SQL code.\n- Output only the instruction text.", "system_prompt": "You create clear, realistic business SQL tasks for training data.\n", - "with_trace": "none" + "with_trace": "none", + "extract_reasoning_content": false }, { "column_type": "llm-code", "name": "sql", "drop": false, - "model_alias": "model_1", + "model_alias": "sql-pro", "prompt": "Write SQL for this instruction:\n{{ sql_prompt }}\nReturn ONE SQL script with this exact structure:\n-- SCHEMA\n[CREATE TABLE statements]\n[INSERT statements with sample rows]\n-- QUERY\n[final SELECT query solving the instruction]\nRules:\n- Use 2-3 tables max.\n- Use realistic snake_case names.\n- Include 5-8 rows of sample data per table.\n- Query must match task type \"{{ sql_task_type }}\".\n- Use only tables/columns you created.\n- No markdown fences.\n- No explanation text outside SQL comments shown above.", "system_prompt": "You are an expert SQL engineer. Produce correct, runnable SQL only.\n", + "with_trace": "none", + "extract_reasoning_content": false, "code_lang": "sql:ansi" + }, + { + "column_type": "validation", + "name": "sql-validator", + "drop": false, + "target_columns": [ + "sql" + ], + "validator_type": "code", + "validator_params": { + "code_lang": "sql:ansi" + }, + "batch_size": 10 } ], "processors": [] @@ -119,58 +134,10 @@ }, "ui": { "nodes": [ - { - "id": "provider_1", - "x": -1092.2003193114556, - "y": 715.157165665104, - "width": 400 - }, - { - "id": "model_1", - "x": -546.1001596557278, - "y": 681.8114012018752, - "width": 400 - }, - { - "id": "domain", - "x": -18.379173679952572, - "y": 137.70260329000595, - "width": 400 - }, - { - "id": "topic", - "x": -18.6022437080035, - "y": 373.0271253737222, - "width": 400 - }, - { - "id": "sql_task_type", - "x": 477.85851567876665, - "y": 137.4202046707293, - "width": 400 - }, - { - "id": "instruction_phrase", - "x": -477.8585156787667, - "y": 138.42770371608808, - "width": 400 - }, - { - "id": "sql_prompt", - "x": -18.188598798124787, - "y": 701.5157165665104, - "width": 400 - }, - { - "id": "sql", - "x": -18.188598798124758, - "y": 950.647309355259, - "width": 400 - }, { "id": "note_1", - "x": -103.00586025666547, - "y": -332.088439142397, + "x": 338, + "y": 1020, "width": 600, "node_type": "markdown_note", "name": "note_1", @@ -180,8 +147,8 @@ }, { "id": "note_2", - "x": 517.0372102151987, - "y": 600.4949327304814, + "x": 1675.8410596026492, + "y": 1644.2185430463576, "width": 400, "node_type": "markdown_note", "name": "note_2", @@ -191,25 +158,79 @@ }, { "id": "note_3", - "x": 12.635681904967385, - "y": 1224.7626182706356, + "x": 2198.980132450331, + "y": 1723.1456953642385, "width": 400, "node_type": "markdown_note", "name": "note_3", - "markdown": "The **LLM Code** block (`sql`) generates SQL script from `{{ sql_prompt }}`.\n\n##### In this recipe it returns:\n\n- schema section (`CREATE TABLE`)\n- sample seed rows (`INSERT`)\n- final query (`SELECT`)\n\n##### Current status:\n\n- SQL validation block is **not** included yet in this learning recipe\n- We will add SQL validation later", + "markdown": "The **LLM Code** block (`sql`) generates SQL script from `{{ sql_prompt }}`.\n\n##### In this recipe it returns:\n\n- schema section (`CREATE TABLE`)\n- sample seed rows (`INSERT`)\n- final query (`SELECT`)\n", "note_color": "#DBEAFE", "note_opacity": "35" }, { "id": "note_4", - "x": -1044, - "y": 108.64730935525904, + "x": 1264, + "y": 1037, "width": 400, "node_type": "markdown_note", "name": "note_4", "markdown": "Sampler columns are useful during generation, but often noisy in final output.\n\nSet helper columns to **drop=true** (like in this recipe), keep only output columns you want to export.\n\n#### Final keep we have set here:\n\n- `sql_prompt`\n- `sql`\n\n", "note_color": "#DBEAFE", "note_opacity": "35" + }, + { + "id": "vllm", + "x": 1939.5364238410598, + "y": 781.25, + "width": 400 + }, + { + "id": "sql-pro", + "x": 1939.5364238410593, + "y": 975.25, + "width": 400 + }, + { + "id": "domain", + "x": 680, + "y": 1495, + "width": 400 + }, + { + "id": "topic", + "x": 1160, + "y": 1413, + "width": 400 + }, + { + "id": "sql_task_type", + "x": 1160, + "y": 1577, + "width": 400 + }, + { + "id": "instruction_phrase", + "x": 100, + "y": 1495, + "width": 400 + }, + { + "id": "sql_prompt", + "x": 1672.6490066225165, + "y": 1457.6854304635763, + "width": 400 + }, + { + "id": "sql", + "x": 2194.9006622516554, + "y": 1457.110927152318, + "width": 400 + }, + { + "id": "sql-validator", + "x": 2682.5827814569534, + "y": 1491.0413907284767, + "width": 400 } ], "edges": [ @@ -217,8 +238,8 @@ "from": "domain", "to": "topic", "type": "canvas", - "source_handle": "data-out-bottom", - "target_handle": "data-in-top" + "source_handle": "data-out", + "target_handle": "data-in" }, { "from": "domain", @@ -238,38 +259,52 @@ "from": "topic", "to": "sql_prompt", "type": "canvas", - "source_handle": "data-out-bottom", - "target_handle": "data-in-top" + "source_handle": "data-out", + "target_handle": "data-in" }, { "from": "sql_prompt", "to": "sql", "type": "canvas", - "source_handle": "data-out-bottom", - "target_handle": "data-in-top" - }, - { - "from": "provider_1", - "to": "model_1", - "type": "semantic", - "source_handle": "semantic-out", - "target_handle": "semantic-in" - }, - { - "from": "model_1", - "to": "sql_prompt", - "type": "semantic", - "source_handle": "semantic-out", + "source_handle": "data-out", "target_handle": "data-in" }, { - "from": "model_1", + "from": "vllm", + "to": "sql-pro", + "type": "semantic", + "source_handle": "semantic-out-bottom", + "target_handle": "semantic-in-top" + }, + { + "from": "sql-pro", "to": "sql", "type": "semantic", "source_handle": "semantic-out-bottom", + "target_handle": "data-in-top" + }, + { + "from": "sql", + "to": "sql-validator", + "type": "semantic", + "source_handle": "data-out", "target_handle": "data-in" + }, + { + "from": "sql-pro", + "to": "sql_prompt", + "type": "semantic", + "source_handle": "semantic-out-bottom", + "target_handle": "data-in-top" + }, + { + "from": "sql_prompt", + "to": "sql_task_type", + "type": "canvas", + "source_handle": "data-out-left", + "target_handle": "data-in-right" } ], "layout_direction": "LR" } -} +} \ No newline at end of file diff --git a/studio/frontend/src/features/data-recipes/pages/data-recipes-page.tsx b/studio/frontend/src/features/data-recipes/pages/data-recipes-page.tsx index 056f895c78..e9e85e23b8 100644 --- a/studio/frontend/src/features/data-recipes/pages/data-recipes-page.tsx +++ b/studio/frontend/src/features/data-recipes/pages/data-recipes-page.tsx @@ -24,7 +24,7 @@ import { import { ShineBorder } from "@/components/ui/shine-border"; import { toastError } from "@/shared/toast"; import { - AiChat02Icon, + Album02Icon, ArrowDown01Icon, CodeIcon, CookBookIcon, @@ -60,20 +60,20 @@ type TemplateCard = { const TEMPLATE_CARDS: TemplateCard[] = [ { - title: "Structured Outputs + Jinja Expressions", + title: "Instruction from Answer", description: - "Support ticket triage dataset with structured JSON outputs and Jinja if/else refs.", - icon: FunctionIcon, - difficulty: "Advanced", - learningBadges: ["Structured LLM", "Expression", "Jinja"], + "Start from seed answer fields and generate matching user instructions for SFT pairs.", + icon: Plant01Icon, + difficulty: "Easy", + learningBadges: ["Seed Dataset", "LLM Text", "Prompting"], surfaceClassName: - "from-cyan-500/15 via-sky-500/5 to-transparent dark:from-cyan-400/30 dark:via-sky-400/14 dark:to-cyan-950/16", + "from-emerald-500/15 via-green-500/5 to-transparent dark:from-emerald-400/30 dark:via-green-400/14 dark:to-emerald-950/16", shineColor: [ - "rgb(6 182 212 / 0.45)", - "rgb(56 189 248 / 0.4)", - "rgb(34 211 238 / 0.45)", + "rgb(16 185 129 / 0.45)", + "rgb(34 197 94 / 0.4)", + "rgb(52 211 153 / 0.45)", ], - learningRecipeId: "structured-outputs-jinja", + learningRecipeId: "instruction-from-answer", }, { title: "PDF Document QA", @@ -92,20 +92,20 @@ const TEMPLATE_CARDS: TemplateCard[] = [ learningRecipeId: "pdf-grounded-qa", }, { - title: "Instruction from Answer", + title: "OCR Document Extraction", description: - "Start from seed answer fields and generate matching user instructions for SFT pairs.", - icon: Plant01Icon, - difficulty: "Easy", - learningBadges: ["Seed Dataset", "LLM Text", "Prompting"], + "Use image context from seed data to generate OCR-style extraction outputs.", + icon: Album02Icon, + difficulty: "Starter", + learningBadges: ["Vision", "LLM Text", "Image Context"], surfaceClassName: - "from-emerald-500/15 via-green-500/5 to-transparent dark:from-emerald-400/30 dark:via-green-400/14 dark:to-emerald-950/16", + "from-lime-500/15 via-emerald-500/5 to-transparent dark:from-lime-400/30 dark:via-emerald-400/14 dark:to-lime-950/16", shineColor: [ - "rgb(16 185 129 / 0.45)", - "rgb(34 197 94 / 0.4)", - "rgb(52 211 153 / 0.45)", + "rgb(132 204 22 / 0.45)", + "rgb(16 185 129 / 0.4)", + "rgb(74 222 128 / 0.45)", ], - learningRecipeId: "instruction-from-answer", + learningRecipeId: "ocr-document-extraction", }, { title: "Text to Python", @@ -140,20 +140,20 @@ const TEMPLATE_CARDS: TemplateCard[] = [ learningRecipeId: "text-to-sql", }, { - title: "Multi-Turn Chat", + title: "Structured Outputs + Jinja Expressions", description: - "Role-based multi-turn conversations for assistant behavior, memory, and response quality.", - icon: AiChat02Icon, - difficulty: "Easy", - learningBadges: ["Structured LLM", "LLM Text"], + "Support ticket triage dataset with structured JSON outputs and Jinja if/else refs.", + icon: FunctionIcon, + difficulty: "Advanced", + learningBadges: ["Structured LLM", "Expression", "Jinja"], surfaceClassName: - "from-rose-500/15 via-pink-500/5 to-transparent dark:from-rose-400/30 dark:via-pink-400/14 dark:to-rose-950/16", + "from-cyan-500/15 via-sky-500/5 to-transparent dark:from-cyan-400/30 dark:via-sky-400/14 dark:to-cyan-950/16", shineColor: [ - "rgb(244 63 94 / 0.45)", - "rgb(236 72 153 / 0.4)", - "rgb(251 113 133 / 0.45)", + "rgb(6 182 212 / 0.45)", + "rgb(56 189 248 / 0.4)", + "rgb(34 211 238 / 0.45)", ], - learningRecipeId: "conversation", + learningRecipeId: "structured-outputs-jinja", }, ]; @@ -207,7 +207,10 @@ function LearningRecipeCards({ loadingTemplateId === template.learningRecipeId; const isDisabled = !isReady || isLoading || Boolean(loadingTemplateId); const visibleLearningBadges = template.learningBadges.slice(0, 4); - const extraLearningBadgeCount = Math.max(0, template.learningBadges.length - 4); + const extraLearningBadgeCount = Math.max( + 0, + template.learningBadges.length - 4, + ); return ( + + + + ); +} diff --git a/studio/frontend/src/features/recipe-studio/components/controls/viewport-controls.tsx b/studio/frontend/src/features/recipe-studio/components/controls/viewport-controls.tsx index e104841100..2ba13bfb33 100644 --- a/studio/frontend/src/features/recipe-studio/components/controls/viewport-controls.tsx +++ b/studio/frontend/src/features/recipe-studio/components/controls/viewport-controls.tsx @@ -7,11 +7,13 @@ import { RECIPE_FLOATING_ICON_BUTTON_CLASS } from "../recipe-floating-icon-butto type ViewportControlsProps = { interactive: boolean; + lockDisabled?: boolean; onToggleInteractive: () => void; }; export function ViewportControls({ interactive, + lockDisabled = false, onToggleInteractive, }: ViewportControlsProps): ReactElement { const { zoomIn, zoomOut, fitView, getNodes } = useReactFlow(); @@ -68,6 +70,7 @@ export function ViewportControls({ variant="ghost" size="icon" className={RECIPE_FLOATING_ICON_BUTTON_CLASS} + disabled={lockDisabled} onClick={onToggleInteractive} aria-label={interactive ? "Lock interaction" : "Unlock interaction"} > diff --git a/studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsx b/studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsx index 56ca1d5057..3d4cc11ab6 100644 --- a/studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsx +++ b/studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsx @@ -12,7 +12,7 @@ import { import { cn } from "@/lib/utils"; import { isExecutionInProgress } from "../../executions/execution-helpers"; import type { RecipeExecutionRecord } from "../../execution-types"; -import { formatCellValue, isExpandableCellValue } from "./executions-view-helpers"; +import { hasExpandableTextCell } from "./executions-view-helpers"; type ExecutionDataTabProps = { execution: RecipeExecutionRecord; @@ -129,9 +129,7 @@ export function ExecutionDataTab({ columns={tableColumns} data={datasetRowsForTable} getRowClassName={(row, _rowIndex, rowId) => { - const canExpand = visibleDatasetColumnNames.some((columnName) => - isExpandableCellValue(formatCellValue(row[columnName])), - ); + const canExpand = hasExpandableTextCell(row, visibleDatasetColumnNames); if (!canExpand) { return undefined; } @@ -141,9 +139,7 @@ export function ExecutionDataTab({ ); }} onRowClick={(row, _rowIndex, rowId) => { - const canExpand = visibleDatasetColumnNames.some((columnName) => - isExpandableCellValue(formatCellValue(row[columnName])), - ); + const canExpand = hasExpandableTextCell(row, visibleDatasetColumnNames); if (!canExpand || !selectedExecutionIdSafe) { return; } diff --git a/studio/frontend/src/features/recipe-studio/components/executions/execution-overview-tab.tsx b/studio/frontend/src/features/recipe-studio/components/executions/execution-overview-tab.tsx index f0808c1a5d..e24d23bc8e 100644 --- a/studio/frontend/src/features/recipe-studio/components/executions/execution-overview-tab.tsx +++ b/studio/frontend/src/features/recipe-studio/components/executions/execution-overview-tab.tsx @@ -57,7 +57,7 @@ export function ExecutionOverviewTab({ {showSummaryCards && (
-
+

Run summary

Final stage - {execution.stage ?? "--"} + {execution.stage ?? "--"}

-
+

Insights

{nullRate?.toFixed(1) ?? "--"}%

- Dropped columns + Side-effect columns {formatMetricValue(sideEffects.length)}

{sideEffects.length > 0 && ( @@ -128,12 +128,12 @@ export function ExecutionOverviewTab({
{lowUniquenessColumns.slice(0, 3).map((name) => ( - + {name} ))} {lowUniquenessColumns.length > 3 && ( - + +{lowUniquenessColumns.length - 3} more )} @@ -143,7 +143,7 @@ export function ExecutionOverviewTab({
-
+

Model usage

@@ -151,7 +151,7 @@ export function ExecutionOverviewTab({ {modelUsageRows.length === 0 ? (

No model usage yet.

) : ( -
+
diff --git a/studio/frontend/src/features/recipe-studio/components/executions/execution-sidebar.tsx b/studio/frontend/src/features/recipe-studio/components/executions/execution-sidebar.tsx index fac1f30ea9..75441daf09 100644 --- a/studio/frontend/src/features/recipe-studio/components/executions/execution-sidebar.tsx +++ b/studio/frontend/src/features/recipe-studio/components/executions/execution-sidebar.tsx @@ -2,7 +2,11 @@ import type { ReactElement } from "react"; import { Badge } from "@/components/ui/badge"; import { cn } from "@/lib/utils"; import type { RecipeExecutionRecord } from "../../execution-types"; -import { isExecutionInProgress } from "../../executions/execution-helpers"; +import { + executionLabel, + isExecutionInProgress, + normalizeRunName, +} from "../../executions/execution-helpers"; import { formatStatus, formatTimestamp, @@ -22,55 +26,62 @@ export function ExecutionSidebar({ onSelectExecution, }: ExecutionSidebarProps): ReactElement { return ( -