merge nightly, resolve conflict in use-chat-model-runtime

This commit is contained in:
Roland Tannous 2026-03-09 13:19:17 +00:00
commit 91dd7fc762
133 changed files with 8339 additions and 1573 deletions

3
.gitignore vendored
View file

@ -7,6 +7,7 @@ __pycache__/
*$py.class
*.so
.Python
*.egg-info/
# Virtual environments
.venv/
@ -23,6 +24,7 @@ unsloth_compiled_cache/
outputs/
exports/
/datasets/
studio/backend/assets/datasets/
unsloth_training_checkpoints/
*.gguf
*.safetensors
@ -53,6 +55,7 @@ firebase-debug.log
# Other
resources/
tmp/
**/node_modules/
auth.db
studio/frontend/package-lock.json

View file

@ -22,6 +22,9 @@ SCRIPT_DIR = Path(__file__).resolve().parent
REQ_ROOT = SCRIPT_DIR / "studio" / "backend" / "requirements"
SINGLE_ENV = REQ_ROOT / "single-env"
CONSTRAINTS = SINGLE_ENV / "constraints.txt"
LOCAL_DD_UNSTRUCTURED_PLUGIN = (
SCRIPT_DIR / "studio" / "backend" / "plugins" / "data-designer-unstructured-seed"
)
# ── Color support ──────────────────────────────────────────────────────
@ -235,13 +238,28 @@ def install_python_stack() -> int:
req=SINGLE_ENV / "data-designer.txt",
)
# 11. Patch metadata for single-env compatibility
# 11. Local Data Designer seed plugin
if not LOCAL_DD_UNSTRUCTURED_PLUGIN.is_dir():
print(
_red(
f"❌ Missing local plugin directory: {LOCAL_DD_UNSTRUCTURED_PLUGIN}",
),
)
return 1
pip_install(
"Installing local data-designer unstructured plugin",
"--no-cache-dir", "--no-deps",
"-e", str(LOCAL_DD_UNSTRUCTURED_PLUGIN),
constrain=False,
)
# 12. Patch metadata for single-env compatibility
run(
"Patching single-env metadata",
[sys.executable, str(SINGLE_ENV / "patch_metadata.py")],
)
# 12. Final check (silent — third-party conflicts are expected)
# 13. Final check (silent; third-party conflicts are expected)
subprocess.run(
[sys.executable, "-m", "pip", "check"],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,

View file

@ -17,6 +17,7 @@ $PackageDir = Split-Path -Parent $ScriptDir
# Detect if running from pip install (no studio/frontend/ dir in repo)
$FrontendDir = Join-Path $ScriptDir "studio\frontend"
$OxcValidatorDir = Join-Path $ScriptDir "studio\backend\core\data_recipe\oxc-validator"
$IsPipInstall = -not (Test-Path $FrontendDir)
# ─────────────────────────────────────────────
@ -599,6 +600,23 @@ if ($IsPipInstall) {
Write-Host "[OK] Frontend built to studio/frontend/dist" -ForegroundColor Green
}
if (Test-Path $OxcValidatorDir) {
Write-Host "Installing OXC validator runtime..." -ForegroundColor Cyan
$prevEAP_oxc = $ErrorActionPreference
$ErrorActionPreference = "Continue"
Push-Location $OxcValidatorDir
npm install 2>&1 | Out-Null
if ($LASTEXITCODE -ne 0) {
Pop-Location
$ErrorActionPreference = $prevEAP_oxc
Write-Host "[ERROR] OXC validator npm install failed (exit code $LASTEXITCODE)" -ForegroundColor Red
exit 1
}
Pop-Location
$ErrorActionPreference = $prevEAP_oxc
Write-Host "[OK] OXC validator runtime installed" -ForegroundColor Green
}
# ==========================================================================
# PHASE 3: Python environment + dependencies
# ==========================================================================
@ -992,4 +1010,4 @@ Write-Host "| IMPORTANT: Open a NEW terminal, then run: |" -ForegroundColor
Write-Host "| |" -ForegroundColor Green
Write-Host "| unsloth-studio -H 0.0.0.0 -p 8000 |" -ForegroundColor Green
Write-Host "| |" -ForegroundColor Green
Write-Host "+===============================================+" -ForegroundColor Green
Write-Host "+===============================================+" -ForegroundColor Green

View file

@ -99,6 +99,8 @@ echo "Building frontend..."
cd "$SCRIPT_DIR/studio/frontend"
run_quiet "npm install" npm install
run_quiet "npm run build" npm run build
cd "$SCRIPT_DIR/studio/backend/core/data_recipe/oxc-validator"
run_quiet "npm install (oxc validator runtime)" npm install
cd "$SCRIPT_DIR"
echo "✅ Frontend built to studio/frontend/dist"

View file

@ -13,7 +13,7 @@ from typing import Any
import multiprocessing as mp
from ..jsonable import to_jsonable
from ..jsonable import to_preview_jsonable
from .constants import (
EVENT_JOB_CANCELLING,
EVENT_JOB_CANCELLED,
@ -138,6 +138,7 @@ class JobManager:
"status": job.status,
"stage": job.stage,
"current_column": job.current_column,
"completed_columns": list(job.completed_columns),
"batch": {"idx": job.batch.idx, "total": job.batch.total},
"progress": {
"done": job.progress.done,
@ -298,7 +299,7 @@ class JobManager:
dataframe = dataframe.drop(columns=[helper_col])
rows = dataframe.to_dict(orient="records")
return {"dataset": to_jsonable(rows), "total": total}
return {"dataset": to_preview_jsonable(rows), "total": total}
@staticmethod
def _load_dataset_page_with_data_designer(
@ -312,7 +313,7 @@ class JobManager:
dataframe = read_parquet_dataset(parquet_dir)
total = int(len(dataframe.index))
rows = dataframe.iloc[offset:offset + limit].to_dict(orient="records")
return {"dataset": to_jsonable(rows), "total": total}
return {"dataset": to_preview_jsonable(rows), "total": total}
def subscribe(self, job_id: str, *, after_seq: int | None = None) -> Subscription | None:
"""SSE subscribe: get replay buffer + live events stream."""

View file

@ -151,6 +151,15 @@ def apply_update(job: Job, update: ParsedUpdate) -> None:
job.cols = update.cols
if update.progress is not None:
job.column_progress = update.progress
if (
job.current_column
and update.progress.done is not None
and update.progress.total is not None
and update.progress.total > 0
and update.progress.done >= update.progress.total
and job.current_column not in job.completed_columns
):
job.completed_columns.append(job.current_column)
job.progress = _compute_overall_progress(job, update.progress)
if update.batch_idx is not None:
job.batch.idx = update.batch_idx
@ -240,5 +249,5 @@ def _compute_overall_progress(job: Job, column_progress: Progress) -> Progress:
def coerce_event(obj: Any) -> dict:
# worker sends dict already
"""Normalize worker payload into event dict."""
return obj if isinstance(obj, dict) else {"type": "log", "message": str(obj)}

View file

@ -66,6 +66,7 @@ class Job:
processor_artifacts: dict[str, Any] | None = None
model_usage: dict[str, ModelUsage] = field(default_factory=dict)
progress_columns_total: int | None = None
completed_columns: list[str] = field(default_factory=list)
_current_usage_model: str | None = None
_in_usage_summary: bool = False
_seen_generation_columns: list[str] = field(default_factory=list)

View file

@ -1,13 +1,16 @@
from __future__ import annotations
import json
import logging
import re
import shutil
import time
import traceback
import unicodedata
from pathlib import Path
from typing import Any
from ..jsonable import to_jsonable
from ..jsonable import to_jsonable, to_preview_jsonable
from .constants import EVENT_JOB_COMPLETED, EVENT_JOB_ERROR, EVENT_JOB_STARTED
from ..service import build_config_builder, create_data_designer
@ -34,6 +37,27 @@ class _QueueLogHandler(logging.Handler):
pass
def _slugify_run_name(value: str) -> str:
normalized = unicodedata.normalize("NFKD", value)
ascii_only = normalized.encode("ascii", "ignore").decode("ascii")
slug = re.sub(r"[^a-zA-Z0-9]+", "-", ascii_only).strip("-").lower()
if not slug:
return ""
return slug[:80].strip("-")
def _build_dataset_name(*, run_name: str | None, job_id: str, artifact_root: Path) -> str:
fallback = f"recipe_{job_id}"
slug = _slugify_run_name(run_name or "")
base_name = f"recipe_{slug}" if slug else fallback
candidate = base_name
suffix = 2
while (artifact_root / candidate).exists():
candidate = f"{base_name}_{suffix}"
suffix += 1
return candidate
def run_job_process(
*,
event_queue,
@ -53,7 +77,13 @@ def run_job_process(
job_id = str(run.get("_job_id") or "").strip()
if not job_id:
job_id = f"{int(time.time())}"
dataset_name = f"recipe_{job_id}"
run_name_raw = run.get("run_name")
run_name = run_name_raw if isinstance(run_name_raw, str) else None
dataset_name = _build_dataset_name(
run_name=run_name,
job_id=job_id,
artifact_root=_ARTIFACT_ROOT,
)
merge_batches = bool(run.get("merge_batches"))
_ARTIFACT_ROOT.mkdir(parents=True, exist_ok=True)
run_config_raw = run.get("run_config") or {}
@ -84,7 +114,7 @@ def run_job_process(
dataset = (
[]
if results.dataset is None
else to_jsonable(results.dataset.to_dict(orient="records"))
else to_preview_jsonable(results.dataset.to_dict(orient="records"))
)
processor_artifacts = (
None
@ -142,4 +172,40 @@ def _merge_batches_to_single_parquet(base_dataset_path: Path) -> None:
dataframe = read_parquet_dataset(parquet_dir)
shutil.rmtree(parquet_dir)
parquet_dir.mkdir(parents=True, exist_ok=True)
dataframe.to_parquet(parquet_dir / "batch_00000.parquet", index=False)
merged_file = parquet_dir / "batch_00000.parquet"
dataframe.to_parquet(merged_file, index=False)
_rewrite_merged_metadata(
base_dataset_path=base_dataset_path,
parquet_file=merged_file,
)
def _rewrite_merged_metadata(*, base_dataset_path: Path, parquet_file: Path) -> None:
metadata_path = base_dataset_path / "metadata.json"
if not metadata_path.exists():
return
try:
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
except (OSError, TypeError, ValueError):
return
if not isinstance(metadata, dict):
return
relative_parquet_path = str(parquet_file.relative_to(base_dataset_path))
file_paths = metadata.get("file_paths")
if not isinstance(file_paths, dict):
file_paths = {}
file_paths["parquet-files"] = [relative_parquet_path]
metadata["file_paths"] = file_paths
metadata["total_num_batches"] = 1
metadata["num_completed_batches"] = 1
try:
metadata_path.write_text(
json.dumps(metadata, indent=2, sort_keys=True),
encoding="utf-8",
)
except OSError:
return

View file

@ -1,8 +1,63 @@
from __future__ import annotations
import base64
import io
from pathlib import Path
from typing import Any
def _pil_to_preview_payload(image: Any) -> dict[str, Any]:
buffer = io.BytesIO()
image.convert("RGB").save(buffer, format="JPEG", quality=85)
return {
"type": "image",
"mime": "image/jpeg",
"width": image.width,
"height": image.height,
"data": base64.b64encode(buffer.getvalue()).decode("ascii"),
}
def _open_pil_image_from_bytes(raw_bytes: bytes):
from PIL import Image # type: ignore
with Image.open(io.BytesIO(raw_bytes)) as image:
return image.copy()
def _to_pil_from_hf_image_dict(value: Any) -> Any | None:
if not isinstance(value, dict):
return None
raw_bytes = value.get("bytes")
if isinstance(raw_bytes, (bytes, bytearray)) and len(raw_bytes) > 0:
try:
return _open_pil_image_from_bytes(bytes(raw_bytes))
except (OSError, ValueError):
pass
if (
isinstance(raw_bytes, list)
and len(raw_bytes) > 0
and all(isinstance(item, int) and 0 <= item <= 255 for item in raw_bytes)
):
try:
return _open_pil_image_from_bytes(bytes(raw_bytes))
except (OSError, ValueError):
pass
path_value = value.get("path")
if isinstance(path_value, str) and path_value.strip():
try:
from PIL import Image # type: ignore
with Image.open(Path(path_value)) as image:
return image.copy()
except (OSError, ValueError, TypeError):
return None
return None
def to_jsonable(value: Any) -> Any:
"""Convert numpy/pandas-ish values into plain JSON-safe values."""
try:
@ -28,3 +83,36 @@ def to_jsonable(value: Any) -> Any:
return value
return value
def _to_preview_image_payload(value: Any) -> dict[str, Any] | None:
try:
from PIL.Image import Image as PILImage # type: ignore
except ImportError: # pragma: no cover
return None
if not isinstance(value, PILImage):
hf_image = _to_pil_from_hf_image_dict(value)
if hf_image is None:
return None
value = hf_image
return _pil_to_preview_payload(value)
def to_preview_jsonable(value: Any) -> Any:
"""Convert values into JSON-safe preview values, including PIL images."""
image_payload = _to_preview_image_payload(value)
if image_payload is not None:
return image_payload
converted = to_jsonable(value)
if converted is None or isinstance(converted, (str, int, float, bool)):
return converted
if isinstance(converted, dict):
return {str(k): to_preview_jsonable(v) for k, v in converted.items()}
if isinstance(converted, (list, tuple, set)):
return [to_preview_jsonable(v) for v in converted]
if isinstance(converted, (bytes, bytearray)):
return base64.b64encode(bytes(converted)).decode("ascii")
return str(converted)

View file

@ -0,0 +1,315 @@
from __future__ import annotations
import json
import logging
import subprocess
from copy import deepcopy
from dataclasses import dataclass
from functools import lru_cache
from pathlib import Path
from typing import Any
logger = logging.getLogger(__name__)
OXC_VALIDATION_FN_MARKER = "unsloth_oxc_validator"
_OXC_LANG_TO_NODE_LANG = {
"javascript": "js",
"typescript": "ts",
"jsx": "jsx",
"tsx": "tsx",
}
_OXC_VALIDATION_MODES = {"syntax", "lint", "syntax+lint"}
_OXC_CODE_SHAPES = {"auto", "module", "snippet"}
_OXC_TOOL_DIR = Path(__file__).resolve().parent / "oxc-validator"
_OXC_RUNNER_PATH = _OXC_TOOL_DIR / "validate.mjs"
@dataclass(frozen=True)
class OxcLocalCallableValidatorSpec:
name: str
drop: bool
target_columns: list[str]
batch_size: int
code_lang: str
validation_mode: str
code_shape: str
def split_oxc_local_callable_validators(
recipe_core: dict[str, Any],
) -> tuple[dict[str, Any], list[OxcLocalCallableValidatorSpec]]:
columns = recipe_core.get("columns")
if not isinstance(columns, list):
return recipe_core, []
sanitized = deepcopy(recipe_core)
sanitized_columns = sanitized.get("columns")
if not isinstance(sanitized_columns, list):
return sanitized, []
kept_columns: list[Any] = []
oxc_specs: list[OxcLocalCallableValidatorSpec] = []
for column in sanitized_columns:
if not isinstance(column, dict):
kept_columns.append(column)
continue
maybe_spec = _parse_oxc_spec(column=column)
if maybe_spec is None:
kept_columns.append(column)
continue
oxc_specs.append(maybe_spec)
sanitized["columns"] = kept_columns
return sanitized, oxc_specs
def register_oxc_local_callable_validators(
*,
builder,
specs: list[OxcLocalCallableValidatorSpec],
) -> None:
if not specs:
return
from data_designer.config.column_configs import ValidationColumnConfig
from data_designer.config.validator_params import (
LocalCallableValidatorParams,
ValidatorType,
)
for spec in specs:
validation_function = _build_oxc_validation_function(
spec.code_lang,
spec.validation_mode,
spec.code_shape,
)
builder.add_column(
ValidationColumnConfig(
name=spec.name,
drop=spec.drop,
target_columns=spec.target_columns,
validator_type=ValidatorType.LOCAL_CALLABLE,
validator_params=LocalCallableValidatorParams(
validation_function=validation_function,
),
batch_size=spec.batch_size,
)
)
def _parse_oxc_spec(
*,
column: dict[str, Any],
) -> OxcLocalCallableValidatorSpec | None:
if str(column.get("column_type") or "").strip() != "validation":
return None
if str(column.get("validator_type") or "").strip() != "local_callable":
return None
params = column.get("validator_params")
if not isinstance(params, dict):
return None
fn_raw = params.get("validation_function")
fn_name = fn_raw.strip() if isinstance(fn_raw, str) else ""
if not fn_name.startswith(OXC_VALIDATION_FN_MARKER):
return None
name = str(column.get("name") or "").strip()
if not name:
return None
target_columns_raw = column.get("target_columns")
target_columns = (
[value.strip() for value in target_columns_raw if isinstance(value, str) and value.strip()]
if isinstance(target_columns_raw, list)
else []
)
if not target_columns:
return None
code_lang, validation_mode, code_shape = _parse_oxc_validation_marker(fn_name)
batch_size = _parse_batch_size(column.get("batch_size"))
drop = bool(column.get("drop") is True)
return OxcLocalCallableValidatorSpec(
name=name,
drop=drop,
target_columns=target_columns,
batch_size=batch_size,
code_lang=code_lang,
validation_mode=validation_mode,
code_shape=code_shape,
)
def _parse_batch_size(value: Any) -> int:
try:
parsed = int(value)
except (TypeError, ValueError):
return 10
return parsed if parsed >= 1 else 10
def _parse_oxc_validation_marker(fn_name: str) -> tuple[str, str, str]:
marker = f"{OXC_VALIDATION_FN_MARKER}:"
if not fn_name.startswith(marker):
return "javascript", "syntax", "auto"
suffix = fn_name[len(marker) :]
parts = [part.strip() for part in suffix.split(":") if part.strip()]
if len(parts) < 2:
return "javascript", "syntax", "auto"
code_lang = parts[0] if parts[0] in _OXC_LANG_TO_NODE_LANG else "javascript"
mode = parts[1] if parts[1] in _OXC_VALIDATION_MODES else "syntax"
code_shape = parts[2] if len(parts) >= 3 and parts[2] in _OXC_CODE_SHAPES else "auto"
return code_lang, mode, code_shape
@lru_cache(maxsize=8)
def _build_oxc_validation_function(lang: str, validation_mode: str, code_shape: str):
node_lang = _OXC_LANG_TO_NODE_LANG.get(lang, "js")
mode = (
validation_mode
if validation_mode in _OXC_VALIDATION_MODES
else "syntax"
)
normalized_code_shape = code_shape if code_shape in _OXC_CODE_SHAPES else "auto"
def _validator(df):
import pandas as pd # imported lazily for local callable runtime
row_count = int(len(df.index))
if row_count == 0:
return pd.DataFrame({"is_valid": []})
code_column = str(df.columns[0]) if len(df.columns) > 0 else ""
code_values = (
["" for _ in range(row_count)]
if not code_column
else ["" if value is None else str(value) for value in df[code_column].tolist()]
)
results = _run_oxc_batch(
node_lang=node_lang,
validation_mode=mode,
code_shape=normalized_code_shape,
code_values=code_values,
)
if len(results) != row_count:
results = _fallback_results(
row_count,
"OXC validator returned mismatched result size.",
)
return pd.DataFrame(results)
_validator.__name__ = (
f"{OXC_VALIDATION_FN_MARKER}_{node_lang}_{mode.replace('+', '_')}_{normalized_code_shape}"
)
return _validator
def _run_oxc_batch(
*,
node_lang: str,
validation_mode: str,
code_shape: str,
code_values: list[str],
) -> list[dict[str, Any]]:
if not _OXC_RUNNER_PATH.exists():
return _fallback_results(
len(code_values),
f"OXC runner missing at {_OXC_RUNNER_PATH}",
)
payload = {
"lang": node_lang,
"mode": validation_mode,
"code_shape": code_shape,
"codes": code_values,
}
try:
proc = subprocess.run(
["node", str(_OXC_RUNNER_PATH)],
cwd=str(_OXC_TOOL_DIR),
input=json.dumps(payload),
text=True,
capture_output=True,
check=False,
)
except (OSError, ValueError) as exc:
logger.warning("OXC subprocess launch failed: %s", exc)
return _fallback_results(len(code_values), f"OXC launch failed: {exc}")
if proc.returncode != 0:
message = (proc.stderr or proc.stdout or "unknown error").strip()
if len(message) > 300:
message = f"{message[:300]}..."
return _fallback_results(len(code_values), f"OXC failed: {message}")
try:
raw = json.loads(proc.stdout)
except json.JSONDecodeError:
return _fallback_results(len(code_values), "OXC output parse failed.")
if not isinstance(raw, list):
return _fallback_results(len(code_values), "OXC output must be an array.")
out: list[dict[str, Any]] = []
for item in raw:
if not isinstance(item, dict):
out.append(
{
"is_valid": False,
"error_count": 1,
"error_message": "Invalid OXC result entry.",
"severity": None,
"code": None,
"labels": [],
"codeframe": None,
"warning_count": 0,
}
)
continue
is_valid_raw = item.get("is_valid")
error_count_raw = item.get("error_count")
message_raw = item.get("error_message")
severity_raw = item.get("severity")
code_raw = item.get("code")
labels_raw = item.get("labels")
codeframe_raw = item.get("codeframe")
warning_count_raw = item.get("warning_count")
out.append(
{
"is_valid": bool(is_valid_raw) if isinstance(is_valid_raw, bool) else False,
"error_count": int(error_count_raw) if isinstance(error_count_raw, int) else 0,
"error_message": str(message_raw or ""),
"severity": str(severity_raw) if isinstance(severity_raw, str) else None,
"code": str(code_raw) if isinstance(code_raw, str) else None,
"labels": labels_raw if isinstance(labels_raw, list) else [],
"codeframe": str(codeframe_raw) if isinstance(codeframe_raw, str) else None,
"warning_count": int(warning_count_raw)
if isinstance(warning_count_raw, int)
else 0,
}
)
return out
def _fallback_results(row_count: int, message: str) -> list[dict[str, Any]]:
return [
{
"is_valid": False,
"error_count": 1,
"error_message": message,
"severity": None,
"code": None,
"labels": [],
"codeframe": None,
"warning_count": 0,
}
for _ in range(row_count)
]

View file

@ -0,0 +1,794 @@
{
"name": "unsloth-oxc-validator-runtime",
"version": "0.0.1",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "unsloth-oxc-validator-runtime",
"version": "0.0.1",
"dependencies": {
"oxc-parser": "^0.116.0",
"oxlint": "^1.51.0"
}
},
"node_modules/@emnapi/core": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.8.1.tgz",
"integrity": "sha512-AvT9QFpxK0Zd8J0jopedNm+w/2fIzvtPKPjqyw9jwvBaReTTqPBk9Hixaz7KbjimP+QNz605/XnjFcDAL2pqBg==",
"license": "MIT",
"optional": true,
"dependencies": {
"@emnapi/wasi-threads": "1.1.0",
"tslib": "^2.4.0"
}
},
"node_modules/@emnapi/runtime": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.1.tgz",
"integrity": "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==",
"license": "MIT",
"optional": true,
"dependencies": {
"tslib": "^2.4.0"
}
},
"node_modules/@emnapi/wasi-threads": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz",
"integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==",
"license": "MIT",
"optional": true,
"dependencies": {
"tslib": "^2.4.0"
}
},
"node_modules/@napi-rs/wasm-runtime": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.1.tgz",
"integrity": "sha512-p64ah1M1ld8xjWv3qbvFwHiFVWrq1yFvV4f7w+mzaqiR4IlSgkqhcRdHwsGgomwzBH51sRY4NEowLxnaBjcW/A==",
"license": "MIT",
"optional": true,
"dependencies": {
"@emnapi/core": "^1.7.1",
"@emnapi/runtime": "^1.7.1",
"@tybys/wasm-util": "^0.10.1"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@oxc-parser/binding-android-arm-eabi": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm-eabi/-/binding-android-arm-eabi-0.116.0.tgz",
"integrity": "sha512-AOET7YIOU3+ANO/3xQeRVGN5Xx6+JGXaIwlqkcHSfxJ/zzw2B6jb0YaLhX45SeRluKVTU8rka4N/tHtNoJjoCg==",
"cpu": [
"arm"
],
"license": "MIT",
"optional": true,
"os": [
"android"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxc-parser/binding-android-arm64": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm64/-/binding-android-arm64-0.116.0.tgz",
"integrity": "sha512-yh0Zvth5cQ6XZkP3QF9MDrXf695zr5XxXq/wBQqpZb0uAgI9wpr98/Hx2RZITMfnNjkIq2VcyU44o3A0bdEmlQ==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"android"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxc-parser/binding-darwin-arm64": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-arm64/-/binding-darwin-arm64-0.116.0.tgz",
"integrity": "sha512-plcTd/Jska55dToZz6XdRBPRVsj+asjD8QCpQFvt3Wj8pY+10D1pE53Mei3POAS/wSRSy7HiQ2twrm7H2A0CjA==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxc-parser/binding-darwin-x64": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-x64/-/binding-darwin-x64-0.116.0.tgz",
"integrity": "sha512-ahqcF3e3x5Z2ZepzXpZ8ugREdmxvBL+g1nQ0SxO11pIZfck6UtbOtwtdAAxnQXBHHtidu7lPcrBq1SEx26t1PQ==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxc-parser/binding-freebsd-x64": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-freebsd-x64/-/binding-freebsd-x64-0.116.0.tgz",
"integrity": "sha512-yo2/LaSXtlzKBurvNbwun/sN/RJwW3XhbMr069FwNVtft7GBnaLLdPIz/sf47icxw/BPViEX6wFvzeD12mtrAg==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"freebsd"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxc-parser/binding-linux-arm-gnueabihf": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-0.116.0.tgz",
"integrity": "sha512-EiZeliIPPdFsuaPx8PzDMVijD/4YaUxO46/eYPk5raRocJqjjxOG6GAacQ8UrG3fbrgYjaEChfYL1e8DyE445A==",
"cpu": [
"arm"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxc-parser/binding-linux-arm-musleabihf": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-0.116.0.tgz",
"integrity": "sha512-Nf7hnKRYRSIgglQcLAqE2St4b/Yr6dh+Z7in8mxol065Knevw71XZAiV1fmPSojq6uKPLV9eoH/wFrgr4TnZXw==",
"cpu": [
"arm"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxc-parser/binding-linux-arm64-gnu": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-0.116.0.tgz",
"integrity": "sha512-9SJI0S4Qggn3QHpT8Y1jtZceA0m4BlpvO3ne2Wxd33UdTHMmelAnrXryjWutHWQtjCzOwSnFBEoQAdNNyt1u3A==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxc-parser/binding-linux-arm64-musl": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-musl/-/binding-linux-arm64-musl-0.116.0.tgz",
"integrity": "sha512-wMZ6//GI+q1JwO7G2OR51+eA5P8Gr3BobU8RAzCGJptvyGMkWb7KQ1E8s8naVZRr6bSGWAL2p3mCzKOxmEPmrA==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxc-parser/binding-linux-ppc64-gnu": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-0.116.0.tgz",
"integrity": "sha512-5BO0KCzTG2HZTnp3r6SCAOeCs/GwFBQJ1WAOG/ROfDf1fVVEy6hrtLKTLCuUMaamH38v+1+RVEmzRkzBj+rMDQ==",
"cpu": [
"ppc64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxc-parser/binding-linux-riscv64-gnu": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-0.116.0.tgz",
"integrity": "sha512-M24gYb/ocVMnLwnH2wY5sLt4sRBkAUHDmfiYtyUYdKTkfPOKtpopd5otsL/BPLnIhpMD8zby4uXVvw7BU0UIlw==",
"cpu": [
"riscv64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxc-parser/binding-linux-riscv64-musl": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-0.116.0.tgz",
"integrity": "sha512-LHLXTHCH0bdvGjlitwr1ngeh32GAgq9HYzQ5VAgt0k0UT84AS8AkXj9Spoa9l20fXkVgSvAKcCEkydi4Ol23Dw==",
"cpu": [
"riscv64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxc-parser/binding-linux-s390x-gnu": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-0.116.0.tgz",
"integrity": "sha512-VE+XsztuE5jdHvLIDIQMuyDpz5NJGq1Vx/8EXYF0sS/gehlv9GhDpGVWU0SCZ/LjzIy4io/Z0W84UudqufvP3g==",
"cpu": [
"s390x"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxc-parser/binding-linux-x64-gnu": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-gnu/-/binding-linux-x64-gnu-0.116.0.tgz",
"integrity": "sha512-rxUkauyjjCmgA7BoR63ogRGEtgubROnCm8AXE9ydg+p42jCGLLqG05mFcS2eC+FYyAU58ZFJNXXeqFW1iCyTGQ==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxc-parser/binding-linux-x64-musl": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-musl/-/binding-linux-x64-musl-0.116.0.tgz",
"integrity": "sha512-0zoZlk9MmXe6oTgSh5lT1D51SDC1bfwC96JmE1amMFAPdEbJk5MFRisfTN9TFBpBigQua65842tjaxqMiorAYw==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxc-parser/binding-openharmony-arm64": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-openharmony-arm64/-/binding-openharmony-arm64-0.116.0.tgz",
"integrity": "sha512-PGS7Xqik77U9WMyW626gAD5A2rSN629UvyYJKAl/tgpT+KqZI4+56pJfExhv8IW/PpSHjYHwjmakwobLikz8ww==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"openharmony"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxc-parser/binding-wasm32-wasi": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-wasm32-wasi/-/binding-wasm32-wasi-0.116.0.tgz",
"integrity": "sha512-lGNf/9PU8XxB4Gt1Gr1AKwSrjxGYa6os0PlrT4bpoQsfE3gaZonQTKwJyKhiQdgy7pBCI+ed1LB1NNib1FYULw==",
"cpu": [
"wasm32"
],
"license": "MIT",
"optional": true,
"dependencies": {
"@napi-rs/wasm-runtime": "^1.1.1"
},
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/@oxc-parser/binding-win32-arm64-msvc": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-0.116.0.tgz",
"integrity": "sha512-tcsOHE31duBSRQXZ7NfdtjmMKZwQYlS00PwAMJ4w5oXs3iPCvisUuIXP7Ko4FzeOBTRvkd64btxtQ6cRM0Kwlw==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxc-parser/binding-win32-ia32-msvc": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-0.116.0.tgz",
"integrity": "sha512-higCz/x+dOQ264YEk22hnu4RDqvjhfehjFORpxoh42QyUxsP6eIembYesBUu5ilALWo0HvRD+m89az2BSTwqpQ==",
"cpu": [
"ia32"
],
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxc-parser/binding-win32-x64-msvc": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.116.0.tgz",
"integrity": "sha512-Lg2SRmVHpGG85knDVLbv44r1bYn0OpIV0vg9jVmoEIpDj3Q4kwXuQ6MWVtuslwHR8o2CSiqdBeEn1n1URrs6Eg==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxc-project/types": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.116.0.tgz",
"integrity": "sha512-uOT8S1tlPmDckNxMNtIudN/yXpLdnhlJMX2oLS7cxCd7L0sUF09A/EbSVMWT3Y/iT44IwXCJSJfgfSxXAqWf9Q==",
"license": "MIT",
"funding": {
"url": "https://github.com/sponsors/Boshen"
}
},
"node_modules/@oxlint/binding-android-arm-eabi": {
"version": "1.51.0",
"resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm-eabi/-/binding-android-arm-eabi-1.51.0.tgz",
"integrity": "sha512-jJYIqbx4sX+suIxWstc4P7SzhEwb4ArWA2KVrmEuu9vH2i0qM6QIHz/ehmbGE4/2fZbpuMuBzTl7UkfNoqiSgw==",
"cpu": [
"arm"
],
"license": "MIT",
"optional": true,
"os": [
"android"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxlint/binding-android-arm64": {
"version": "1.51.0",
"resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm64/-/binding-android-arm64-1.51.0.tgz",
"integrity": "sha512-GtXyBCcH4ti98YdiMNCrpBNGitx87EjEWxevnyhcBK12k/Vu4EzSB45rzSC4fGFUD6sQgeaxItRCEEWeVwPafw==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"android"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxlint/binding-darwin-arm64": {
"version": "1.51.0",
"resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-arm64/-/binding-darwin-arm64-1.51.0.tgz",
"integrity": "sha512-3QJbeYaMHn6Bh2XeBXuITSsbnIctyTjvHf5nRjKYrT9pPeErNIpp5VDEeAXC0CZSwSVTsc8WOSDwgrAI24JolQ==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxlint/binding-darwin-x64": {
"version": "1.51.0",
"resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-x64/-/binding-darwin-x64-1.51.0.tgz",
"integrity": "sha512-NzErhMaTEN1cY0E8C5APy74lw5VwsNfJfVPBMWPVQLqAbO0k4FFLjvHURvkUL+Y18Wu+8Vs1kbqPh2hjXYA4pg==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxlint/binding-freebsd-x64": {
"version": "1.51.0",
"resolved": "https://registry.npmjs.org/@oxlint/binding-freebsd-x64/-/binding-freebsd-x64-1.51.0.tgz",
"integrity": "sha512-msAIh3vPAoKoHlOE/oe6Q5C/n9umypv/k81lED82ibrJotn+3YG2Qp1kiR8o/Dg5iOEU97c6tl0utxcyFenpFw==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"freebsd"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxlint/binding-linux-arm-gnueabihf": {
"version": "1.51.0",
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.51.0.tgz",
"integrity": "sha512-CqQPcvqYyMe9ZBot2stjGogEzk1z8gGAngIX7srSzrzexmXixwVxBdFZyxTVM0CjGfDeV+Ru0w25/WNjlMM2Hw==",
"cpu": [
"arm"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxlint/binding-linux-arm-musleabihf": {
"version": "1.51.0",
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-1.51.0.tgz",
"integrity": "sha512-dstrlYQgZMnyOssxSbolGCge/sDbko12N/35RBNuqLpoPbft2aeBidBAb0dvQlyBd9RJ6u8D4o4Eh8Un6iTgyQ==",
"cpu": [
"arm"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxlint/binding-linux-arm64-gnu": {
"version": "1.51.0",
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.51.0.tgz",
"integrity": "sha512-QEjUpXO7d35rP1/raLGGbAsBLLGZIzV3ZbeSjqWlD3oRnxpRIZ6iL4o51XQHkconn3uKssc+1VKdtHJ81BBhDA==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxlint/binding-linux-arm64-musl": {
"version": "1.51.0",
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.51.0.tgz",
"integrity": "sha512-YSJua5irtG4DoMAjUapDTPhkQLHhBIY0G9JqlZS6/SZPzqDkPku/1GdWs0D6h/wyx0Iz31lNCfIaWKBQhzP0wQ==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxlint/binding-linux-ppc64-gnu": {
"version": "1.51.0",
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.51.0.tgz",
"integrity": "sha512-7L4Wj2IEUNDETKssB9IDYt16T6WlF+X2jgC/hBq3diGHda9vJLpAgb09+D3quFq7TdkFtI7hwz/jmuQmQFPc1Q==",
"cpu": [
"ppc64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxlint/binding-linux-riscv64-gnu": {
"version": "1.51.0",
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-1.51.0.tgz",
"integrity": "sha512-cBUHqtOXy76G41lOB401qpFoKx1xq17qYkhWrLSM7eEjiHM9sOtYqpr6ZdqCnN9s6ZpzudX4EkeHOFH2E9q0vA==",
"cpu": [
"riscv64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxlint/binding-linux-riscv64-musl": {
"version": "1.51.0",
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-1.51.0.tgz",
"integrity": "sha512-WKbg8CysgZcHfZX0ixQFBRSBvFZUHa3SBnEjHY2FVYt2nbNJEjzTxA3ZR5wMU0NOCNKIAFUFvAh5/XJKPRJuJg==",
"cpu": [
"riscv64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxlint/binding-linux-s390x-gnu": {
"version": "1.51.0",
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.51.0.tgz",
"integrity": "sha512-N1QRUvJTxqXNSu35YOufdjsAVmKVx5bkrggOWAhTWBc3J4qjcBwr1IfyLh/6YCg8sYRSR1GraldS9jUgJL/U4A==",
"cpu": [
"s390x"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxlint/binding-linux-x64-gnu": {
"version": "1.51.0",
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.51.0.tgz",
"integrity": "sha512-e0Mz0DizsCoqNIjeOg6OUKe8JKJWZ5zZlwsd05Bmr51Jo3AOL4UJnPvwKumr4BBtBrDZkCmOLhCvDGm95nJM2g==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxlint/binding-linux-x64-musl": {
"version": "1.51.0",
"resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-musl/-/binding-linux-x64-musl-1.51.0.tgz",
"integrity": "sha512-wD8HGTWhYBKXvRDvoBVB1y+fEYV01samhWQSy1Zkxq2vpezvMnjaFKRuiP6tBNITLGuffbNDEXOwcAhJ3gI5Ug==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxlint/binding-openharmony-arm64": {
"version": "1.51.0",
"resolved": "https://registry.npmjs.org/@oxlint/binding-openharmony-arm64/-/binding-openharmony-arm64-1.51.0.tgz",
"integrity": "sha512-5NSwQ2hDEJ0GPXqikjWtwzgAQCsS7P9aLMNenjjKa+gknN3lTCwwwERsT6lKXSirfU3jLjexA2XQvQALh5h27w==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"openharmony"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxlint/binding-win32-arm64-msvc": {
"version": "1.51.0",
"resolved": "https://registry.npmjs.org/@oxlint/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.51.0.tgz",
"integrity": "sha512-JEZyah1M0RHMw8d+jjSSJmSmO8sABA1J1RtrHYujGPeCkYg1NeH0TGuClpe2h5QtioRTaF57y/TZfn/2IFV6fA==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxlint/binding-win32-ia32-msvc": {
"version": "1.51.0",
"resolved": "https://registry.npmjs.org/@oxlint/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-1.51.0.tgz",
"integrity": "sha512-q3cEoKH6kwjz/WRyHwSf0nlD2F5Qw536kCXvmlSu+kaShzgrA0ojmh45CA81qL+7udfCaZL2SdKCZlLiGBVFlg==",
"cpu": [
"ia32"
],
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@oxlint/binding-win32-x64-msvc": {
"version": "1.51.0",
"resolved": "https://registry.npmjs.org/@oxlint/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.51.0.tgz",
"integrity": "sha512-Q14+fOGb9T28nWF/0EUsYqERiRA7cl1oy4TJrGmLaqhm+aO2cV+JttboHI3CbdeMCAyDI1+NoSlrM7Melhp/cw==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": "^20.19.0 || >=22.12.0"
}
},
"node_modules/@tybys/wasm-util": {
"version": "0.10.1",
"resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz",
"integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==",
"license": "MIT",
"optional": true,
"dependencies": {
"tslib": "^2.4.0"
}
},
"node_modules/oxc-parser": {
"version": "0.116.0",
"resolved": "https://registry.npmjs.org/oxc-parser/-/oxc-parser-0.116.0.tgz",
"integrity": "sha512-ugEo6wwqaqCGcpi7GsLCwSkoD7gIXzvtdaTxE+mbrXFYazU5Q9YdpZdAj9z2b79i/xlv+uW2aAvyzGAlpUzhKQ==",
"license": "MIT",
"dependencies": {
"@oxc-project/types": "^0.116.0"
},
"engines": {
"node": "^20.19.0 || >=22.12.0"
},
"funding": {
"url": "https://github.com/sponsors/Boshen"
},
"optionalDependencies": {
"@oxc-parser/binding-android-arm-eabi": "0.116.0",
"@oxc-parser/binding-android-arm64": "0.116.0",
"@oxc-parser/binding-darwin-arm64": "0.116.0",
"@oxc-parser/binding-darwin-x64": "0.116.0",
"@oxc-parser/binding-freebsd-x64": "0.116.0",
"@oxc-parser/binding-linux-arm-gnueabihf": "0.116.0",
"@oxc-parser/binding-linux-arm-musleabihf": "0.116.0",
"@oxc-parser/binding-linux-arm64-gnu": "0.116.0",
"@oxc-parser/binding-linux-arm64-musl": "0.116.0",
"@oxc-parser/binding-linux-ppc64-gnu": "0.116.0",
"@oxc-parser/binding-linux-riscv64-gnu": "0.116.0",
"@oxc-parser/binding-linux-riscv64-musl": "0.116.0",
"@oxc-parser/binding-linux-s390x-gnu": "0.116.0",
"@oxc-parser/binding-linux-x64-gnu": "0.116.0",
"@oxc-parser/binding-linux-x64-musl": "0.116.0",
"@oxc-parser/binding-openharmony-arm64": "0.116.0",
"@oxc-parser/binding-wasm32-wasi": "0.116.0",
"@oxc-parser/binding-win32-arm64-msvc": "0.116.0",
"@oxc-parser/binding-win32-ia32-msvc": "0.116.0",
"@oxc-parser/binding-win32-x64-msvc": "0.116.0"
}
},
"node_modules/oxlint": {
"version": "1.51.0",
"resolved": "https://registry.npmjs.org/oxlint/-/oxlint-1.51.0.tgz",
"integrity": "sha512-g6DNPaV9/WI9MoX2XllafxQuxwY1TV++j7hP8fTJByVBuCoVtm3dy9f/2vtH/HU40JztcgWF4G7ua+gkainklQ==",
"license": "MIT",
"bin": {
"oxlint": "bin/oxlint"
},
"engines": {
"node": "^20.19.0 || >=22.12.0"
},
"funding": {
"url": "https://github.com/sponsors/Boshen"
},
"optionalDependencies": {
"@oxlint/binding-android-arm-eabi": "1.51.0",
"@oxlint/binding-android-arm64": "1.51.0",
"@oxlint/binding-darwin-arm64": "1.51.0",
"@oxlint/binding-darwin-x64": "1.51.0",
"@oxlint/binding-freebsd-x64": "1.51.0",
"@oxlint/binding-linux-arm-gnueabihf": "1.51.0",
"@oxlint/binding-linux-arm-musleabihf": "1.51.0",
"@oxlint/binding-linux-arm64-gnu": "1.51.0",
"@oxlint/binding-linux-arm64-musl": "1.51.0",
"@oxlint/binding-linux-ppc64-gnu": "1.51.0",
"@oxlint/binding-linux-riscv64-gnu": "1.51.0",
"@oxlint/binding-linux-riscv64-musl": "1.51.0",
"@oxlint/binding-linux-s390x-gnu": "1.51.0",
"@oxlint/binding-linux-x64-gnu": "1.51.0",
"@oxlint/binding-linux-x64-musl": "1.51.0",
"@oxlint/binding-openharmony-arm64": "1.51.0",
"@oxlint/binding-win32-arm64-msvc": "1.51.0",
"@oxlint/binding-win32-ia32-msvc": "1.51.0",
"@oxlint/binding-win32-x64-msvc": "1.51.0"
},
"peerDependencies": {
"oxlint-tsgolint": ">=0.15.0"
},
"peerDependenciesMeta": {
"oxlint-tsgolint": {
"optional": true
}
}
},
"node_modules/tslib": {
"version": "2.8.1",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
"integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
"license": "0BSD",
"optional": true
}
}
}

View file

@ -0,0 +1,10 @@
{
"name": "unsloth-oxc-validator-runtime",
"private": true,
"version": "0.0.1",
"type": "module",
"dependencies": {
"oxc-parser": "^0.116.0",
"oxlint": "^1.51.0"
}
}

View file

@ -0,0 +1,573 @@
import { spawnSync } from "node:child_process";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { basename, dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { parseSync } from "oxc-parser";
const LANG_TO_EXT = {
js: "js",
jsx: "jsx",
ts: "ts",
tsx: "tsx",
};
const VALIDATION_MODES = new Set(["syntax", "lint", "syntax+lint"]);
const CODE_SHAPES = new Set(["auto", "module", "snippet"]);
const SNIPPET_PREFIX = "(() => {\n";
const SNIPPET_SUFFIX = "\n})();\nexport {};\n";
const OXLINT_SUPPRESSED_RULES = ["no-unused-vars", "no-new-array"];
const TOOL_DIR = dirname(fileURLToPath(import.meta.url));
function mapLang(value) {
const normalized = String(value || "").trim().toLowerCase();
if (normalized === "javascript" || normalized === "js") {
return "js";
}
if (normalized === "typescript" || normalized === "ts") {
return "ts";
}
if (normalized === "jsx") {
return "jsx";
}
if (normalized === "tsx") {
return "tsx";
}
return "js";
}
function mapMode(value) {
const normalized = String(value || "").trim().toLowerCase();
if (VALIDATION_MODES.has(normalized)) {
return normalized;
}
return "syntax";
}
function mapCodeShape(value) {
const normalized = String(value || "").trim().toLowerCase();
if (CODE_SHAPES.has(normalized)) {
return normalized;
}
return "auto";
}
function parseFileIndex(filePath) {
if (typeof filePath !== "string") {
return null;
}
const match = basename(filePath).match(/^snippet_(\d+)\./);
if (!match) {
return null;
}
const parsed = Number.parseInt(match[1], 10);
return Number.isFinite(parsed) ? parsed : null;
}
function toCodeString(code) {
return typeof code === "string" ? code : String(code ?? "");
}
function makeValidationEntry({ code, index, lang, codeShape }) {
const source = toCodeString(code);
if (codeShape === "snippet") {
return {
index,
lang,
code: `${SNIPPET_PREFIX}${source}${SNIPPET_SUFFIX}`,
offset: SNIPPET_PREFIX.length,
};
}
return {
index,
lang,
code: source,
offset: 0,
};
}
function shiftOffset(value, offset) {
if (!Number.isInteger(value)) {
return null;
}
const shifted = value - offset;
return shifted >= 0 ? shifted : null;
}
function remapDiagnosticOffsets(diagnostic, offset) {
if (!diagnostic || typeof diagnostic !== "object" || offset <= 0) {
return diagnostic;
}
return {
...diagnostic,
labels: Array.isArray(diagnostic.labels)
? diagnostic.labels.map((label) => ({
...label,
start: shiftOffset(label.start, offset),
end: shiftOffset(label.end, offset),
}))
: [],
};
}
function normalizeParserError(error) {
if (typeof error === "string") {
return {
code: null,
message: error.trim() || "Unknown parser error",
severity: null,
labels: [],
codeframe: null,
};
}
if (!error || typeof error !== "object") {
return {
code: null,
message: "Unknown parser error",
severity: null,
labels: [],
codeframe: null,
};
}
const code = typeof error.code === "string" ? error.code : null;
const message = String(error.message || error.reason || "").trim() || "Unknown parser error";
const severity = typeof error.severity === "string" ? error.severity : null;
const labels = Array.isArray(error.labels)
? error.labels.map((label) => ({
message:
label && typeof label === "object" && typeof label.message === "string"
? label.message
: null,
start:
label && typeof label === "object" && Number.isInteger(label.start)
? label.start
: null,
end:
label && typeof label === "object" && Number.isInteger(label.end)
? label.end
: null,
}))
: [];
const codeframe = typeof error.codeframe === "string" ? error.codeframe : null;
return {
code,
message,
severity,
labels,
codeframe,
};
}
function normalizeLintDiagnostic(diagnostic) {
if (!diagnostic || typeof diagnostic !== "object") {
return null;
}
const readString = (value) =>
typeof value === "string" ? value : null;
const readInt = (value) =>
Number.isInteger(value) ? value : null;
const asObject = (value) =>
value && typeof value === "object" ? value : null;
const message = String(diagnostic.message || "").trim();
if (!message) {
return null;
}
const severityRaw = String(diagnostic.severity || "").trim().toLowerCase();
const severity = severityRaw === "error" ? "error" : "warning";
const labels = [];
if (Array.isArray(diagnostic.labels)) {
for (const label of diagnostic.labels) {
const labelObj = asObject(label);
const span = asObject(labelObj?.span);
const start = readInt(span?.offset);
const length = readInt(span?.length);
labels.push({
message: readString(labelObj?.label),
start,
end: start !== null && length !== null ? start + length : null,
});
}
}
const code = typeof diagnostic.code === "string" ? diagnostic.code : null;
return {
code,
message: code ? `${code}: ${message}` : message,
severity,
labels,
codeframe: null,
};
}
function makeResult({
isValid,
errorCount,
warningCount = 0,
message = "",
severity = null,
code = null,
labels = [],
codeframe = null,
}) {
return {
is_valid: Boolean(isValid),
error_count: Number.isInteger(errorCount) ? errorCount : 0,
warning_count: Number.isInteger(warningCount) ? warningCount : 0,
error_message: String(message || ""),
severity: typeof severity === "string" ? severity : null,
code: typeof code === "string" ? code : null,
labels: Array.isArray(labels) ? labels : [],
codeframe: typeof codeframe === "string" ? codeframe : null,
};
}
function syntaxResultFromErrors(errors) {
const first = errors[0] ?? null;
return makeResult({
isValid: errors.length === 0,
errorCount: errors.length,
warningCount: 0,
message: errors.slice(0, 3).map((error) => error.message).join(" | "),
severity: first ? first.severity : null,
code: first ? first.code : null,
labels: first ? first.labels : [],
codeframe: first ? first.codeframe : null,
});
}
function runSyntaxParse(entry) {
const ext = LANG_TO_EXT[entry.lang] ?? "js";
const filename = `snippet_${entry.index}.${ext}`;
try {
const parsed = parseSync(filename, entry.code, {
lang: entry.lang,
sourceType: "module",
showSemanticErrors: true,
});
const errors = Array.isArray(parsed?.errors)
? parsed.errors
.map(normalizeParserError)
.filter(Boolean)
.map((error) => remapDiagnosticOffsets(error, entry.offset))
: [];
return errors;
} catch (error) {
return [
remapDiagnosticOffsets(
normalizeParserError(error),
entry.offset,
),
];
}
}
function pickPreferredErrorList(firstErrors, secondErrors) {
if (secondErrors.length < firstErrors.length) {
return secondErrors;
}
return firstErrors;
}
function validateSyntaxOne({ code, lang, index, codeShape }) {
if (codeShape !== "auto") {
const lintEntry = makeValidationEntry({
code,
index,
lang,
codeShape,
});
const errors = runSyntaxParse(lintEntry);
return {
result: syntaxResultFromErrors(errors),
lintEntry,
};
}
const moduleEntry = makeValidationEntry({
code,
index,
lang,
codeShape: "module",
});
const moduleErrors = runSyntaxParse(moduleEntry);
if (moduleErrors.length === 0) {
return {
result: syntaxResultFromErrors(moduleErrors),
lintEntry: moduleEntry,
};
}
const snippetEntry = makeValidationEntry({
code,
index,
lang,
codeShape: "snippet",
});
const snippetErrors = runSyntaxParse(snippetEntry);
if (snippetErrors.length === 0) {
return {
result: syntaxResultFromErrors(snippetErrors),
lintEntry: snippetEntry,
};
}
const chosenErrors = pickPreferredErrorList(moduleErrors, snippetErrors);
const lintEntry = chosenErrors === snippetErrors ? snippetEntry : moduleEntry;
return {
result: syntaxResultFromErrors(chosenErrors),
lintEntry,
};
}
function resolveLintEntry({ code, lang, index, codeShape }) {
if (codeShape !== "auto") {
return makeValidationEntry({
code,
index,
lang,
codeShape,
});
}
const moduleEntry = makeValidationEntry({
code,
index,
lang,
codeShape: "module",
});
if (runSyntaxParse(moduleEntry).length === 0) {
return moduleEntry;
}
const snippetEntry = makeValidationEntry({
code,
index,
lang,
codeShape: "snippet",
});
if (runSyntaxParse(snippetEntry).length === 0) {
return snippetEntry;
}
return moduleEntry;
}
function fallbackLintResults(entries, message) {
return new Map(
entries.map((entry) => [
entry.index,
makeResult({
isValid: false,
errorCount: 1,
warningCount: 0,
message,
severity: "error",
}),
]),
);
}
function runLintBatch(entries) {
if (entries.length === 0) {
return new Map();
}
const entryByIndex = new Map(entries.map((entry) => [entry.index, entry]));
const tempDir = mkdtempSync(join(tmpdir(), "oxlint-"));
try {
for (const entry of entries) {
const ext = LANG_TO_EXT[entry.lang] ?? "js";
const filePath = join(tempDir, `snippet_${entry.index}.${ext}`);
writeFileSync(filePath, entry.code, "utf8");
}
const oxlintBin = join(TOOL_DIR, "node_modules", ".bin", "oxlint");
const oxlintArgs = [
...OXLINT_SUPPRESSED_RULES.flatMap((rule) => ["-A", rule]),
"--format",
"json",
tempDir,
];
const exec = spawnSync(oxlintBin, oxlintArgs, {
encoding: "utf8",
cwd: TOOL_DIR,
});
if (exec.error) {
return fallbackLintResults(
entries,
`oxlint execution failed: ${exec.error.message}`,
);
}
const stdout = String(exec.stdout || "").trim();
if (!stdout) {
const stderr = String(exec.stderr || "").trim();
return fallbackLintResults(
entries,
stderr || "oxlint returned empty output",
);
}
let parsed;
try {
parsed = JSON.parse(stdout);
} catch {
return fallbackLintResults(entries, "oxlint JSON parse failed");
}
const rawDiagnostics = Array.isArray(parsed?.diagnostics)
? parsed.diagnostics
: [];
const byIndex = new Map();
for (const diag of rawDiagnostics) {
const filenameRaw =
typeof diag?.filename === "string" ? diag.filename : "";
const filename = filenameRaw.startsWith("file://")
? filenameRaw.replace("file://", "")
: filenameRaw;
const index = parseFileIndex(filename);
if (index === null) {
continue;
}
const normalized = normalizeLintDiagnostic(diag);
if (!normalized) {
continue;
}
const entry = entryByIndex.get(index);
const remapped = remapDiagnosticOffsets(normalized, entry?.offset ?? 0);
const list = byIndex.get(index) ?? [];
list.push(remapped);
byIndex.set(index, list);
}
const results = new Map();
for (const entry of entries) {
const diagnostics = byIndex.get(entry.index) ?? [];
const errorDiagnostics = diagnostics.filter(
(diag) => diag.severity === "error",
);
const warningDiagnostics = diagnostics.filter(
(diag) => diag.severity !== "error",
);
const top = errorDiagnostics[0] ?? warningDiagnostics[0] ?? null;
const messageSource =
errorDiagnostics.length > 0 ? errorDiagnostics : warningDiagnostics;
results.set(
entry.index,
makeResult({
isValid: errorDiagnostics.length === 0,
errorCount: errorDiagnostics.length,
warningCount: warningDiagnostics.length,
message: messageSource
.slice(0, 3)
.map((diag) => diag.message)
.join(" | "),
severity: top ? top.severity : null,
code: top ? top.code : null,
labels: top ? top.labels : [],
codeframe: top ? top.codeframe : null,
}),
);
}
return results;
} catch (error) {
return fallbackLintResults(entries, `oxlint execution failed: ${error}`);
} finally {
rmSync(tempDir, { recursive: true, force: true });
}
}
function readStdin() {
return new Promise((resolve, reject) => {
let data = "";
process.stdin.setEncoding("utf8");
process.stdin.on("data", (chunk) => {
data += chunk;
});
process.stdin.on("end", () => resolve(data));
process.stdin.on("error", (error) => reject(error));
});
}
function runValidation({ codes, lang, mode, codeShape }) {
if (mode === "syntax") {
return codes.map((code, index) =>
validateSyntaxOne({ code, lang, index, codeShape }).result,
);
}
if (mode === "lint") {
const entries = codes.map((code, index) =>
resolveLintEntry({ code, lang, index, codeShape }),
);
const lintMap = runLintBatch(entries);
return entries.map(
(entry) =>
lintMap.get(entry.index) ??
makeResult({
isValid: true,
errorCount: 0,
warningCount: 0,
}),
);
}
const syntaxRuns = codes.map((code, index) =>
validateSyntaxOne({ code, lang, index, codeShape }),
);
const lintTargets = syntaxRuns
.filter((run) => run.result.is_valid === true)
.map((run) => run.lintEntry);
const lintMap = runLintBatch(lintTargets);
return syntaxRuns.map((run) => {
if (run.result.is_valid !== true) {
return run.result;
}
return (
lintMap.get(run.lintEntry.index) ??
makeResult({
isValid: true,
errorCount: 0,
warningCount: 0,
})
);
});
}
async function main() {
const raw = await readStdin();
let payload;
try {
payload = JSON.parse(raw || "{}");
} catch {
process.stdout.write(
JSON.stringify([
makeResult({
isValid: false,
errorCount: 1,
warningCount: 0,
message: "Invalid JSON payload",
severity: "error",
}),
]),
);
return;
}
const lang = mapLang(payload?.lang);
const mode = mapMode(payload?.mode);
const codeShape = mapCodeShape(payload?.code_shape);
const codes = Array.isArray(payload?.codes) ? payload.codes : [];
const out = runValidation({ codes, lang, mode, codeShape });
process.stdout.write(JSON.stringify(out));
}
main().catch((error) => {
process.stderr.write(String(error?.stack || error));
process.exit(1);
});

View file

@ -1,9 +1,125 @@
from __future__ import annotations
import base64
import io
import os
from pathlib import Path
from typing import Any
from .jsonable import to_jsonable
from .local_callable_validators import (
register_oxc_local_callable_validators,
split_oxc_local_callable_validators,
)
_IMAGE_CONTEXT_PATCHED = False
def _encode_bytes_to_base64(value: bytes | bytearray) -> str:
return base64.b64encode(bytes(value)).decode("utf-8")
def _load_image_file_to_base64(path_value: str, *, base_path: str | None = None) -> str | None:
try:
path = Path(path_value)
candidates: list[Path] = []
if path.is_absolute():
candidates.append(path)
else:
if base_path:
candidates.append(Path(base_path) / path)
candidates.append(Path.cwd() / path)
for candidate in candidates:
if not candidate.exists() or not candidate.is_file():
continue
with candidate.open("rb") as f:
return _encode_bytes_to_base64(f.read())
except (OSError, TypeError, ValueError):
return None
return None
def _pil_image_to_base64(value: Any) -> str | None:
try:
from PIL.Image import Image as PILImage # type: ignore
except ImportError:
return None
if not isinstance(value, PILImage):
return None
buffer = io.BytesIO()
image_format = str(getattr(value, "format", "") or "").upper()
if image_format not in {"PNG", "JPEG", "JPG", "WEBP", "GIF"}:
image_format = "PNG"
value.save(buffer, format=image_format)
return _encode_bytes_to_base64(buffer.getvalue())
def _normalize_image_context_value(value: Any, *, base_path: str | None = None) -> Any:
if isinstance(value, str):
return value
if isinstance(value, (bytes, bytearray)):
return _encode_bytes_to_base64(value)
pil_base64 = _pil_image_to_base64(value)
if pil_base64 is not None:
return pil_base64
if isinstance(value, dict):
url = value.get("url")
if isinstance(url, str):
return url
image_url = value.get("image_url")
if isinstance(image_url, str):
return image_url
if isinstance(image_url, dict):
nested_url = image_url.get("url")
if isinstance(nested_url, str):
return nested_url
inline_data = value.get("data")
if isinstance(inline_data, str):
return inline_data
raw_bytes = value.get("bytes")
if isinstance(raw_bytes, (bytes, bytearray)):
return _encode_bytes_to_base64(raw_bytes)
if isinstance(raw_bytes, str) and raw_bytes.strip():
return raw_bytes
path_value = value.get("path")
if isinstance(path_value, str) and path_value.strip():
if as_base64 := _load_image_file_to_base64(path_value, base_path=base_path):
return as_base64
return path_value
return value
def _apply_data_designer_image_context_patch() -> None:
global _IMAGE_CONTEXT_PATCHED
if _IMAGE_CONTEXT_PATCHED:
return
try:
from data_designer.config.models import ImageContext
except ImportError:
return
if getattr(ImageContext, "_unsloth_image_context_patch_applied", False):
_IMAGE_CONTEXT_PATCHED = True
return
original_auto_resolve = ImageContext._auto_resolve_context_value
def _patched_auto_resolve(self: Any, context_value: Any, base_path: str | None) -> Any:
normalized = _normalize_image_context_value(context_value, base_path=base_path)
return original_auto_resolve(self, normalized, base_path)
ImageContext._auto_resolve_context_value = _patched_auto_resolve
setattr(ImageContext, "_unsloth_image_context_patch_applied", True)
_IMAGE_CONTEXT_PATCHED = True
def build_model_providers(recipe: dict[str, Any]):
@ -75,6 +191,7 @@ def build_mcp_providers(
def build_config_builder(recipe: dict[str, Any]):
_apply_data_designer_image_context_patch()
from data_designer.config import DataDesignerConfigBuilder
from data_designer.config.processors import ProcessorType
@ -83,7 +200,14 @@ def build_config_builder(recipe: dict[str, Any]):
for key, value in recipe.items()
if key not in {"model_providers", "mcp_providers"}
}
recipe_core, oxc_local_callable_specs = split_oxc_local_callable_validators(
recipe_core
)
builder = DataDesignerConfigBuilder.from_config({"data_designer": recipe_core})
register_oxc_local_callable_validators(
builder=builder,
specs=oxc_local_callable_specs,
)
# DataDesignerConfigBuilder.from_config currently skips processors.
# Re-attach explicitly so drop_columns/schema_transform survive API payload.
@ -107,6 +231,7 @@ def create_data_designer(
*,
artifact_path: str | None = None,
):
_apply_data_designer_image_context_patch()
from data_designer.interface.data_designer import DataDesigner
return DataDesigner(

View file

@ -18,6 +18,7 @@ import threading
import math
import logging
import time
from pathlib import Path
from typing import Optional, Callable
from dataclasses import dataclass
import pandas as pd
@ -31,6 +32,9 @@ from trl import SFTTrainer, SFTConfig
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
_BACKEND_ROOT = Path(__file__).resolve().parents[2]
_ASSETS_DATASETS_ROOT = _BACKEND_ROOT / "assets" / "datasets"
@dataclass
class TrainingProgress:
@ -1803,20 +1807,38 @@ class UnslothTrainer:
file_path = dataset_file
else:
# Fallback: try relative to assets/datasets
script_dir = Path(__file__).parent.parent
assets_datasets_dir = script_dir / "assets" / "datasets"
file_path = assets_datasets_dir / dataset_file
if str(file_path).endswith('.json'):
with open(file_path, 'r', encoding='utf-8') as f:
file_path = _ASSETS_DATASETS_ROOT / dataset_file
file_path_obj = Path(file_path)
file_path_str = str(file_path_obj)
if file_path_obj.is_dir():
parquet_dir = (
file_path_obj / "parquet-files"
if (file_path_obj / "parquet-files").exists()
else file_path_obj
)
parquet_files = sorted(parquet_dir.glob("*.parquet"))
if parquet_files:
for parquet_file in parquet_files:
df = pd.read_parquet(parquet_file)
all_data.extend(df.to_dict("records"))
continue
if file_path_str.endswith('.json'):
with open(file_path_obj, 'r', encoding='utf-8') as f:
data = json.load(f)
if isinstance(data, list):
all_data.extend(data)
else:
all_data.append(data)
elif str(file_path).endswith('.csv'):
df = pd.read_csv(file_path)
elif file_path_str.endswith('.csv'):
df = pd.read_csv(file_path_obj)
all_data.extend(df.to_dict('records'))
elif file_path_str.endswith('.parquet'):
df = pd.read_parquet(file_path_obj)
all_data.extend(df.to_dict('records'))
continue
if all_data:
dataset = Dataset.from_list(all_data)

View file

@ -49,6 +49,9 @@ class SeedInspectUploadRequest(BaseModel):
filename: str = Field(min_length=1)
content_base64: str = Field(min_length=1)
preview_size: int = Field(default=10, ge=1, le=50)
seed_source_type: str | None = None
unstructured_chunk_size: int | None = Field(default=None, ge=1, le=20000)
unstructured_chunk_overlap: int | None = Field(default=None, ge=0, le=20000)
class SeedInspectResponse(BaseModel):

View file

@ -1,8 +1,9 @@
"""
Dataset-related Pydantic models for API requests and responses.
"""
from pydantic import BaseModel, model_validator
from typing import Any, Optional, Dict, List
from typing import Any, Dict, List, Optional
from pydantic import BaseModel, Field, model_validator
class CheckFormatRequest(BaseModel):
@ -38,3 +39,23 @@ class CheckFormatResponse(BaseModel):
preview_samples: Optional[List[Dict]] = None
total_rows: Optional[int] = None
warning: Optional[str] = None
class LocalDatasetItem(BaseModel):
class Metadata(BaseModel):
actual_num_records: Optional[int] = None
target_num_records: Optional[int] = None
total_num_batches: Optional[int] = None
num_completed_batches: Optional[int] = None
columns: Optional[List[str]] = None
id: str
label: str
path: str
rows: Optional[int] = None
updated_at: Optional[float] = None
metadata: Optional[Metadata] = None
class LocalDatasetsResponse(BaseModel):
datasets: List[LocalDatasetItem] = Field(default_factory=list)

View file

@ -29,6 +29,34 @@ class UnloadRequest(BaseModel):
model_path: str = Field(..., description="Model identifier to unload")
class ValidateModelRequest(BaseModel):
"""
Lightweight validation request to check whether a model identifier
*can be resolved* into a ModelConfig.
This does NOT actually load weights into GPU memory.
"""
model_path: str = Field(..., description="Model identifier or local path")
hf_token: Optional[str] = Field(None, description="HuggingFace token for gated models")
gguf_variant: Optional[str] = Field(None, description="GGUF quantization variant (e.g. 'Q4_K_M')")
class ValidateModelResponse(BaseModel):
"""
Result of model validation.
valid == True means ModelConfig.from_identifier() succeeded and basic
introspection (GGUF / LoRA / vision flags) is available.
"""
valid: bool = Field(..., description="Whether the model identifier looks valid")
message: str = Field(..., description="Human-readable validation message")
identifier: Optional[str] = Field(None, description="Resolved model identifier")
display_name: Optional[str] = Field(None, description="Display name derived from identifier")
is_gguf: bool = Field(False, description="Whether this is a GGUF model (llama.cpp)")
is_lora: bool = Field(False, description="Whether this is a LoRA adapter")
is_vision: bool = Field(False, description="Whether this is a vision-capable model")
class GenerateRequest(BaseModel):
"""Request for text generation (legacy /generate/stream endpoint)"""
messages: List[dict] = Field(..., description="Chat messages in OpenAI format")

View file

@ -0,0 +1,22 @@
[build-system]
requires = ["setuptools>=68", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "data-designer-unstructured-seed"
version = "0.1.0"
description = "Local Data Designer unstructured seed reader plugin"
requires-python = ">=3.11"
dependencies = [
"data-designer-engine>=0.5.1,<0.6",
"pandas>=2,<3",
]
[project.entry-points."data_designer.plugins"]
unstructured = "data_designer_unstructured_seed.plugin:unstructured_seed_plugin"
[tool.setuptools]
package-dir = {"" = "src"}
[tool.setuptools.packages.find]
where = ["src"]

View file

@ -0,0 +1,21 @@
from .chunking import (
DEFAULT_CHUNK_OVERLAP,
DEFAULT_CHUNK_SIZE,
build_unstructured_preview_rows,
materialize_unstructured_seed_dataset,
resolve_chunking,
)
from .config import UnstructuredSeedSource
from .impl import UnstructuredSeedReader
from .plugin import unstructured_seed_plugin
__all__ = [
"DEFAULT_CHUNK_OVERLAP",
"DEFAULT_CHUNK_SIZE",
"build_unstructured_preview_rows",
"materialize_unstructured_seed_dataset",
"resolve_chunking",
"UnstructuredSeedSource",
"UnstructuredSeedReader",
"unstructured_seed_plugin",
]

View file

@ -0,0 +1,186 @@
from __future__ import annotations
import hashlib
import re
from pathlib import Path
from typing import Any
DEFAULT_CHUNK_SIZE = 1200
DEFAULT_CHUNK_OVERLAP = 200
MAX_CHUNK_SIZE = 20000
_MIN_BREAK_RATIO = 0.6
_CACHE_DIR = Path.home() / ".cache" / "unsloth" / "data-recipe" / "unstructured-seed-cache"
def resolve_chunking(
chunk_size: Any,
chunk_overlap: Any,
) -> tuple[int, int]:
size = _to_int(chunk_size, DEFAULT_CHUNK_SIZE)
size = max(1, min(size, MAX_CHUNK_SIZE))
overlap = _to_int(chunk_overlap, DEFAULT_CHUNK_OVERLAP)
overlap = max(0, min(overlap, max(0, size - 1)))
return size, overlap
def build_unstructured_preview_rows(
*,
source_path: Path,
preview_size: int,
chunk_size: Any,
chunk_overlap: Any,
) -> list[dict[str, str]]:
parquet_path, rows = materialize_unstructured_seed_dataset(
source_path=source_path,
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
)
count = max(0, int(preview_size))
if rows:
return rows[:count]
try:
import pandas as pd
except ImportError as exc: # pragma: no cover
raise RuntimeError(f"pandas is required for unstructured seed processing: {exc}") from exc
dataframe = pd.read_parquet(parquet_path).head(count)
return [
{"chunk_text": str(value.get("chunk_text", "")).strip()}
for value in dataframe.to_dict(orient="records")
if str(value.get("chunk_text", "")).strip()
]
def materialize_unstructured_seed_dataset(
*,
source_path: Path,
chunk_size: Any,
chunk_overlap: Any,
) -> tuple[Path, list[dict[str, str]]]:
resolved = source_path.expanduser().resolve()
if not resolved.is_file():
raise FileNotFoundError(f"unstructured seed file not found: {resolved}")
size, overlap = resolve_chunking(chunk_size, chunk_overlap)
key = _compute_cache_key(
source_path=resolved,
chunk_size=size,
chunk_overlap=overlap,
)
parquet_path = _CACHE_DIR / f"{key}.parquet"
if parquet_path.exists():
return parquet_path, []
text = load_unstructured_text_file(resolved)
chunks = split_text_into_chunks(
text=text,
chunk_size=size,
chunk_overlap=overlap,
)
if not chunks:
raise ValueError("No text found in unstructured seed source.")
rows = [{"chunk_text": chunk} for chunk in chunks]
_CACHE_DIR.mkdir(parents=True, exist_ok=True)
try:
import pandas as pd
except ImportError as exc: # pragma: no cover
raise RuntimeError(f"pandas is required for unstructured seed processing: {exc}") from exc
tmp_path = _CACHE_DIR / f"{key}.tmp.parquet"
pd.DataFrame(rows).to_parquet(tmp_path, index=False)
tmp_path.replace(parquet_path)
return parquet_path, rows
def load_unstructured_text_file(path: Path) -> str:
ext = path.suffix.lower()
if ext not in {".txt", ".md"}:
raise ValueError(f"Unsupported unstructured seed file type: {ext}")
raw = path.read_text(encoding="utf-8", errors="ignore")
return normalize_unstructured_text(raw)
def normalize_unstructured_text(text: str) -> str:
normalized = text.replace("\r\n", "\n").replace("\r", "\n")
return re.sub(r"\n{3,}", "\n\n", normalized).strip()
def split_text_into_chunks(
*,
text: str,
chunk_size: int,
chunk_overlap: int,
) -> list[str]:
if not text:
return []
if chunk_size <= 0:
return [text]
chunks: list[str] = []
start = 0
min_break_index = int(chunk_size * _MIN_BREAK_RATIO)
text_len = len(text)
while start < text_len:
end = min(text_len, start + chunk_size)
if end < text_len:
window = text[start:end]
cut = _find_break_index(window, min_break_index)
if cut is not None and cut > 0:
end = start + cut
if end <= start:
end = min(text_len, start + chunk_size)
chunk = text[start:end].strip()
if chunk:
chunks.append(chunk)
if end >= text_len:
break
next_start = end - chunk_overlap
if next_start <= start:
next_start = end
start = max(0, next_start)
return chunks
def _find_break_index(window: str, min_index: int) -> int | None:
breakpoints = ["\n\n", "\n", " "]
for token in breakpoints:
idx = window.rfind(token)
if idx >= min_index:
return idx + len(token)
return None
def _to_int(value: Any, fallback: int) -> int:
if isinstance(value, bool):
return fallback
try:
parsed = int(str(value).strip())
except (TypeError, ValueError):
return fallback
return parsed
def _compute_cache_key(
*,
source_path: Path,
chunk_size: int,
chunk_overlap: int,
) -> str:
stat = source_path.stat()
payload = "|".join(
[
str(source_path),
str(stat.st_size),
str(stat.st_mtime_ns),
str(chunk_size),
str(chunk_overlap),
]
).encode("utf-8")
return hashlib.sha256(payload).hexdigest()

View file

@ -0,0 +1,38 @@
from __future__ import annotations
from pathlib import Path
from typing import Literal
from pydantic import Field, field_validator
from data_designer.config.seed_source import SeedSource
from .chunking import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, resolve_chunking
class UnstructuredSeedSource(SeedSource):
seed_type: Literal["unstructured"] = "unstructured"
path: str = Field(..., min_length=1)
chunk_size: int = DEFAULT_CHUNK_SIZE
chunk_overlap: int = DEFAULT_CHUNK_OVERLAP
@field_validator("path", mode="after")
@classmethod
def _validate_path(cls, value: str) -> str:
path = Path(value).expanduser()
if not path.is_file():
raise ValueError(f"Unstructured seed path is not a file: {path}")
return value
@field_validator("chunk_size", mode="after")
@classmethod
def _validate_chunk_size(cls, value: int) -> int:
size, _ = resolve_chunking(value, 0)
return size
@field_validator("chunk_overlap", mode="after")
@classmethod
def _validate_chunk_overlap(cls, value: int, info) -> int:
size = info.data.get("chunk_size", cls.model_fields["chunk_size"].default)
_, overlap = resolve_chunking(size, value)
return overlap

View file

@ -0,0 +1,22 @@
from __future__ import annotations
from pathlib import Path
import data_designer.lazy_heavy_imports as lazy
from data_designer.engine.resources.seed_reader import SeedReader
from .chunking import materialize_unstructured_seed_dataset
from .config import UnstructuredSeedSource
class UnstructuredSeedReader(SeedReader[UnstructuredSeedSource]):
def create_duckdb_connection(self):
return lazy.duckdb.connect()
def get_dataset_uri(self) -> str:
path, _ = materialize_unstructured_seed_dataset(
source_path=Path(self.source.path),
chunk_size=self.source.chunk_size,
chunk_overlap=self.source.chunk_overlap,
)
return str(path)

View file

@ -0,0 +1,7 @@
from data_designer.plugins.plugin import Plugin, PluginType
unstructured_seed_plugin = Plugin(
impl_qualified_name="data_designer_unstructured_seed.impl.UnstructuredSeedReader",
config_qualified_name="data_designer_unstructured_seed.config.UnstructuredSeedSource",
plugin_type=PluginType.SEED_READER,
)

View file

@ -5,7 +5,9 @@ from __future__ import annotations
import sys
from pathlib import Path
from fastapi import APIRouter
from fastapi import APIRouter, Depends
from auth.authentication import get_current_subject
backend_path = Path(__file__).parent.parent.parent
if str(backend_path) not in sys.path:
@ -15,7 +17,7 @@ from .jobs import router as jobs_router
from .seed import router as seed_router
from .validate import router as validate_router
router = APIRouter()
router = APIRouter(dependencies=[Depends(get_current_subject)])
router.include_router(seed_router)
router.include_router(validate_router)
router.include_router(jobs_router)

View file

@ -14,6 +14,17 @@ from models.data_recipe import JobCreateResponse, RecipePayload
router = APIRouter()
def _normalize_run_name(value: Any) -> str | None:
if value is None:
return None
if not isinstance(value, str):
raise HTTPException(status_code=400, detail="invalid run_name: must be a string")
trimmed = value.strip()
if not trimmed:
return None
return trimmed[:120]
@router.post("/jobs", response_class=JSONResponse, response_model=JobCreateResponse)
def create_job(payload: RecipePayload):
recipe = payload.recipe
@ -27,6 +38,7 @@ def create_job(payload: RecipePayload):
if execution_type not in {"preview", "full"}:
raise HTTPException(status_code=400, detail="invalid execution_type: must be 'preview' or 'full'")
run["execution_type"] = execution_type
run["run_name"] = _normalize_run_name(run.get("run_name"))
run_config_raw = run.get("run_config")
if run_config_raw is not None:
try:

View file

@ -10,6 +10,11 @@ from typing import Any
from uuid import uuid4
from fastapi import APIRouter, HTTPException
from data_designer_unstructured_seed.chunking import (
build_unstructured_preview_rows,
resolve_chunking,
)
from core.data_recipe.jsonable import to_preview_jsonable
from models.data_recipe import (
SeedInspectRequest,
@ -22,17 +27,12 @@ router = APIRouter()
DATA_EXTS = (".parquet", ".jsonl", ".json", ".csv")
DEFAULT_SPLIT = "train"
LOCAL_UPLOAD_EXTS = {".csv", ".json", ".jsonl"}
UNSTRUCTURED_UPLOAD_EXTS = {".txt", ".md"}
SEED_UPLOAD_DIR = Path.home() / ".cache" / "unsloth" / "data-recipe" / "seed-uploads"
def _serialize_preview_value(value: Any) -> Any:
if value is None or isinstance(value, (str, int, float, bool)):
return value
if isinstance(value, dict):
return {str(key): _serialize_preview_value(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [_serialize_preview_value(item) for item in value]
return str(value)
return to_preview_jsonable(value)
def _serialize_preview_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
@ -63,10 +63,10 @@ def _list_hf_data_files(*, dataset_name: str, token: str | None) -> list[str]:
return []
def _select_best_file(data_files: list[str]) -> str | None:
def _select_best_file(data_files: list[str], split: str = DEFAULT_SPLIT) -> str | None:
if not data_files:
return None
split_lower = DEFAULT_SPLIT
split_lower = split.lower()
def score(path: str) -> tuple[int, int]:
name = path.lower()
@ -85,8 +85,8 @@ def _select_best_file(data_files: list[str]) -> str | None:
return sorted(data_files, key=score)[0]
def _resolve_seed_hf_path(dataset_name: str, data_files: list[str]) -> str | None:
selected = _select_best_file(data_files)
def _resolve_seed_hf_path(dataset_name: str, data_files: list[str], split: str = DEFAULT_SPLIT) -> str | None:
selected = _select_best_file(data_files, split)
if not selected:
return None
@ -185,6 +185,26 @@ def _read_preview_rows_from_local_file(path: Path, preview_size: int) -> list[di
return _serialize_preview_rows(rows)
def _read_preview_rows_from_unstructured_file(
*,
path: Path,
preview_size: int,
chunk_size: int | None,
chunk_overlap: int | None,
) -> list[dict[str, Any]]:
size, overlap = resolve_chunking(chunk_size, chunk_overlap)
try:
rows = build_unstructured_preview_rows(
source_path=path,
preview_size=preview_size,
chunk_size=size,
chunk_overlap=overlap,
)
except (FileNotFoundError, RuntimeError, ValueError, OSError) as exc:
raise HTTPException(status_code=422, detail=f"seed inspect failed: {exc}") from exc
return _serialize_preview_rows(rows)
@router.post("/seed/inspect", response_model=SeedInspectResponse)
def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse:
dataset_name = payload.dataset_name.strip()
@ -196,7 +216,7 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse:
except ImportError as exc:
raise HTTPException(status_code=500, detail=f"seed inspect dependencies unavailable: {exc}") from exc
split = DEFAULT_SPLIT
split = _normalize_optional_text(payload.split) or DEFAULT_SPLIT
subset = _normalize_optional_text(payload.subset)
token = _normalize_optional_text(payload.hf_token)
preview_size = int(payload.preview_size)
@ -204,12 +224,12 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse:
preview_rows: list[dict[str, Any]] = []
data_files = _list_hf_data_files(dataset_name=dataset_name, token=token)
selected_file = _select_best_file(data_files)
selected_file = _select_best_file(data_files, split)
if selected_file:
try:
single_file_kwargs = _build_stream_load_kwargs(
dataset_name=dataset_name,
split=DEFAULT_SPLIT,
split=split,
subset=subset,
token=token,
data_file=selected_file,
@ -246,7 +266,7 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse:
if not data_files:
resolved_path = f"datasets/{dataset_name}/**/*.parquet"
else:
resolved_path = _resolve_seed_hf_path(dataset_name, data_files)
resolved_path = _resolve_seed_hf_path(dataset_name, data_files, split)
if not resolved_path:
raise HTTPException(status_code=422, detail="unable to resolve seed dataset path")
@ -255,18 +275,24 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse:
resolved_path=resolved_path,
columns=columns,
preview_rows=preview_rows,
split=None,
split=split,
subset=subset,
)
@router.post("/seed/inspect-upload", response_model=SeedInspectResponse)
def inspect_seed_upload(payload: SeedInspectUploadRequest) -> SeedInspectResponse:
seed_source_type = _normalize_optional_text(payload.seed_source_type) or "local"
filename = _sanitize_filename(payload.filename)
ext = Path(filename).suffix.lower()
if ext not in LOCAL_UPLOAD_EXTS:
allowed = ", ".join(sorted(LOCAL_UPLOAD_EXTS))
raise HTTPException(status_code=400, detail=f"unsupported file type: {ext}. allowed: {allowed}")
if seed_source_type == "unstructured":
if ext not in UNSTRUCTURED_UPLOAD_EXTS:
allowed = ", ".join(sorted(UNSTRUCTURED_UPLOAD_EXTS))
raise HTTPException(status_code=400, detail=f"unsupported file type: {ext}. allowed: {allowed}")
else:
if ext not in LOCAL_UPLOAD_EXTS:
allowed = ", ".join(sorted(LOCAL_UPLOAD_EXTS))
raise HTTPException(status_code=400, detail=f"unsupported file type: {ext}. allowed: {allowed}")
file_bytes = _decode_base64_payload(payload.content_base64)
if not file_bytes:
@ -280,10 +306,18 @@ def inspect_seed_upload(payload: SeedInspectUploadRequest) -> SeedInspectRespons
stored_path = SEED_UPLOAD_DIR / stored_name
stored_path.write_bytes(file_bytes)
preview_rows = _read_preview_rows_from_local_file(
stored_path,
int(payload.preview_size),
)
if seed_source_type == "unstructured":
preview_rows = _read_preview_rows_from_unstructured_file(
path=stored_path,
preview_size=int(payload.preview_size),
chunk_size=payload.unstructured_chunk_size,
chunk_overlap=payload.unstructured_chunk_overlap,
)
else:
preview_rows = _read_preview_rows_from_local_file(
stored_path,
int(payload.preview_size),
)
if not preview_rows:
raise HTTPException(status_code=422, detail="dataset appears empty or unreadable")
columns = _extract_columns(preview_rows)

View file

@ -3,6 +3,7 @@ Datasets API routes
"""
import base64
import io
import json
import sys
from pathlib import Path
from fastapi import APIRouter, Depends, HTTPException
@ -30,7 +31,12 @@ if not logger.handlers:
logger.setLevel(logging.INFO)
from models.datasets import CheckFormatRequest, CheckFormatResponse
from models.datasets import (
CheckFormatRequest,
CheckFormatResponse,
LocalDatasetItem,
LocalDatasetsResponse,
)
def _serialize_preview_value(value):
@ -82,6 +88,175 @@ DATA_EXTS = (
'.gz', '.zst',
'.zip',
)
LOCAL_FILE_EXTS = ('.json', '.jsonl', '.csv', '.parquet')
BACKEND_ROOT = Path(__file__).resolve().parents[1]
LOCAL_DATASETS_ROOT = BACKEND_ROOT / "assets" / "datasets"
def _safe_read_metadata(path: Path) -> dict | None:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError, TypeError):
return None
if not isinstance(payload, dict):
return None
return payload
def _safe_read_rows_from_metadata(payload: dict | None) -> int | None:
if not payload:
return None
for key in ("actual_num_records", "target_num_records"):
value = payload.get(key)
if isinstance(value, int):
return value
return None
def _safe_read_metadata_summary(payload: dict | None) -> dict | None:
if not payload:
return None
actual_num_records = (
payload.get("actual_num_records")
if isinstance(payload.get("actual_num_records"), int)
else None
)
target_num_records = (
payload.get("target_num_records")
if isinstance(payload.get("target_num_records"), int)
else actual_num_records
)
columns: list[str] | None = None
schema = payload.get("schema")
if isinstance(schema, dict):
columns = [str(key) for key in schema.keys()]
if not columns:
stats = payload.get("column_statistics")
if isinstance(stats, list):
derived = [
str(item.get("column_name"))
for item in stats
if isinstance(item, dict) and item.get("column_name")
]
columns = derived or None
parquet_files_count = None
file_paths = payload.get("file_paths")
if isinstance(file_paths, dict):
parquet_files = file_paths.get("parquet-files")
if isinstance(parquet_files, list):
parquet_files_count = len(parquet_files)
total_num_batches = (
payload.get("total_num_batches")
if isinstance(payload.get("total_num_batches"), int)
else parquet_files_count
)
num_completed_batches = (
payload.get("num_completed_batches")
if isinstance(payload.get("num_completed_batches"), int)
else total_num_batches
)
return {
"actual_num_records": actual_num_records,
"target_num_records": target_num_records,
"total_num_batches": total_num_batches,
"num_completed_batches": num_completed_batches,
"columns": columns,
}
def _build_local_dataset_items() -> list[LocalDatasetItem]:
if not LOCAL_DATASETS_ROOT.exists():
return []
items: list[LocalDatasetItem] = []
for entry in LOCAL_DATASETS_ROOT.iterdir():
if not entry.is_dir() or not entry.name.startswith("recipe_"):
continue
parquet_dir = entry / "parquet-files"
if not parquet_dir.exists() or not any(parquet_dir.glob("*.parquet")):
continue
rows = None
metadata_summary = None
metadata_path = entry / "metadata.json"
if metadata_path.exists():
metadata_payload = _safe_read_metadata(metadata_path)
rows = _safe_read_rows_from_metadata(metadata_payload)
metadata_summary = _safe_read_metadata_summary(metadata_payload)
try:
updated_at = entry.stat().st_mtime
except OSError:
updated_at = None
items.append(
LocalDatasetItem(
id=entry.name,
label=entry.name,
path=str(parquet_dir.resolve()),
rows=rows,
updated_at=updated_at,
metadata=metadata_summary,
)
)
items.sort(key=lambda item: item.updated_at or 0, reverse=True)
return items
def _load_local_preview_slice(*, dataset_path: Path, train_split: str, preview_size: int):
from datasets import load_dataset
if dataset_path.is_dir():
parquet_dir = dataset_path / "parquet-files" if (dataset_path / "parquet-files").exists() else dataset_path
parquet_files = sorted(parquet_dir.glob("*.parquet"))
if parquet_files:
dataset = load_dataset(
"parquet",
data_files=[str(path) for path in parquet_files],
split=train_split,
)
total_rows = len(dataset)
preview_slice = dataset.select(range(min(preview_size, total_rows)))
return preview_slice, total_rows
else:
candidate_files: list[Path] = []
for ext in LOCAL_FILE_EXTS:
candidate_files.extend(sorted(dataset_path.glob(f"*{ext}")))
if not candidate_files:
raise HTTPException(
status_code=400,
detail="Unsupported local dataset directory (expected parquet/json/jsonl/csv files)",
)
dataset_path = candidate_files[0]
if dataset_path.suffix in ['.json', '.jsonl']:
dataset = load_dataset('json', data_files=str(dataset_path), split=train_split)
elif dataset_path.suffix == '.csv':
dataset = load_dataset('csv', data_files=str(dataset_path), split=train_split)
elif dataset_path.suffix == '.parquet':
dataset = load_dataset('parquet', data_files=str(dataset_path), split=train_split)
else:
raise HTTPException(
status_code=400,
detail=f"Unsupported file format: {dataset_path.suffix}"
)
total_rows = len(dataset)
preview_slice = dataset.select(range(min(preview_size, total_rows)))
return preview_slice, total_rows
@router.get("/local", response_model=LocalDatasetsResponse)
def list_local_datasets(
current_subject: str = Depends(get_current_subject),
) -> LocalDatasetsResponse:
return LocalDatasetsResponse(datasets=_build_local_dataset_items())
@router.post("/check-format", response_model=CheckFormatResponse)
@ -116,19 +291,12 @@ def check_format(
if dataset_path.exists():
# ── Local file ──────────────────────────────────────────
if dataset_path.suffix in ['.json', '.jsonl']:
dataset = load_dataset('json', data_files=str(dataset_path), split=request.train_split)
elif dataset_path.suffix == '.csv':
dataset = load_dataset('csv', data_files=str(dataset_path), split=request.train_split)
elif dataset_path.suffix == '.parquet':
dataset = load_dataset('parquet', data_files=str(dataset_path), split=request.train_split)
else:
raise HTTPException(
status_code=400,
detail=f"Unsupported file format: {dataset_path.suffix}"
)
total_rows = len(dataset)
preview_slice = dataset.select(range(min(PREVIEW_SIZE, total_rows)))
train_split = request.train_split or "train"
preview_slice, total_rows = _load_local_preview_slice(
dataset_path=dataset_path,
train_split=train_split,
preview_size=PREVIEW_SIZE,
)
else:
# ── HuggingFace dataset ─────────────────────────────────
# Tier 1: list_repo_files → load only the first data file

View file

@ -51,6 +51,8 @@ from models.inference import (
ChoiceDelta,
CompletionChoice,
CompletionMessage,
ValidateModelRequest,
ValidateModelResponse,
)
from auth.authentication import get_current_subject
@ -277,6 +279,50 @@ async def load_model(
)
@router.post("/validate", response_model=ValidateModelResponse)
async def validate_model(
request: ValidateModelRequest,
current_subject: str = Depends(get_current_subject),
):
"""
Lightweight validation endpoint for model identifiers.
This checks that ModelConfig.from_identifier() can resolve the given
model_path, but it does NOT actually load model weights into GPU memory.
"""
try:
config = ModelConfig.from_identifier(
model_id=request.model_path,
hf_token=request.hf_token,
gguf_variant=request.gguf_variant,
)
if not config:
raise HTTPException(
status_code=400,
detail=f"Invalid model identifier: {request.model_path}",
)
return ValidateModelResponse(
valid=True,
message="Model identifier is valid.",
identifier=config.identifier,
display_name=getattr(config, "display_name", config.identifier),
is_gguf=getattr(config, "is_gguf", False),
is_lora=getattr(config, "is_lora", False),
is_vision=getattr(config, "is_vision", False),
)
except HTTPException:
raise
except Exception as e:
logger.error(f"Error validating model identifier '{request.model_path}': {e}", exc_info=True)
raise HTTPException(
status_code=400,
detail=f"Invalid model: {str(e)}",
)
@router.post("/unload", response_model=UnloadResponse)
async def unload_model(
request: UnloadRequest,

View file

@ -113,6 +113,7 @@ async def start_training(
# Validate dataset paths if provided
if request.local_datasets:
validated_datasets = []
missing_datasets = []
# Get the backend directory (where this file is located)
backend_dir = Path(__file__).parent.parent
assets_datasets_dir = backend_dir / "assets" / "datasets"
@ -133,12 +134,20 @@ async def start_training(
dataset_file = candidate
if not dataset_file.exists():
logger.warning(
f"Dataset file not found: {dataset_path} (resolved: {dataset_file})"
missing_datasets.append(
f"{dataset_path} (resolved: {dataset_file})"
)
else:
logger.info(f"Found dataset file: {dataset_file}")
continue
logger.info(f"Found dataset file: {dataset_file}")
validated_datasets.append(str(dataset_file))
if missing_datasets:
missing_detail = "; ".join(missing_datasets[:3])
raise HTTPException(
status_code=400,
detail=f"Local dataset not found: {missing_detail}",
)
request.local_datasets = validated_datasets
# Convert request to kwargs for backend

View file

@ -18,7 +18,6 @@
"@hugeicons/react": "^1.1.5",
"@huggingface/hub": "^2.9.0",
"@langchain/core": "^1.1.27",
"@langchain/textsplitters": "^1.0.1",
"@radix-ui/react-checkbox": "^1.3.3",
"@radix-ui/react-label": "^2.1.8",
"@radix-ui/react-select": "^2.2.6",
@ -389,8 +388,6 @@
"@langchain/core": ["@langchain/core@1.1.28", "", { "dependencies": { "@cfworker/json-schema": "^4.0.2", "ansi-styles": "^5.0.0", "camelcase": "6", "decamelize": "1.2.0", "js-tiktoken": "^1.0.12", "langsmith": ">=0.5.0 <1.0.0", "mustache": "^4.2.0", "p-queue": "^6.6.2", "uuid": "^10.0.0", "zod": "^3.25.76 || ^4" } }, "sha512-6FAGdezEp8zHY92LtnsAiv54KaG41nBdsuukk+R+1484edV20cVOyIc36ANuGKPx0pmYFCBWhCUdO0jxB/zn2Q=="],
"@langchain/textsplitters": ["@langchain/textsplitters@1.0.1", "", { "dependencies": { "js-tiktoken": "^1.0.12" }, "peerDependencies": { "@langchain/core": "^1.0.0" } }, "sha512-rheJlB01iVtrOUzttscutRgLybPH9qR79EyzBEbf1u97ljWyuxQfCwIWK+SjoQTM9O8M7GGLLRBSYE26Jmcoww=="],
"@mermaid-js/parser": ["@mermaid-js/parser@1.0.0", "", { "dependencies": { "langium": "^4.0.0" } }, "sha512-vvK0Hi/VWndxoh03Mmz6wa1KDriSPjS2XMZL/1l19HFwygiObEEoEwSDxOqyLzzAI6J2PU3261JjTMTO7x+BPw=="],
"@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.27.1", "", { "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-sr6GbP+4edBwFndLbM60gf07z0FQ79gaExpnsjMGePXqFcSSb7t6iscpjk9DhFhwd+mTEQrzNafGP8/iGGFYaA=="],

View file

@ -2,9 +2,9 @@
<html lang="en">
<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
<link rel="icon" type="image/png" href="/favicon.png" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>vite-app</title>
<title>Unsloth Studio</title>
</head>
<body>
<div id="root"></div>

View file

@ -26,7 +26,6 @@
"@hugeicons/react": "^1.1.5",
"@huggingface/hub": "^2.9.0",
"@langchain/core": "^1.1.27",
"@langchain/textsplitters": "^1.0.1",
"@radix-ui/react-checkbox": "^1.3.3",
"@radix-ui/react-label": "^2.1.8",
"@radix-ui/react-select": "^2.2.6",

Binary file not shown.

After

Width:  |  Height:  |  Size: 13 KiB

View file

@ -10,6 +10,7 @@ import type {
OpenAIChatChunk,
OpenAIChatCompletionsRequest,
UnloadModelRequest,
ValidateModelResponse,
} from "../types/api";
function parseErrorText(status: number, body: unknown): string {
@ -67,6 +68,21 @@ export async function loadModel(
return parseJsonOrThrow<LoadModelResponse>(response);
}
export async function validateModel(
payload: LoadModelRequest,
): Promise<ValidateModelResponse> {
const response = await authFetch("/api/inference/validate", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
model_path: payload.model_path,
hf_token: payload.hf_token,
gguf_variant: payload.gguf_variant ?? null,
}),
});
return parseJsonOrThrow<ValidateModelResponse>(response);
}
export async function unloadModel(payload: UnloadModelRequest): Promise<void> {
const response = await authFetch("/api/inference/unload", {
method: "POST",

View file

@ -6,6 +6,7 @@ import {
listModels,
loadModel,
unloadModel,
validateModel,
} from "../api/chat-api";
import { useChatRuntimeStore } from "../stores/chat-runtime-store";
import type { LoadModelResponse } from "../types/api";
@ -192,6 +193,17 @@ export function useChatModelRuntime() {
const displayName = model?.name || lora?.name || modelId;
const currentCheckpoint =
useChatRuntimeStore.getState().params.checkpoint;
const previousCheckpoint = currentCheckpoint;
const previousVariant =
useChatRuntimeStore.getState().activeGgufVariant ?? null;
const previousModel = previousCheckpoint
? models.find((entry) => entry.id === previousCheckpoint)
: undefined;
const previousLora = previousCheckpoint
? loras.find((entry) => entry.id === previousCheckpoint)
: undefined;
const previousIsLora =
previousModel?.isLora ?? (previousLora ? true : false);
const loadingDescription = [
currentCheckpoint ? "Unloading previous model first." : null,
extraLoadingDescription ?? null,
@ -204,26 +216,61 @@ export function useChatModelRuntime() {
setLoadingModel({ id: modelId, displayName });
try {
async function performLoad(): Promise<void> {
let previousWasUnloaded = false;
const currentCheckpoint =
useChatRuntimeStore.getState().params.checkpoint;
if (currentCheckpoint) {
await unloadModel({ model_path: currentCheckpoint });
try {
// Lightweight pre-flight validation: avoid unloading a working model
// if the new identifier is clearly invalid (e.g. bad HF id / path).
await validateModel({
model_path: modelId,
hf_token: null,
max_seq_length: DEFAULT_MODEL_MAX_SEQ_LENGTH,
load_in_4bit: true,
is_lora: isLora,
gguf_variant: ggufVariant ?? null,
});
if (currentCheckpoint) {
await unloadModel({ model_path: currentCheckpoint });
previousWasUnloaded = true;
}
const paramsBeforeLoad = useChatRuntimeStore.getState().params;
const loadResponse = await loadModel({
model_path: modelId,
hf_token: null,
max_seq_length: DEFAULT_MODEL_MAX_SEQ_LENGTH,
load_in_4bit: true,
is_lora: isLora,
gguf_variant: ggufVariant ?? null,
trust_remote_code: paramsBeforeLoad.trustRemoteCode ?? false,
});
const currentParams = useChatRuntimeStore.getState().params;
setParams(
mergeRecommendedInference(currentParams, loadResponse, modelId),
);
await refresh();
} catch (error) {
// If we unloaded a previous model and the new load failed, attempt a rollback.
if (previousWasUnloaded && previousCheckpoint) {
try {
await loadModel({
model_path: previousCheckpoint,
hf_token: null,
max_seq_length: DEFAULT_MODEL_MAX_SEQ_LENGTH,
load_in_4bit: true,
is_lora: previousIsLora,
gguf_variant: previousVariant,
});
await refresh();
} catch {
// If rollback also fails, surface the original error.
}
}
throw error;
}
const currentParams = useChatRuntimeStore.getState().params;
const loadResponse = await loadModel({
model_path: modelId,
hf_token: null,
max_seq_length: DEFAULT_MODEL_MAX_SEQ_LENGTH,
load_in_4bit: true,
is_lora: isLora,
gguf_variant: ggufVariant ?? null,
trust_remote_code: currentParams.trustRemoteCode ?? false,
});
const paramsAfterLoad = useChatRuntimeStore.getState().params;
setParams(mergeRecommendedInference(paramsAfterLoad, loadResponse, modelId));
await refresh();
}
const loadPromise = performLoad().finally(() => {

View file

@ -38,6 +38,16 @@ export interface LoadModelRequest {
trust_remote_code?: boolean;
}
export interface ValidateModelResponse {
valid: boolean;
message: string;
identifier?: string | null;
display_name?: string | null;
is_gguf?: boolean;
is_lora?: boolean;
is_vision?: boolean;
}
export interface GgufVariantDetail {
filename: string;
quant: string;

View file

@ -12,7 +12,10 @@ const instructionFromAnswerUrl = new URL(
).href;
const textToPythonUrl = new URL("./text-to-python.json", import.meta.url).href;
const textToSqlUrl = new URL("./text-to-sql.json", import.meta.url).href;
const conversationUrl = new URL("./conversation.json", import.meta.url).href;
const ocrDocumentExtractionUrl = new URL(
"./ocr-document-extraction.json",
import.meta.url,
).href;
function isRecord(value: unknown): value is Record<string, unknown> {
return !!value && typeof value === "object" && !Array.isArray(value);
@ -125,10 +128,10 @@ export const LEARNING_RECIPES: LearningRecipeDef[] = [
loadPayload: () => loadPayloadFromUrl(textToSqlUrl),
},
{
id: "conversation",
title: "Multi-Turn Chat",
id: "ocr-document-extraction",
title: "OCR Document Extraction",
description:
"Generate realistic user-assistant conversations with structured message output.",
loadPayload: () => loadPayloadFromUrl(conversationUrl),
"Use image context to generate OCR-style document extraction output.",
loadPayload: () => loadPayloadFromUrl(ocrDocumentExtractionUrl),
},
];

View file

@ -3,7 +3,7 @@
"model_providers": [
{
"name": "openai_provider",
"endpoint": "https://openrouter.ai/api/v1",
"endpoint": "",
"provider_type": "openai",
"extra_headers": {},
"extra_body": {}
@ -13,19 +13,18 @@
"model_configs": [
{
"alias": "ministral",
"model": "mistralai/ministral-8b-2512",
"model": "",
"provider": "openai_provider",
"inference_parameters": {
"temperature": 0.7,
"max_tokens": 1024
"max_tokens": 2048
}
}
],
"seed_config": {
"source": {
"seed_type": "hf",
"path": "datasets/unsloth/alpaca-cleaned/**/*.json",
"token": null,
"path": "unsloth/alpaca-cleaned",
"endpoint": "https://huggingface.co"
},
"sampling_strategy": "ordered",
@ -42,10 +41,20 @@
"drop": false,
"model_alias": "ministral",
"prompt": "Based on this target answer:\n{{ output }}\n\nWrite one high-quality plain text short and brief user instruction that this answer would satisfy.\nReturn only the instruction.",
"with_trace": "none"
"with_trace": "none",
"extract_reasoning_content": false
}
],
"processors": []
"processors": [
{
"processor_type": "drop_columns",
"name": "drop_seed_columns",
"column_names": [
"input",
"instruction"
]
}
]
},
"run": {
"rows": 5,
@ -63,7 +72,7 @@
"width": 400,
"node_type": "markdown_note",
"name": "note_1",
"markdown": "#### Hugginface seed block\nThis recipe uses a ** HuggingFace dataset ** as seed data.\nYou provide dataset identity, load columns, then generate new fields from seed columns.\n\n##### Setup:\n\n1. Paste dataset id as `org/repo` (example: `unsloth/alpaca-cleaned`)\n2. Add token only if dataset is gated/private\n3. Load columns + preview rows so variables are available in prompts\n\n##### Why this matters:\n- Seed columns can drive generation quality\n- You can reference seed values directly in prompts (for example `{{ output }}`)",
"markdown": "#### Hugging Face seed block\nThis recipe uses a ** HuggingFace dataset ** as seed data.\nYou select a Hugging Face dataset, load columns, then generate new fields from seed columns. Each column in the Hugging Face dataset becomes a valid variable that you can reference eg. `{{ topic }}`\n\n##### Setup:\n\n1. Search for a dataset and select one in the dropdown (example: `unsloth/alpaca-cleaned`)\n2. Add token only if dataset is gated/private\n3. Load columns + preview rows so variables are available in prompts\n\n##### Why this matters:\n- Seed columns can drive generation quality\n- You can reference seed values directly in prompts (for example `{{ output }}`)",
"note_color": "#DCFCE7",
"note_opacity": "35"
},
@ -74,7 +83,7 @@
"width": 400,
"node_type": "markdown_note",
"name": "note_2",
"markdown": "##### Drop columns behavior:\n\n- You can mark specific seed columns to **drop from final output**\n- Those columns are still used during generation\n- They are removed only from exported final dataset\n\n##### Example:\n- Keep `generated_instruction` from llm-text block\n- Drop original `instruction`, `input`, `output` from the hugginface dataset from final artifact\n- Result: clean training output while still using source columns as generation context\n",
"markdown": "##### Drop columns behavior:\n\n- You can mark specific seed columns to **drop from final output**\n- Those columns are still used during generation\n- They are removed only from exported final dataset\n\n##### Example:\n- Keep `generated_instruction` from llm-text block\n- Drop original `instruction`, `input`, `output` from the hugginface dataset from final dataset\n- Result: clean training output while still using source columns as generation context\n",
"note_color": "#DCFCE7",
"note_opacity": "35"
},
@ -128,64 +137,9 @@
],
"layout_direction": "LR",
"seed_source_type": "hf",
"seed_columns": [
"instruction",
"input",
"output"
],
"seed_columns": [],
"seed_drop_columns": [],
"seed_preview_rows": [
{
"instruction": "Give three tips for staying healthy.",
"input": "",
"output": "1. Eat a balanced and nutritious diet: Make sure your meals are inclusive of a variety of fruits and vegetables, lean protein, whole grains, and healthy fats. This helps to provide your body with the essential nutrients to function at its best and can help prevent chronic diseases.\n\n2. Engage in regular physical activity: Exercise is crucial for maintaining strong bones, muscles, and cardiovascular health. Aim for at least 150 minutes of moderate aerobic exercise or 75 minutes of vigorous exercise each week.\n\n3. Get enough sleep: Getting enough quality sleep is crucial for physical and mental well-being. It helps to regulate mood, improve cognitive function, and supports healthy growth and immune function. Aim for 7-9 hours of sleep each night."
},
{
"instruction": "What are the three primary colors?",
"input": "",
"output": "The three primary colors are red, blue, and yellow. These colors are called primary because they cannot be created by mixing other colors and all other colors can be made by combining them in various proportions. In the additive color system, used for light, the primary colors are red, green, and blue (RGB)."
},
{
"instruction": "Describe the structure of an atom.",
"input": "",
"output": "An atom is the basic building block of all matter and is made up of three types of particles: protons, neutrons, and electrons. The structure of an atom can be described as a nucleus at the center surrounded by a cloud of electrons.\n\nThe nucleus of an atom is made up of protons and neutrons. Protons are positively charged particles and neutrons are neutral particles with no charge. Both of these particles are located in the nucleus of the atom, which is at the center of the atom and contains most of the atom's mass.\n\nSurrounding the nucleus of the atom is a cloud of electrons. Electrons are negatively charged particles that are in constant motion around the nucleus. The electron cloud is divided into shells or orbitals, and each shell can hold a certain number of electrons. The number of electrons in the outermost shell, called the valence shell, determines the chemical properties of the atom. \n\nIn a neutral atom, the number of protons in the nucleus is equal to the number of electrons in the electron cloud, so the positive and negative charges balance out and the atom has no overall charge. The number of protons, also called the atomic number, determines what element the atom is."
},
{
"instruction": "How can we reduce air pollution?",
"input": "",
"output": "There are several ways to reduce air pollution, including:\n\n1. Reduce energy consumption: By conserving energy, we reduce the amount of pollution emitted from power plants. Some ways to achieve this include: using energy-efficient appliances and lighting, insulating homes, and shutting off lights and electronics when not in use.\n\n2. Use public transportation or carpool: Cars and trucks are major sources of pollution. By using public transportation, carpooling or biking, you can reduce the number of cars on the road, which in turn reduces emissions.\n\n3. Be responsible with waste: When waste is burned or decomposed, they can release harmful gases into the air. Dispose of hazardous waste correctly and recycle materials like plastics, paper, and glass when possible.\n\n4. Support clean energy sources: Renewable energy sources like solar and wind power generate little or no pollution. Support clean energy companies and projects, and consider installing renewable energy systems in your home.\n\n5. Drive efficiently: If you need to drive, make sure your vehicle is well maintained and drive efficiently by avoiding rapid acceleration and braking, and reducing idling.\n\n6. Be mindful of product choices: Some products and practices release more pollution than others. Choose products with eco-friendly packaging and try to minimize your use of aerosol sprays, which can release harmful chemicals into the air."
},
{
"instruction": "Pretend you are a project manager of a construction company. Describe a time when you had to make a difficult decision.",
"input": "",
"output": "I had to make a difficult decision when I was working as a project manager at a construction company. I was in charge of a project that needed to be completed by a certain date in order to meet the clients expectations. However, due to unexpected delays, we were not able to meet the deadline and so I had to make a difficult decision. I decided to extend the deadline, but I had to stretch the teams resources even further and increase the budget. Although it was a risky decision, I ultimately decided to go ahead with it to ensure that the project was completed on time and that the clients expectations were met. The project was eventually successfully completed and this was seen as a testament to my leadership and decision-making abilities."
},
{
"instruction": "Write a concise summary of the following:\n\"Commodore 64 (commonly known as the C64 or CBM 64) was manufactured by Commodore Business Machine (CBM) in August 1982 with a starting price of $595. It was an 8-bit home computer with remarkable market success. Between 1983-1986, C64 sales amounted to about 17 million units sold, making them the best-selling single personal computer model of all time in 1983-1986. \n\nAdditionally, the Commodore 64 dominated the market with between 30% and 40% share and 2 million units sold per year, outselling the IBM PC clones, Apple Computers, and Atari computers. Adding to their success, Sam Tramiel (former Atari president), during an interview in 1989, said they were building 400,000 C64s a month for a couple of years. \"",
"input": "",
"output": "The Commodore 64 was a highly successful 8-bit home computer manufactured by Commodore Business Machine (CBM) in 1982, with sales amounting to approximately 17 million units sold between 1983-1986. It dominated the market with between 30% and 40% share and outsold its competitors, including IBM PC clones, Apple Computers, and Atari computers. At its peak, CBM was building 400,000 C64s a month for a couple of years."
},
{
"instruction": "Explain why the following fraction is equivalent to 1/4",
"input": "4/16",
"output": "The fraction 4/16 is equivalent to 1/4 because both fractions represent the same value. A fraction can be simplified by dividing both the numerator and the denominator by a common factor. In this case, 4 is a common factor of both the numerator and the denominator of 4/16. When we divide both by 4, we get 4/4 = 1 and 16/4 = 4, so the simplified fraction is 1/4. Alternatively, we can think of this in terms of multiplication. For example, if we multiply the numerator and denominator of the fraction 1/4 by 4, we get (1x4)/(4x4), or 4/16. Since both fractions can be derived from the other through multiplication or division by the same number, they represent the same value and are equivalent."
},
{
"instruction": "Write a short story in third person narration about a protagonist who has to make an important career decision.",
"input": "",
"output": "Sophie sat at her desk, staring blankly at the computer screen. Her mind was racing as she weighed the options in front of her. She had been offered a promotion to a senior management position within her company, but it meant moving across the country and leaving her friends and family behind. At the same time, her dream job at a rival company had finally opened up, and she had been invited for an interview. \n\nThe thought of leaving her comfortable life behind, to take a chance on something new and uncertain, made Sophie feel anxious, but also excited. She had always been one to take risks, and it had paid off in the past. But now, with such a big decision to make, Sophie couldn't help but feel overwhelmed.\n\nAfter much deliberation, Sophie made her choice. She packed her bags, said goodbye to her loved ones, and moved across the country for the new job. It was a difficult first few months, adjusting to a new city and a new company culture. But as time went by, Sophie realized that she had made the best decision for her career. The job was challenging, but also rewarding, and she was surrounded by passionate, driven colleagues who inspired her to grow and develop.\n\nIn the end, Sophie's bold move paid off. She was thriving in her new role, and had made new friends and connections in her new city. Looking back, Sophie was grateful for the opportunity to take a risk and make a change, and was confident that she had made the right decision for her career."
},
{
"instruction": "Evaluate this sentence for spelling and grammar mistakes",
"input": "He finnished his meal and left the resturant",
"output": "There are two spelling errors in the sentence. The corrected sentence should be: \"He finished his meal and left the restaurant.\""
},
{
"instruction": "How did Julius Caesar die?",
"input": "",
"output": "Julius Caesar, the Roman Military general, and statesman was assassinated on March 15, 44 BC. He was stabbed to death at a meeting of the Senate in the Theatre of Pompey by a group of senators who called themselves the Liberatores. Led by Gaius Cassius Longinus, and Marcus Junius Brutus, they believed that Caesar's dictatorship was subverting the Roman Republic and sought to restore the traditional republican government. The exact number of assailants is not known, but it is believed that a group of as many as 60 conspirators participated in the assassination, and each of them took turns in stabbing Caesar. The day of his death is still known as the Ides of March."
}
],
"seed_preview_rows": [],
"local_file_name": "",
"unstructured_file_name": "",
"unstructured_chunk_size": "1200",

View file

@ -0,0 +1,148 @@
{
"recipe": {
"model_providers": [
{
"name": "provider_1",
"endpoint": "https://openrouter.ai/api/v1",
"provider_type": "openai",
"extra_headers": {},
"extra_body": {}
}
],
"mcp_providers": [],
"model_configs": [
{
"alias": "provider_column",
"model": "google/gemini-2.0-flash-001",
"provider": "provider_1",
"inference_parameters": {
"temperature": 0.2,
"max_tokens": 4096
}
}
],
"seed_config": {
"source": {
"seed_type": "hf",
"path": "datasets/ylecun/mnist/mnist/**/*.parquet"
},
"sampling_strategy": "ordered",
"selection_strategy": null
},
"tool_configs": [],
"columns": [
{
"column_type": "llm-text",
"name": "ocr_text",
"drop": false,
"model_alias": "provider_column",
"prompt": "Transcribe all text from this document image.",
"multi_modal_context": [
{
"modality": "image",
"column_name": "image"
}
]
}
],
"processors": []
},
"run": {
"rows": 5,
"preview": true,
"output_formats": ["jsonl"]
},
"ui": {
"nodes": [
{
"id": "note_1",
"x": -180,
"y": 43,
"width": 400,
"node_type": "markdown_note",
"name": "note_1",
"markdown": "This recipe uses **Gemini 2.0 Flash** via OpenRouter to transcribe document images into clean text.\n\nThe Seed block is prefilled with `ylecun/mnist` so you can run immediately. You can swap to any Hugging Face dataset that includes an `image` column.\n\nOutput: `ocr_text` column with the raw transcribed text per image.",
"note_color": "#DCFCE7",
"note_opacity": "35"
},
{
"id": "note_2",
"x": 283,
"y": -333,
"width": 400,
"node_type": "markdown_note",
"name": "note_2",
"markdown": "##### Setup\n\nAdd your OpenRouter API key to the **Model Provider** block — same as every other recipe.\n\nGemini 2.0 Flash is well-suited for OCR: fast, cheap, and strong on tables, receipts, forms, and multi-column layouts.\n\nWant a purpose-built OCR model? Swap the endpoint to a local vLLM server running `lightonai/LightOnOCR-2-1B` for maximum throughput.",
"note_color": "#DCFCE7",
"note_opacity": "35"
},
{
"id": "note_3",
"x": 303,
"y": 299,
"width": 400,
"node_type": "markdown_note",
"name": "note_3",
"markdown": "##### Seed: HF dataset with image column\n\nThis template starts with `ylecun/mnist` so first run works without seed setup.\n\nTo use your own data: open Seed → keep **HF dataset** selected → choose a dataset that contains an `image` column → click **Load**.\n\nThen open the LLM Text block and set **Image Context** to the `image` column so each row image is sent with the prompt.\n\nTip: datasets with embedded image columns are more reliable than URL-only image fields.",
"note_color": "#DCFCE7",
"note_opacity": "35"
},
{
"id": "seed",
"x": 295,
"y": 108,
"width": 400
},
{
"id": "provider_1",
"x": 960,
"y": -465,
"width": 400
},
{
"id": "provider_column",
"x": 959,
"y": -180,
"width": 400
},
{
"id": "ocr_text",
"x": 960,
"y": 108,
"width": 400
}
],
"edges": [
{
"from": "seed",
"to": "ocr_text",
"type": "canvas",
"source_handle": "data-out",
"target_handle": "data-in"
},
{
"from": "provider_1",
"to": "provider_column",
"type": "semantic",
"source_handle": "semantic-out-bottom",
"target_handle": "semantic-in-top"
},
{
"from": "provider_column",
"to": "ocr_text",
"type": "semantic",
"source_handle": "semantic-out-bottom",
"target_handle": "data-in-top"
}
],
"layout_direction": "LR",
"seed_source_type": "hf",
"seed_columns": [],
"seed_drop_columns": [],
"seed_preview_rows": [],
"local_file_name": "",
"unstructured_file_name": "",
"unstructured_chunk_size": "900",
"unstructured_chunk_overlap": "150"
}
}

View file

@ -3,7 +3,7 @@
"model_providers": [
{
"name": "provider_1",
"endpoint": "https://openrouter.ai/api/v1",
"endpoint": "",
"provider_type": "openai",
"extra_headers": {},
"extra_body": {}
@ -13,18 +13,19 @@
"model_configs": [
{
"alias": "provider_column",
"model": "mistralai/ministral-8b-2512",
"model": "",
"provider": "provider_1",
"inference_parameters": {
"temperature": 0.7,
"max_tokens": 256
"temperature": 0.7
}
}
],
"seed_config": {
"source": {
"seed_type": "local",
"path": "/home/wasim/.cache/unsloth/data-recipe/seed-uploads/dc8ef7c169ed43399b7d37164495150f_50 page sample PDF.indd.jsonl"
"seed_type": "unstructured",
"path": "",
"chunk_size": 1200,
"chunk_overlap": 200
},
"sampling_strategy": "ordered",
"selection_strategy": null
@ -37,6 +38,8 @@
"drop": false,
"model_alias": "provider_column",
"prompt": "Given ONLY this chunk: {{ chunk_text }} generate one answerable question, answer, and exact supporting quote from chunk. If not answerable, skip.",
"with_trace": "none",
"extract_reasoning_content": false,
"output_format": {
"type": "object",
"additionalProperties": false,
@ -72,8 +75,8 @@
"nodes": [
{
"id": "note_1",
"x": -180.01113025994076,
"y": 43.27382247773167,
"x": 474.6120044693708,
"y": 1229.5810476890458,
"width": 400,
"node_type": "markdown_note",
"name": "note_1",
@ -83,8 +86,8 @@
},
{
"id": "note_2",
"x": 283.8131688769869,
"y": -333.10847089567505,
"x": 26.758540311329455,
"y": 963.3465578835235,
"width": 400,
"node_type": "markdown_note",
"name": "note_2",
@ -94,8 +97,8 @@
},
{
"id": "note_3",
"x": 303.52241671566657,
"y": 299.62272507131615,
"x": 473.62551435180245,
"y": 741.5352258256931,
"width": 400,
"node_type": "markdown_note",
"name": "note_3",
@ -105,37 +108,30 @@
},
{
"id": "seed",
"x": 295.56977201312833,
"y": 108.19964868337735,
"x": 484.36210245413577,
"y": 1059.99180558796,
"width": 400
},
{
"id": "provider_1",
"x": 960.0115722892822,
"y": -465.06410256410254,
"x": 960,
"y": 622,
"width": 400
},
{
"id": "provider_column",
"x": 959.90231990232,
"y": -180.56654456654456,
"x": 960,
"y": 816,
"width": 400
},
{
"id": "llm_structured_1",
"x": 960,
"y": 108.25,
"y": 1077,
"width": 400
}
],
"edges": [
{
"from": "seed",
"to": "llm_structured_1",
"type": "canvas",
"source_handle": "data-out",
"target_handle": "data-in"
},
{
"from": "provider_1",
"to": "provider_column",
@ -149,49 +145,23 @@
"type": "semantic",
"source_handle": "semantic-out-bottom",
"target_handle": "data-in-top"
},
{
"from": "llm_structured_1",
"to": "seed",
"type": "canvas",
"source_handle": "data-out-left",
"target_handle": "data-in-right"
}
],
"layout_direction": "LR",
"seed_source_type": "unstructured",
"seed_columns": [
"chunk_text"
],
"seed_columns": [],
"seed_drop_columns": [],
"seed_preview_rows": [
{
"chunk_text": "[Citation Needed] The Best of Wikipedias Worst Writing Conor Lastowka and Josh Fruhlinger Boring Legal Fine Print Each entry in this book contains material from Wikipedia, although the text we use may not represent the current version of any article. The URL at the bottom of each page will direct you to the source Wikipedia article; use the articles History tab to find a list of contributors. All material in this book that is taken from Wikipedia is licensed under the Creative Commons-Attribution Share Alike 3.0 license. Heres a quick human-readable summary of your rights to use this content: You are free: to Share—to copy, distribute and transmit the work, and to Remix—to adapt the work Under the following conditions: Attribution—You must attribute the work in the manner specified by the author or licensor (but not in any way that suggests that they endorse you or your use of the work.) Share Alike—If you alter, transform, or build upon this work, you may distribute the resulting work only under the same, similar or a compatible license. With the understanding that: Waiver—Any of the above conditions can be waived if you get permission from the copyright holder. Other Rights—In"
},
{
"chunk_text": "only under the same, similar or a compatible license. With the understanding that: Waiver—Any of the above conditions can be waived if you get permission from the copyright holder. Other Rights—In no way are any of the following rights affected by the license: your fair dealing or fair use rights; the authors moral rights; and rights other persons may have either in the work itself or in how the work is used, such as publicity or privacy rights. Notice—For any reuse or distribution, you must make clear to others the license terms of this work. The best way to do this is with a link to: http://creativecommons.org/licenses/by-sa/3.0/ Italicized material beneath each Wikipedia entry is © 2011 Conor Lastowka and Josh Fruhlinger. Copy-edited by Lauren Lastowka Cover design by Jaime Robinson ISBN # 978-1466346987 This book is dedicated to every person who wrote an entry that appears in it. May your citations always be needed. 6 Introduction Wikipedia. Whether youve used it to settle an argument, plagiarized a history report from it, or simply replaced the entire text of the biography of a respected humanitarian with the single word “dogballs,” its an inescapable part of the Internet"
},
{
"chunk_text": "plagiarized a history report from it, or simply replaced the entire text of the biography of a respected humanitarian with the single word “dogballs,” its an inescapable part of the Internet experience. Since its launch in 2001, it has rapidly risen to become the seventh most popular website, with over 365 million readers (Source: Wikipedia). If youre like us, when you want to know the name of the kangaroo on Shirt Tales or just want to confirm that Mother Teresa was a dogballs who helped the farts (Source: Wikipedia), The Encyclopedia That Anyone Can Edit will probably be the first place you check. But heres the thing about letting anybody edit your encyclopedia: it means that anybody can edit your encyclopedia. And while in theory this means that one day Stephen Hawking might decide to weigh in on the entry for string theory, in reality it means that somebody who deeply cares about pro wrestling is going to call someone else a Nazi when they revert his edits about Wrestlemania XI on Razor Ramons page. And so we arrive at a cosmic intersection, where an obscure topic of dubious relevance is written about by the type of weirdo who logs on to Wikipedia to write about obscure"
},
{
"chunk_text": "XI on Razor Ramons page. And so we arrive at a cosmic intersection, where an obscure topic of dubious relevance is written about by the type of weirdo who logs on to Wikipedia to write about obscure topics of dubious relevance. Were these authors re-watching their video of Wrestlemania XI instead of completing basic 8th grade English assignments? Its very likely. Does this 7 stop them from attempting to emulate the academic tone of the great encyclopedias of the past as they describe a large mammalian species from the Star Wars universe that shares a common ancestor with the Wookies? It does not. The result? Some really terrible Wikipedia writing. For the past two years, we have collected this writing on our blog, [Citation Needed]. Fascinated and delighted by the brilliantly bad writing we encountered in our Wikipedia browsing, we set out to curate The Best of Wikipedias Worst Writing. Starting the blog was a no-brainer; our only concern was whether, after a few months of our daily mining, the well of awful Wikipedia writing would eventually run dry. By the time you read this, we will have published our thousandth entry. We started a podcast. Instead of drying up, the ocean of"
},
{
"chunk_text": "mining, the well of awful Wikipedia writing would eventually run dry. By the time you read this, we will have published our thousandth entry. We started a podcast. Instead of drying up, the ocean of ineptitude has proven far more vast than we ever could have imagined. Through our own browsing, and with the help of a dedicated group of readers who are exploring the topics they submit for God knows what reason, weve continually lowered and re-lowered the bar for bad Wikipedia writing. Now, lets get one thing straight: we love each and every entry written in this book. If you are one of the authors who have chosen to use your valuable time on this planet to write straight-faced exegeses on the subject of forgotten action figures from the seventies, we hope you dont take offense. And if you do, we have an acceptable retort prepared for you: “You guys ran a blog about Wikipedia for two years, who the hell are you to talk?” Feel free to use it! Others may criticize us for not doing our part to help Wikipedia become “better” by revising these passages. Nothing that does not involve electrodes near our genitals would make us more miserable. In our opinion, many of the passages in this"
},
{
"chunk_text": "to help Wikipedia become “better” by revising these passages. Nothing that does not involve electrodes near our genitals would make us more miserable. In our opinion, many of the passages in this book stand alone as works of art. Think of us as photographers preserving the memory of the great street art of the world before the joyless police come and whitewash over it. (Is that an official police responsibility? It seems beneath them. If its not, but theyre still forced to do it, that might explain the joylessness.) The point is, if youre moved to correct these entries, were powerless to stop you. Theyve already given us joy, and were just happy to have encountered them. Enough introduction. Here are over two hundred of our favorite bad Wikipedia articles of all time. Comments in italics are ours. Everything else is a faithful reproduction of the way the entry stood at the moment we or our informants encountered it. We hope you will laugh, cry, maybe even learn something, and always remember to dogballs. —Conor Lastowka & Josh Fruhlinger citationneeded.tumblr.com 8 9 In barely one decade, Jimmy Wales has succeeded in establishing a worldwide network of knowledge. Wikipedia,"
},
{
"chunk_text": "remember to dogballs. —Conor Lastowka & Josh Fruhlinger citationneeded.tumblr.com 8 9 In barely one decade, Jimmy Wales has succeeded in establishing a worldwide network of knowledge. Wikipedia, his online encyclopaedia, accessible on the Internet for free, has become a symbol of a radical change in the media economy. Moreover, it revolutionized the access to knowledge as mans most important resource and thus contributed to democratizing knowledge. The Gottlieb Duttweiler Institute, awarding the 2011 Gottlieb Duttweiler Prize to Wikipedia founder Jimmy Wales I saw the Beavis and Butt-Head episode that had Hogans “Real American” music on there. I dont quite remembering it being critiqued by Beavis and Butt-Head. They sounded more like they liked the music and I dont really remember any criticism of it (except for when it was going, when Butt-Head said “homework sucks”, but Im not quite sure if he was referring to music or not). Wikipedia discussion page for Hulk Hogan 10 11 http://en.wikipedia.org/wiki/Polybius_(video_game) Want a Citation for this one? Please see the following 206 pages. The Roach story contained a number of inconsistencies: some of it seems to be directly"
},
{
"chunk_text": "Want a Citation for this one? Please see the following 206 pages. The Roach story contained a number of inconsistencies: some of it seems to be directly sourced from Wikipedia- all in all, an entirely untrustworthy source. Polybius (video game) 12 http://en.wikipedia.org/wiki/General_Mills_monster-themed_breakfast_cereals You can imagine the marketing team having their first meeting after the cereals release. “We have good news and bad news. The good news is, your latest cereal is very, very popular. The bad news is, its not in any way due to the character you came up with, the box design you slaved over, the costly ad campaign, or the hours you put in coming up with free toy ideas. Gentlemen, you should probably sit down....” Franken Berry was very popular when first introduced possibly because the initial batches of the cereal used a dye that didnt break down in the body, causing many childrens feces to be bright pink, a symptom sometimes referred to as “Frankenberry Stool.” General Mills monster-themed breakfast cereals 13 http://en.wikipedia.org/wiki/Skiffle Because if your encyclopedia cant provide you with an unsourced claim that it admits is only one of several"
},
{
"chunk_text": "General Mills monster-themed breakfast cereals 13 http://en.wikipedia.org/wiki/Skiffle Because if your encyclopedia cant provide you with an unsourced claim that it admits is only one of several theories put forth about the subject, and then go on to inform you in the very same sentence that other unidentified parties disagree with that claim, then what the hell good is it? Skiffle is often said to have developed from New Orleans jazz, but this has been disputed. Skiffle 14 http://en.wikipedia.org/wiki/House_Party_%28film%29 “HP4: Coda or Mistake?” was by far the most contentious panel at HoPaCon 2009, with impassioned arguments echoing through the halls of the Kansas City International Airport Days Inn. In 2001, Immature (now going by IMx) starred in a direct-to-video sequel, House Party 4: Down to the Last Minute, which does not feature Kid or Play. The film is not considered a part of the House Party canon amongst fans. [citation needed] House Party (film) 15 http://en.wikipedia.org/wiki/Inglewood,_California So it would appear that, due to the presence of the anti- drug organization D.A.R.E. in Inglewood, there are in fact parts of Inglewood that are attempting to do good."
},
{
"chunk_text": "So it would appear that, due to the presence of the anti- drug organization D.A.R.E. in Inglewood, there are in fact parts of Inglewood that are attempting to do good. Thus the claim that Inglewood is “always” up to no good can be assumed to be false, or at the very least a gross exaggeration. Also, many have speculated that the so-called “Doctor” Dre never actually received a PhD. D.A.R.E. America has its headquarters in Inglewood. Despite this, in the 1996 rap hit “California Love”, Dr. Dre remarks that Inglewood is “always up to no good”. Inglewood, California 16 http://en.wikipedia.org/wiki/Tyler_Perry People who feel that Tyler Perrys 65” stature is severely diminished by his wearing a wig? The line in the sand has been drawn. Another comical aspect is provided by Perrys 6- 5” stature, which is in no way diminished by his wearing a wig. Tyler Perry 17 http://en.wikipedia.org/wiki/Bondage_bed Do not attempt to affix your bondage partner to this question mark using chains and shackles! It is purely metaphorical! It is possible to buy inflatable bondage beds; however, a question mark must remain over how effective they are. Bondage bed 18"
}
],
"seed_preview_rows": [],
"local_file_name": "",
"unstructured_file_name": "50 page sample PDF.indd.pdf",
"unstructured_file_name": "",
"unstructured_chunk_size": "1200",
"unstructured_chunk_overlap": "200"
}
}
}

View file

@ -3,7 +3,7 @@
"model_providers": [
{
"name": "provider_column",
"endpoint": "https://openrouter.ai/api/v1",
"endpoint": "",
"provider_type": "openai",
"extra_headers": {},
"extra_body": {}
@ -13,11 +13,10 @@
"model_configs": [
{
"alias": "ministral",
"model": "mistralai/ministral-8b-2512",
"model": "",
"provider": "provider_column",
"inference_parameters": {
"temperature": 0.7,
"max_tokens": 256
"temperature": 0.7
}
}
],
@ -76,6 +75,8 @@
"drop": false,
"model_alias": "ministral",
"prompt": "Create a realistic support ticket from {{ user_full_name }} using the {{ platform }} platform. Impact scope is {{ impact_scope }}.\n",
"with_trace": "none",
"extract_reasoning_content": false,
"output_format": {
"type": "object",
"additionalProperties": false,
@ -129,6 +130,8 @@
"drop": false,
"model_alias": "ministral",
"prompt": "Write a concise support reply for ticket '{{ ticket.issue_title }}'. Category: {{ ticket.category }}. Priority: {{ ticket.priority }}. SLA target: {{ sla_target }}. {% if ticket.priority == 'P1' %}Tone must be urgent and action-first.{% else %}Tone must be calm and instructional.{% endif %}",
"with_trace": "none",
"extract_reasoning_content": false,
"output_format": {
"type": "object",
"additionalProperties": false,
@ -168,8 +171,8 @@
"nodes": [
{
"id": "note_1",
"x": 1084.767431711644,
"y": -293.4482850247655,
"x": 990.3973509933774,
"y": 1487.5768211920529,
"width": 782,
"node_type": "markdown_note",
"name": "note_1",
@ -179,8 +182,8 @@
},
{
"id": "note_2",
"x": 1944,
"y": 760.9999999999999,
"x": 3217.6543046357615,
"y": 2081.596026490066,
"width": 400,
"node_type": "markdown_note",
"name": "note_2",
@ -190,8 +193,8 @@
},
{
"id": "note_3",
"x": 2381.178207301403,
"y": 790.2196835690842,
"x": 2294.4516556291387,
"y": 2399.6099337748346,
"width": 638,
"node_type": "markdown_note",
"name": "note_3",
@ -201,8 +204,8 @@
},
{
"id": "note_4",
"x": 2405.796928768747,
"y": -362.26583299682716,
"x": 2544.684105960265,
"y": 1126.5490066225163,
"width": 399,
"node_type": "markdown_note",
"name": "note_4",
@ -212,62 +215,62 @@
},
{
"id": "provider_column",
"x": 1947.2039072039072,
"y": 32.08363858363858,
"x": 2542,
"y": 1696,
"width": 400
},
{
"id": "ministral",
"x": 1947.0573870573871,
"y": 271.94139194139194,
"x": 2542,
"y": 1890,
"width": 400
},
{
"id": "user",
"x": 0,
"y": 656.5,
"x": 191,
"y": 2423,
"width": 400
},
{
"id": "platform",
"x": 480,
"y": 656.5,
"x": 858.0384105960266,
"y": 2286.5,
"width": 400
},
{
"id": "impact_scope",
"x": 960,
"y": 656.5,
"x": 1342,
"y": 2286.5,
"width": 400
},
{
"id": "user_first_name",
"x": 1440,
"y": 895,
"x": 1822,
"y": 2505,
"width": 400
},
{
"id": "user_full_name",
"x": 1440,
"y": 269,
"x": 1822,
"y": 1959,
"width": 400
},
{
"id": "ticket",
"x": 1946.9108669108673,
"y": 657.2161172161173,
"x": 2302,
"y": 2286.5,
"width": 400
},
{
"id": "sla_target",
"x": 1440,
"y": 582,
"x": 1822,
"y": 2232,
"width": 400
},
{
"id": "agent_reply",
"x": 2384.5665445665445,
"y": 657.449938949939,
"x": 2782,
"y": 2151,
"width": 400
}
],
@ -353,10 +356,10 @@
"from": "ministral",
"to": "agent_reply",
"type": "semantic",
"source_handle": "semantic-out",
"source_handle": "semantic-out-bottom",
"target_handle": "data-in-top"
}
],
"layout_direction": "LR"
}
}
}

View file

@ -2,8 +2,8 @@
"recipe": {
"model_providers": [
{
"name": "provider_1",
"endpoint": "https://openrouter.ai/api/v1",
"name": "openai-compatible",
"endpoint": "",
"provider_type": "openai",
"extra_headers": {},
"extra_body": {}
@ -12,12 +12,11 @@
"mcp_providers": [],
"model_configs": [
{
"alias": "model_1",
"model": "mistralai/ministral-8b-2512",
"provider": "provider_1",
"alias": "coding-model",
"model": "",
"provider": "openai-compatible",
"inference_parameters": {
"temperature": 0.7,
"max_tokens": 2048
"temperature": 0.7
}
}
],
@ -66,24 +65,29 @@
"column_type": "llm-text",
"name": "instruction",
"drop": false,
"model_alias": "model_1",
"model_alias": "coding-model",
"prompt": "Write one clear Python coding instruction.\nDomain: {{ domain }}\nTask type: {{ task_type }}\n\nKeep it practical and specific.\nReturn only the instruction without any code.",
"with_trace": "none"
"with_trace": "none",
"extract_reasoning_content": false
},
{
"column_type": "llm-code",
"name": "code_implementation",
"drop": false,
"model_alias": "model_1",
"model_alias": "coding-model",
"prompt": "Write Python code for:\n{{ instruction }}\n\nRequirements:\n- runnable script or function\n- include needed imports\n- short comments only where useful\n- no markdown fences",
"with_trace": "none",
"extract_reasoning_content": false,
"code_lang": "python"
},
{
"column_type": "llm-judge",
"name": "code_judge_result",
"drop": false,
"model_alias": "model_1",
"model_alias": "coding-model",
"prompt": "Evaluate generated Python code against the instruction.\n\nInstruction:\n{{ instruction }}\n\nCode:\n{{ code_implementation }}",
"with_trace": "none",
"extract_reasoning_content": false,
"scores": [
{
"name": "Correctness",
@ -109,52 +113,10 @@
},
"ui": {
"nodes": [
{
"id": "provider_1",
"x": 1032.6798211423347,
"y": -450.4885376732656,
"width": 400
},
{
"id": "model_1",
"x": 1538.0273166472973,
"y": -483.2003290046642,
"width": 400
},
{
"id": "domain",
"x": 0,
"y": 24,
"width": 400
},
{
"id": "task_type",
"x": 480,
"y": 24,
"width": 400
},
{
"id": "instruction",
"x": 958.8989453654599,
"y": -9.971266983459952,
"width": 400
},
{
"id": "code_implementation",
"x": 1538.788058529745,
"y": -45.56493974435071,
"width": 400
},
{
"id": "code_judge_result",
"x": 2040.9251520522098,
"y": -13.362336454344792,
"width": 400
},
{
"id": "note_1",
"x": 1482.1328175027095,
"y": 242.4370179053253,
"x": 1526,
"y": 1790.75,
"width": 568,
"node_type": "markdown_note",
"name": "note_1",
@ -164,14 +126,56 @@
},
{
"id": "note_2",
"x": 2513.2527820497985,
"y": -235.2544980991115,
"x": 2597.376821192053,
"y": 1233.2039735099338,
"width": 471,
"node_type": "markdown_note",
"name": "note_2",
"markdown": "The **LLM Judge** block evaluates generated outputs with rubric-style scores.\n\n##### Important:\n\n- A judge can have **one or many scores**\n- Each score has:\n - a name (for example: `Correctness`)\n - a description\n - options (value + meaning)\n\n##### Example multi-score setup:\n\n- Correctness\n- Readability\n- Efficiency\n\n##### Why use multiple scores:\n\n- You get richer quality signals than a single pass/fail\n- Easier filtering and weighting later in training data prep\n\n##### Practical pattern:\n\n1. Generate code with LLM Code\n2. Judge with 2-4 focused scores\n3. Keep high-quality rows based on score thresholds\n",
"note_color": "#FEF3C7",
"note_opacity": "35"
},
{
"id": "openai-compatible",
"x": 1627.1046357615896,
"y": 921.0301324503313,
"width": 400
},
{
"id": "coding-model",
"x": 1627.1046357615894,
"y": 1138.910927152318,
"width": 400
},
{
"id": "domain",
"x": 84,
"y": 1600.5,
"width": 400
},
{
"id": "task_type",
"x": 648,
"y": 1600.5,
"width": 400
},
{
"id": "instruction",
"x": 1128,
"y": 1567,
"width": 400
},
{
"id": "code_implementation",
"x": 1627.1046357615894,
"y": 1531.4728476821192,
"width": 400
},
{
"id": "code_judge_result",
"x": 2124.617218543046,
"y": 1567.076490066225,
"width": 400
}
],
"edges": [
@ -190,11 +194,11 @@
"target_handle": "data-in"
},
{
"from": "provider_1",
"to": "model_1",
"from": "openai-compatible",
"to": "coding-model",
"type": "semantic",
"source_handle": "semantic-out",
"target_handle": "semantic-in"
"source_handle": "semantic-out-bottom",
"target_handle": "semantic-in-top"
},
{
"from": "instruction",
@ -204,14 +208,14 @@
"target_handle": "data-in"
},
{
"from": "model_1",
"from": "coding-model",
"to": "instruction",
"type": "semantic",
"source_handle": "semantic-out-bottom",
"target_handle": "data-in-top"
},
{
"from": "model_1",
"from": "coding-model",
"to": "code_implementation",
"type": "semantic",
"source_handle": "semantic-out-bottom",
@ -225,7 +229,7 @@
"target_handle": "data-in"
},
{
"from": "model_1",
"from": "coding-model",
"to": "code_judge_result",
"type": "semantic",
"source_handle": "semantic-out-bottom",
@ -234,4 +238,4 @@
],
"layout_direction": "LR"
}
}
}

View file

@ -2,8 +2,8 @@
"recipe": {
"model_providers": [
{
"name": "provider_1",
"endpoint": "https://openrouter.ai/api/v1",
"name": "vllm",
"endpoint": "",
"provider_type": "openai",
"extra_headers": {},
"extra_body": {}
@ -12,12 +12,11 @@
"mcp_providers": [],
"model_configs": [
{
"alias": "model_1",
"model": "mistralai/ministral-8b-2512",
"provider": "provider_1",
"alias": "sql-pro",
"model": "",
"provider": "vllm",
"inference_parameters": {
"temperature": 0.7,
"max_tokens": 2048
"temperature": 0.7
}
}
],
@ -93,19 +92,35 @@
"column_type": "llm-text",
"name": "sql_prompt",
"drop": false,
"model_alias": "model_1",
"model_alias": "sql-pro",
"prompt": "Generate one natural-language SQL task.\nContext:\n- Domain: {{ domain }}\n- Topic: {{ topic }}\n- Task type: {{ sql_task_type }}\nRules:\n- Must start exactly with: \"{{ instruction_phrase }}\"\n- Make it specific and practical.\n- Mention expected business outcome.\n- Keep it 1-2 sentences.\n- Do not include SQL code.\n- Output only the instruction text.",
"system_prompt": "You create clear, realistic business SQL tasks for training data.\n",
"with_trace": "none"
"with_trace": "none",
"extract_reasoning_content": false
},
{
"column_type": "llm-code",
"name": "sql",
"drop": false,
"model_alias": "model_1",
"model_alias": "sql-pro",
"prompt": "Write SQL for this instruction:\n{{ sql_prompt }}\nReturn ONE SQL script with this exact structure:\n-- SCHEMA\n[CREATE TABLE statements]\n[INSERT statements with sample rows]\n-- QUERY\n[final SELECT query solving the instruction]\nRules:\n- Use 2-3 tables max.\n- Use realistic snake_case names.\n- Include 5-8 rows of sample data per table.\n- Query must match task type \"{{ sql_task_type }}\".\n- Use only tables/columns you created.\n- No markdown fences.\n- No explanation text outside SQL comments shown above.",
"system_prompt": "You are an expert SQL engineer. Produce correct, runnable SQL only.\n",
"with_trace": "none",
"extract_reasoning_content": false,
"code_lang": "sql:ansi"
},
{
"column_type": "validation",
"name": "sql-validator",
"drop": false,
"target_columns": [
"sql"
],
"validator_type": "code",
"validator_params": {
"code_lang": "sql:ansi"
},
"batch_size": 10
}
],
"processors": []
@ -119,58 +134,10 @@
},
"ui": {
"nodes": [
{
"id": "provider_1",
"x": -1092.2003193114556,
"y": 715.157165665104,
"width": 400
},
{
"id": "model_1",
"x": -546.1001596557278,
"y": 681.8114012018752,
"width": 400
},
{
"id": "domain",
"x": -18.379173679952572,
"y": 137.70260329000595,
"width": 400
},
{
"id": "topic",
"x": -18.6022437080035,
"y": 373.0271253737222,
"width": 400
},
{
"id": "sql_task_type",
"x": 477.85851567876665,
"y": 137.4202046707293,
"width": 400
},
{
"id": "instruction_phrase",
"x": -477.8585156787667,
"y": 138.42770371608808,
"width": 400
},
{
"id": "sql_prompt",
"x": -18.188598798124787,
"y": 701.5157165665104,
"width": 400
},
{
"id": "sql",
"x": -18.188598798124758,
"y": 950.647309355259,
"width": 400
},
{
"id": "note_1",
"x": -103.00586025666547,
"y": -332.088439142397,
"x": 338,
"y": 1020,
"width": 600,
"node_type": "markdown_note",
"name": "note_1",
@ -180,8 +147,8 @@
},
{
"id": "note_2",
"x": 517.0372102151987,
"y": 600.4949327304814,
"x": 1675.8410596026492,
"y": 1644.2185430463576,
"width": 400,
"node_type": "markdown_note",
"name": "note_2",
@ -191,25 +158,79 @@
},
{
"id": "note_3",
"x": 12.635681904967385,
"y": 1224.7626182706356,
"x": 2198.980132450331,
"y": 1723.1456953642385,
"width": 400,
"node_type": "markdown_note",
"name": "note_3",
"markdown": "The **LLM Code** block (`sql`) generates SQL script from `{{ sql_prompt }}`.\n\n##### In this recipe it returns:\n\n- schema section (`CREATE TABLE`)\n- sample seed rows (`INSERT`)\n- final query (`SELECT`)\n\n##### Current status:\n\n- SQL validation block is **not** included yet in this learning recipe\n- We will add SQL validation later",
"markdown": "The **LLM Code** block (`sql`) generates SQL script from `{{ sql_prompt }}`.\n\n##### In this recipe it returns:\n\n- schema section (`CREATE TABLE`)\n- sample seed rows (`INSERT`)\n- final query (`SELECT`)\n",
"note_color": "#DBEAFE",
"note_opacity": "35"
},
{
"id": "note_4",
"x": -1044,
"y": 108.64730935525904,
"x": 1264,
"y": 1037,
"width": 400,
"node_type": "markdown_note",
"name": "note_4",
"markdown": "Sampler columns are useful during generation, but often noisy in final output.\n\nSet helper columns to **drop=true** (like in this recipe), keep only output columns you want to export.\n\n#### Final keep we have set here:\n\n- `sql_prompt`\n- `sql`\n\n",
"note_color": "#DBEAFE",
"note_opacity": "35"
},
{
"id": "vllm",
"x": 1939.5364238410598,
"y": 781.25,
"width": 400
},
{
"id": "sql-pro",
"x": 1939.5364238410593,
"y": 975.25,
"width": 400
},
{
"id": "domain",
"x": 680,
"y": 1495,
"width": 400
},
{
"id": "topic",
"x": 1160,
"y": 1413,
"width": 400
},
{
"id": "sql_task_type",
"x": 1160,
"y": 1577,
"width": 400
},
{
"id": "instruction_phrase",
"x": 100,
"y": 1495,
"width": 400
},
{
"id": "sql_prompt",
"x": 1672.6490066225165,
"y": 1457.6854304635763,
"width": 400
},
{
"id": "sql",
"x": 2194.9006622516554,
"y": 1457.110927152318,
"width": 400
},
{
"id": "sql-validator",
"x": 2682.5827814569534,
"y": 1491.0413907284767,
"width": 400
}
],
"edges": [
@ -217,8 +238,8 @@
"from": "domain",
"to": "topic",
"type": "canvas",
"source_handle": "data-out-bottom",
"target_handle": "data-in-top"
"source_handle": "data-out",
"target_handle": "data-in"
},
{
"from": "domain",
@ -238,38 +259,52 @@
"from": "topic",
"to": "sql_prompt",
"type": "canvas",
"source_handle": "data-out-bottom",
"target_handle": "data-in-top"
"source_handle": "data-out",
"target_handle": "data-in"
},
{
"from": "sql_prompt",
"to": "sql",
"type": "canvas",
"source_handle": "data-out-bottom",
"target_handle": "data-in-top"
},
{
"from": "provider_1",
"to": "model_1",
"type": "semantic",
"source_handle": "semantic-out",
"target_handle": "semantic-in"
},
{
"from": "model_1",
"to": "sql_prompt",
"type": "semantic",
"source_handle": "semantic-out",
"source_handle": "data-out",
"target_handle": "data-in"
},
{
"from": "model_1",
"from": "vllm",
"to": "sql-pro",
"type": "semantic",
"source_handle": "semantic-out-bottom",
"target_handle": "semantic-in-top"
},
{
"from": "sql-pro",
"to": "sql",
"type": "semantic",
"source_handle": "semantic-out-bottom",
"target_handle": "data-in-top"
},
{
"from": "sql",
"to": "sql-validator",
"type": "semantic",
"source_handle": "data-out",
"target_handle": "data-in"
},
{
"from": "sql-pro",
"to": "sql_prompt",
"type": "semantic",
"source_handle": "semantic-out-bottom",
"target_handle": "data-in-top"
},
{
"from": "sql_prompt",
"to": "sql_task_type",
"type": "canvas",
"source_handle": "data-out-left",
"target_handle": "data-in-right"
}
],
"layout_direction": "LR"
}
}
}

View file

@ -24,7 +24,7 @@ import {
import { ShineBorder } from "@/components/ui/shine-border";
import { toastError } from "@/shared/toast";
import {
AiChat02Icon,
Album02Icon,
ArrowDown01Icon,
CodeIcon,
CookBookIcon,
@ -60,20 +60,20 @@ type TemplateCard = {
const TEMPLATE_CARDS: TemplateCard[] = [
{
title: "Structured Outputs + Jinja Expressions",
title: "Instruction from Answer",
description:
"Support ticket triage dataset with structured JSON outputs and Jinja if/else refs.",
icon: FunctionIcon,
difficulty: "Advanced",
learningBadges: ["Structured LLM", "Expression", "Jinja"],
"Start from seed answer fields and generate matching user instructions for SFT pairs.",
icon: Plant01Icon,
difficulty: "Easy",
learningBadges: ["Seed Dataset", "LLM Text", "Prompting"],
surfaceClassName:
"from-cyan-500/15 via-sky-500/5 to-transparent dark:from-cyan-400/30 dark:via-sky-400/14 dark:to-cyan-950/16",
"from-emerald-500/15 via-green-500/5 to-transparent dark:from-emerald-400/30 dark:via-green-400/14 dark:to-emerald-950/16",
shineColor: [
"rgb(6 182 212 / 0.45)",
"rgb(56 189 248 / 0.4)",
"rgb(34 211 238 / 0.45)",
"rgb(16 185 129 / 0.45)",
"rgb(34 197 94 / 0.4)",
"rgb(52 211 153 / 0.45)",
],
learningRecipeId: "structured-outputs-jinja",
learningRecipeId: "instruction-from-answer",
},
{
title: "PDF Document QA",
@ -92,20 +92,20 @@ const TEMPLATE_CARDS: TemplateCard[] = [
learningRecipeId: "pdf-grounded-qa",
},
{
title: "Instruction from Answer",
title: "OCR Document Extraction",
description:
"Start from seed answer fields and generate matching user instructions for SFT pairs.",
icon: Plant01Icon,
difficulty: "Easy",
learningBadges: ["Seed Dataset", "LLM Text", "Prompting"],
"Use image context from seed data to generate OCR-style extraction outputs.",
icon: Album02Icon,
difficulty: "Starter",
learningBadges: ["Vision", "LLM Text", "Image Context"],
surfaceClassName:
"from-emerald-500/15 via-green-500/5 to-transparent dark:from-emerald-400/30 dark:via-green-400/14 dark:to-emerald-950/16",
"from-lime-500/15 via-emerald-500/5 to-transparent dark:from-lime-400/30 dark:via-emerald-400/14 dark:to-lime-950/16",
shineColor: [
"rgb(16 185 129 / 0.45)",
"rgb(34 197 94 / 0.4)",
"rgb(52 211 153 / 0.45)",
"rgb(132 204 22 / 0.45)",
"rgb(16 185 129 / 0.4)",
"rgb(74 222 128 / 0.45)",
],
learningRecipeId: "instruction-from-answer",
learningRecipeId: "ocr-document-extraction",
},
{
title: "Text to Python",
@ -140,20 +140,20 @@ const TEMPLATE_CARDS: TemplateCard[] = [
learningRecipeId: "text-to-sql",
},
{
title: "Multi-Turn Chat",
title: "Structured Outputs + Jinja Expressions",
description:
"Role-based multi-turn conversations for assistant behavior, memory, and response quality.",
icon: AiChat02Icon,
difficulty: "Easy",
learningBadges: ["Structured LLM", "LLM Text"],
"Support ticket triage dataset with structured JSON outputs and Jinja if/else refs.",
icon: FunctionIcon,
difficulty: "Advanced",
learningBadges: ["Structured LLM", "Expression", "Jinja"],
surfaceClassName:
"from-rose-500/15 via-pink-500/5 to-transparent dark:from-rose-400/30 dark:via-pink-400/14 dark:to-rose-950/16",
"from-cyan-500/15 via-sky-500/5 to-transparent dark:from-cyan-400/30 dark:via-sky-400/14 dark:to-cyan-950/16",
shineColor: [
"rgb(244 63 94 / 0.45)",
"rgb(236 72 153 / 0.4)",
"rgb(251 113 133 / 0.45)",
"rgb(6 182 212 / 0.45)",
"rgb(56 189 248 / 0.4)",
"rgb(34 211 238 / 0.45)",
],
learningRecipeId: "conversation",
learningRecipeId: "structured-outputs-jinja",
},
];
@ -207,7 +207,10 @@ function LearningRecipeCards({
loadingTemplateId === template.learningRecipeId;
const isDisabled = !isReady || isLoading || Boolean(loadingTemplateId);
const visibleLearningBadges = template.learningBadges.slice(0, 4);
const extraLearningBadgeCount = Math.max(0, template.learningBadges.length - 4);
const extraLearningBadgeCount = Math.max(
0,
template.learningBadges.length - 4,
);
return (
<button
key={template.title}
@ -259,15 +262,21 @@ function LearningRecipeCards({
</Badge>
))}
{extraLearningBadgeCount > 0 ? (
<Badge variant="outline" className="h-5 shrink-0 px-1.5 text-[10px]">
<Badge
variant="outline"
className="h-5 shrink-0 px-1.5 text-[10px]"
>
+{extraLearningBadgeCount}
</Badge>
) : null}
{!isReady ? (
<Badge variant="secondary" className="h-5 shrink-0 px-1.5 text-[10px]">
{isReady ? null : (
<Badge
variant="secondary"
className="h-5 shrink-0 px-1.5 text-[10px]"
>
Soon
</Badge>
) : null}
)}
</>
)}
</div>

View file

@ -66,7 +66,8 @@ export function DatasetStep() {
hfToken,
setHfToken,
datasetSource,
setDatasetSource,
selectHfDataset,
selectLocalDataset,
datasetFormat,
setDatasetFormat,
dataset,
@ -85,7 +86,8 @@ export function DatasetStep() {
hfToken: s.hfToken,
setHfToken: s.setHfToken,
datasetSource: s.datasetSource,
setDatasetSource: s.setDatasetSource,
selectHfDataset: s.selectHfDataset,
selectLocalDataset: s.selectLocalDataset,
datasetFormat: s.datasetFormat,
setDatasetFormat: s.setDatasetFormat,
dataset: s.dataset,
@ -138,7 +140,9 @@ export function DatasetStep() {
<div className="flex gap-2">
<Button
variant={datasetSource === "huggingface" ? "dark" : "outline"}
onClick={() => setDatasetSource("huggingface")}
onClick={() =>
selectHfDataset(datasetSource === "huggingface" ? dataset : null)
}
className="flex-1"
>
<img
@ -151,7 +155,11 @@ export function DatasetStep() {
</Button>
<Button
variant={datasetSource === "upload" ? "dark" : "outline"}
onClick={() => setDatasetSource("upload")}
onClick={() =>
selectLocalDataset(
datasetSource === "upload" ? uploadedFile : null,
)
}
className="flex-1"
>
<HugeiconsIcon icon={Upload04Icon} data-icon="inline-start" />

View file

@ -1,3 +1,5 @@
import { authFetch } from "@/features/auth";
const DEFAULT_BASE = "/api/data-recipe";
export const DATA_DESIGNER_API_BASE =
@ -15,6 +17,8 @@ export type JobStatusResponse = {
stage?: string | null;
// biome-ignore lint/style/useNamingConvention: api schema
current_column?: string | null;
// biome-ignore lint/style/useNamingConvention: api schema
completed_columns?: string[] | null;
batch?: {
idx?: number | null;
total?: number | null;
@ -87,6 +91,12 @@ export type SeedInspectUploadRequest = {
content_base64: string;
// biome-ignore lint/style/useNamingConvention: api schema
preview_size?: number;
// biome-ignore lint/style/useNamingConvention: api schema
seed_source_type?: "local" | "unstructured";
// biome-ignore lint/style/useNamingConvention: api schema
unstructured_chunk_size?: number;
// biome-ignore lint/style/useNamingConvention: api schema
unstructured_chunk_overlap?: number;
};
export type SeedInspectResponse = {
@ -138,7 +148,7 @@ async function parseErrorResponse(response: Response): Promise<string> {
}
async function postJson<T>(path: string, payload: unknown): Promise<T> {
const response = await fetch(`${DATA_DESIGNER_API_BASE}${path}`, {
const response = await authFetch(`${DATA_DESIGNER_API_BASE}${path}`, {
method: "POST",
headers: {
"Content-Type": "application/json",
@ -154,7 +164,7 @@ async function postJson<T>(path: string, payload: unknown): Promise<T> {
}
async function getJson<T>(path: string): Promise<T> {
const response = await fetch(`${DATA_DESIGNER_API_BASE}${path}`);
const response = await authFetch(`${DATA_DESIGNER_API_BASE}${path}`);
if (!response.ok) {
throw new Error(await parseErrorResponse(response));
}
@ -265,7 +275,7 @@ export async function streamRecipeJobEvents(options: {
query = `?after=${options.lastEventId}`;
}
const response = await fetch(
const response = await authFetch(
`${DATA_DESIGNER_API_BASE}/jobs/${options.jobId}/events${query}`,
{
method: "GET",

View file

@ -17,7 +17,12 @@ import {
TagsIcon,
UserAccountIcon,
} from "@hugeicons/core-free-icons";
import type { LlmType, NodeConfig, SamplerType, SeedSourceType } from "../types";
import type {
LlmType,
NodeConfig,
SamplerType,
SeedSourceType,
} from "../types";
import {
makeExpressionConfig,
makeLlmConfig,
@ -26,12 +31,22 @@ import {
makeModelProviderConfig,
makeSamplerConfig,
makeSeedConfig,
makeValidatorConfig,
} from "../utils";
export type BlockKind = "sampler" | "llm" | "expression" | "seed" | "note";
export type BlockKind =
| "sampler"
| "llm"
| "validator"
| "expression"
| "seed"
| "note";
export type BlockType =
| SamplerType
| LlmType
| "validator_python"
| "validator_sql"
| "validator_oxc"
| "expression"
| "markdown_note"
| "seed"
@ -65,6 +80,7 @@ export type BlockDialogKey =
| "uuid"
| "person"
| "llm"
| "validator"
| "model_provider"
| "model_config"
| "expression";
@ -98,6 +114,12 @@ export const BLOCK_GROUPS: BlockGroup[] = [
description: "Generation, providers, and model aliases.",
icon: PencilEdit02Icon,
},
{
kind: "validator",
title: "Validators",
description: "Validate generated code outputs with built-in engines.",
icon: Shield02Icon,
},
{
kind: "expression",
title: "Expression",
@ -116,7 +138,7 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [
{
kind: "seed",
type: "seed_hf",
title: "Hugginface dataset",
title: "Hugging Face dataset",
description: "Load real rows from HF and use them as generation context.",
icon: Plant01Icon,
dialogKey: "seed",
@ -275,6 +297,36 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [
dialogKey: "model_config",
createConfig: (id, existing) => makeModelConfig(id, existing),
},
{
kind: "validator",
type: "validator_python",
title: "Python Validator",
description: "Validate Python code columns.",
icon: Shield02Icon,
dialogKey: "validator",
createConfig: (id, existing) =>
makeValidatorConfig(id, "code", "python", existing),
},
{
kind: "validator",
type: "validator_sql",
title: "SQL Validator",
description: "Validate SQL code columns.",
icon: Shield02Icon,
dialogKey: "validator",
createConfig: (id, existing) =>
makeValidatorConfig(id, "code", "sql:sqlite", existing),
},
{
kind: "validator",
type: "validator_oxc",
title: "OXC Validator",
description: "Validate JavaScript or TypeScript code columns.",
icon: Shield02Icon,
dialogKey: "validator",
createConfig: (id, existing) =>
makeValidatorConfig(id, "oxc", "javascript", existing),
},
{
kind: "expression",
type: "expression",
@ -331,6 +383,16 @@ export function getBlockDefinitionForConfig(
if (config.kind === "llm") {
return getBlockDefinition("llm", config.llm_type);
}
if (config.kind === "validator") {
if (config.validator_type === "oxc") {
return getBlockDefinition("validator", "validator_oxc");
}
const isSql = config.code_lang.startsWith("sql:");
return getBlockDefinition(
"validator",
isSql ? "validator_sql" : "validator_python",
);
}
if (config.kind === "model_provider") {
return getBlockDefinition("llm", "model_provider");
}

View file

@ -16,6 +16,7 @@ import { TimedeltaDialog } from "../dialogs/samplers/timedelta-dialog";
import { UniformDialog } from "../dialogs/samplers/uniform-dialog";
import { UuidDialog } from "../dialogs/samplers/uuid-dialog";
import { MarkdownNoteDialog } from "../dialogs/markdown-note/markdown-note-dialog";
import { ValidatorDialog } from "../dialogs/validators/validator-dialog";
export function renderBlockDialog(
config: NodeConfig | null,
@ -109,6 +110,10 @@ export function renderBlockDialog(
return config.kind === "expression" ? (
<ExpressionDialog config={config} onUpdate={update} />
) : null;
case "validator":
return config.kind === "validator" ? (
<ValidatorDialog config={config} onUpdate={update} />
) : null;
case "markdown_note":
return config.kind === "markdown_note" ? (
<MarkdownNoteDialog config={config} onUpdate={update} />

View file

@ -1,4 +1,5 @@
import { Button } from "@/components/ui/button";
import { Badge } from "@/components/ui/badge";
import { Input } from "@/components/ui/input";
import {
Sheet,
@ -15,6 +16,7 @@ import {
type Database02Icon,
DragDropVerticalIcon,
PlusSignIcon,
Search01Icon,
Tick02Icon,
Upload01Icon,
} from "@hugeicons/core-free-icons";
@ -39,10 +41,17 @@ type SheetView =
| "sampler"
| "seed"
| "llm"
| "validator"
| "expression"
| "note"
| "processor";
type SheetKind = "sampler" | "seed" | "llm" | "expression" | "note";
type SheetKind =
| "sampler"
| "seed"
| "llm"
| "validator"
| "expression"
| "note";
type RootSheetView = Exclude<SheetView, "root">;
type RootGroup = {
kind: RootSheetView;
@ -63,6 +72,9 @@ type BlockSheetProps = {
onAddModelProvider: () => void;
onAddModelConfig: () => void;
onAddExpression: () => void;
onAddValidator: (
type: "validator_python" | "validator_sql" | "validator_oxc",
) => void;
onAddMarkdownNote: () => void;
onOpenProcessors: () => void;
copied: boolean;
@ -89,6 +101,9 @@ function getSheetTitle(sheetView: SheetView): string {
if (sheetView === "expression") {
return "Expression blocks";
}
if (sheetView === "validator") {
return "Validator blocks";
}
if (sheetView === "note") {
return "Note blocks";
}
@ -103,6 +118,7 @@ const VIEW_KIND: Record<SheetView, SheetKind | null> = {
sampler: "sampler",
seed: "seed",
llm: "llm",
validator: "validator",
expression: "expression",
note: "note",
processor: null,
@ -121,6 +137,7 @@ const SEARCHABLE_KINDS: SheetKind[] = [
"sampler",
"seed",
"llm",
"validator",
"expression",
"note",
];
@ -136,6 +153,8 @@ function BlockSheetButton({
draggable = false,
onDragStart,
trailing = "chevron",
disabled = false,
badge,
}: {
icon: typeof Database02Icon;
title: string;
@ -145,24 +164,38 @@ function BlockSheetButton({
draggable?: boolean;
onDragStart?: (event: ReactDragEvent<HTMLButtonElement>) => void;
trailing?: "chevron" | "drag" | "none";
disabled?: boolean;
badge?: string;
}): ReactElement {
return (
<button
type="button"
onClick={onClick}
draggable={draggable}
onDragStart={onDragStart}
className={`flex w-full items-center gap-3 border-l-2 bg-background px-3 py-3 text-left transition hover:bg-muted/35 ${
onClick={disabled ? undefined : onClick}
disabled={disabled}
draggable={disabled ? false : draggable}
onDragStart={disabled ? undefined : onDragStart}
className={`flex w-full items-center gap-3 border-l-2 bg-background px-3 py-3 text-left transition ${
disabled ? "cursor-not-allowed opacity-60" : "hover:bg-muted/35"
} ${
isActive
? "border-emerald-500"
: "border-transparent hover:border-border/60"
: disabled
? "border-transparent"
: "border-transparent hover:border-border/60"
} ${draggable ? "cursor-grab active:cursor-grabbing" : ""}`}
>
<div className="flex size-9 items-center justify-center rounded-xl text-foreground/70">
<HugeiconsIcon icon={icon} className="size-5" />
</div>
<div className="flex-1">
<p className="text-sm font-semibold text-foreground">{title}</p>
<div className="flex items-center gap-2">
<p className="text-sm font-semibold text-foreground">{title}</p>
{badge ? (
<Badge variant="outline" className="rounded-full text-[10px]">
{badge}
</Badge>
) : null}
</div>
<p className="text-[11px] text-muted-foreground">{description}</p>
</div>
{trailing === "chevron" ? (
@ -193,6 +226,7 @@ export function BlockSheet({
onAddModelProvider,
onAddModelConfig,
onAddExpression,
onAddValidator,
onAddMarkdownNote,
onOpenProcessors,
copied,
@ -302,6 +336,12 @@ export function BlockSheet({
onAddLlm(type as LlmType);
return;
}
if (kind === "validator") {
onAddValidator(
type as "validator_python" | "validator_sql" | "validator_oxc",
);
return;
}
if (kind === "expression") {
onAddExpression();
return;
@ -355,12 +395,18 @@ export function BlockSheet({
)}
<SheetTitle>{sheetTitle}</SheetTitle>
</div>
<Input
value={search}
onChange={(event) => setSearch(event.target.value)}
placeholder="Search blocks..."
className="corner-squircle mt-3 h-9"
/>
<div className="relative mt-3">
<HugeiconsIcon
icon={Search01Icon}
className="pointer-events-none absolute left-2.5 top-1/2 size-4 -translate-y-1/2 text-muted-foreground"
/>
<Input
value={search}
onChange={(event) => setSearch(event.target.value)}
placeholder="Search blocks..."
className="corner-squircle h-9 pl-8"
/>
</div>
</SheetHeader>
<div className=" py-4">
<div className="mt-4 flex flex-col gap-2">
@ -399,8 +445,12 @@ export function BlockSheet({
trailing={
item.kind === "expression" || item.kind === "note"
? "drag"
: "chevron"
: item.kind === "processor"
? "none"
: "chevron"
}
disabled={item.kind === "processor"}
badge={item.kind === "processor" ? "Work in progress" : undefined}
onClick={() => {
if (item.kind === "processor") {
setSheetOpen(false);

View file

@ -0,0 +1,49 @@
import { CookBookIcon, TestTube01Icon } from "@hugeicons/core-free-icons";
import { HugeiconsIcon } from "@hugeicons/react";
import type { ReactElement } from "react";
import { Button } from "@/components/ui/button";
import type { RecipeExecutionKind } from "../../execution-types";
type RunValidateFloatingControlsProps = {
runBusy: boolean;
runDialogKind: RecipeExecutionKind;
validateLoading: boolean;
executionLocked: boolean;
onOpenRunDialog: (kind: RecipeExecutionKind) => void;
onValidate: () => void;
};
export function RunValidateFloatingControls({
runBusy,
runDialogKind,
validateLoading,
executionLocked,
onOpenRunDialog,
onValidate,
}: RunValidateFloatingControlsProps): ReactElement {
return (
<div className="pointer-events-none absolute inset-x-0 bottom-3 z-20 flex justify-center">
<div className="pointer-events-auto flex items-center gap-2">
<Button
type="button"
className="corner-squircle h-11 px-5"
onClick={() => onOpenRunDialog(runDialogKind)}
disabled={runBusy}
>
<HugeiconsIcon icon={CookBookIcon} className="size-4" />
{runBusy ? "Running..." : "Run"}
</Button>
<Button
type="button"
variant="outline"
className="corner-squircle h-11 px-5"
onClick={onValidate}
disabled={validateLoading || executionLocked}
>
<HugeiconsIcon icon={TestTube01Icon} className="size-4" />
{validateLoading ? "Validating..." : "Validate"}
</Button>
</div>
</div>
);
}

View file

@ -7,11 +7,13 @@ import { RECIPE_FLOATING_ICON_BUTTON_CLASS } from "../recipe-floating-icon-butto
type ViewportControlsProps = {
interactive: boolean;
lockDisabled?: boolean;
onToggleInteractive: () => void;
};
export function ViewportControls({
interactive,
lockDisabled = false,
onToggleInteractive,
}: ViewportControlsProps): ReactElement {
const { zoomIn, zoomOut, fitView, getNodes } = useReactFlow();
@ -68,6 +70,7 @@ export function ViewportControls({
variant="ghost"
size="icon"
className={RECIPE_FLOATING_ICON_BUTTON_CLASS}
disabled={lockDisabled}
onClick={onToggleInteractive}
aria-label={interactive ? "Lock interaction" : "Unlock interaction"}
>

View file

@ -12,7 +12,7 @@ import {
import { cn } from "@/lib/utils";
import { isExecutionInProgress } from "../../executions/execution-helpers";
import type { RecipeExecutionRecord } from "../../execution-types";
import { formatCellValue, isExpandableCellValue } from "./executions-view-helpers";
import { hasExpandableTextCell } from "./executions-view-helpers";
type ExecutionDataTabProps = {
execution: RecipeExecutionRecord;
@ -129,9 +129,7 @@ export function ExecutionDataTab({
columns={tableColumns}
data={datasetRowsForTable}
getRowClassName={(row, _rowIndex, rowId) => {
const canExpand = visibleDatasetColumnNames.some((columnName) =>
isExpandableCellValue(formatCellValue(row[columnName])),
);
const canExpand = hasExpandableTextCell(row, visibleDatasetColumnNames);
if (!canExpand) {
return undefined;
}
@ -141,9 +139,7 @@ export function ExecutionDataTab({
);
}}
onRowClick={(row, _rowIndex, rowId) => {
const canExpand = visibleDatasetColumnNames.some((columnName) =>
isExpandableCellValue(formatCellValue(row[columnName])),
);
const canExpand = hasExpandableTextCell(row, visibleDatasetColumnNames);
if (!canExpand || !selectedExecutionIdSafe) {
return;
}

View file

@ -57,7 +57,7 @@ export function ExecutionOverviewTab({
{showSummaryCards && (
<div className="space-y-3">
<div className="grid gap-3 md:grid-cols-2">
<div className="h-full rounded-lg bg-muted/20 p-3">
<div className="h-full rounded-xl border border-border/60 bg-card/55 p-3">
<div className="mb-2 flex items-center justify-between">
<p className="text-xs text-muted-foreground">Run summary</p>
<HugeiconsIcon
@ -82,11 +82,11 @@ export function ExecutionOverviewTab({
</p>
<p className="flex items-center justify-between gap-3">
<span className="text-muted-foreground">Final stage</span>
<span className="font-semibold">{execution.stage ?? "--"}</span>
<span className="truncate font-semibold">{execution.stage ?? "--"}</span>
</p>
</div>
</div>
<div className="h-full rounded-lg bg-muted/20 p-3">
<div className="h-full rounded-xl border border-border/60 bg-card/55 p-3">
<div className="mb-2 flex items-center justify-between">
<p className="text-xs text-muted-foreground">Insights</p>
<HugeiconsIcon
@ -104,7 +104,7 @@ export function ExecutionOverviewTab({
<span className="font-semibold">{nullRate?.toFixed(1) ?? "--"}%</span>
</p>
<p className="flex items-center justify-between gap-3">
<span className="text-muted-foreground">Dropped columns</span>
<span className="text-muted-foreground">Side-effect columns</span>
<span className="font-semibold">{formatMetricValue(sideEffects.length)}</span>
</p>
{sideEffects.length > 0 && (
@ -128,12 +128,12 @@ export function ExecutionOverviewTab({
<div className="pt-0.5">
<div className="flex flex-wrap gap-1.5">
{lowUniquenessColumns.slice(0, 3).map((name) => (
<Badge key={name} variant="secondary">
<Badge key={name} variant="outline">
{name}
</Badge>
))}
{lowUniquenessColumns.length > 3 && (
<Badge variant="secondary">
<Badge variant="outline">
+{lowUniquenessColumns.length - 3} more
</Badge>
)}
@ -143,7 +143,7 @@ export function ExecutionOverviewTab({
</div>
</div>
</div>
<div className="rounded-lg bg-muted/20 p-3">
<div className="rounded-xl border border-border/60 bg-card/55 p-3">
<div className="mb-2 flex items-center justify-between">
<p className="text-xs text-muted-foreground">Model usage</p>
<HugeiconsIcon icon={Flag02Icon} className="size-4 text-muted-foreground" />
@ -151,7 +151,7 @@ export function ExecutionOverviewTab({
{modelUsageRows.length === 0 ? (
<p className="text-xs text-muted-foreground">No model usage yet.</p>
) : (
<div className="overflow-hidden rounded-md border">
<div className="overflow-hidden rounded-lg border border-border/60 bg-card/50">
<Table>
<TableHeader>
<TableRow>

View file

@ -2,7 +2,11 @@ import type { ReactElement } from "react";
import { Badge } from "@/components/ui/badge";
import { cn } from "@/lib/utils";
import type { RecipeExecutionRecord } from "../../execution-types";
import { isExecutionInProgress } from "../../executions/execution-helpers";
import {
executionLabel,
isExecutionInProgress,
normalizeRunName,
} from "../../executions/execution-helpers";
import {
formatStatus,
formatTimestamp,
@ -22,55 +26,62 @@ export function ExecutionSidebar({
onSelectExecution,
}: ExecutionSidebarProps): ReactElement {
return (
<aside className="w-72 shrink-0 border-r">
<div className="flex items-center justify-between border-b px-3 py-2">
<aside className="w-72 shrink-0 border-r border-border/60 bg-card/20">
<div className="flex items-center justify-between border-border/60 px-3 py-2">
<p className="text-xs font-semibold uppercase text-muted-foreground">
Executions
</p>
</div>
<div className="h-[calc(100%-45px)] overflow-auto p-2">
<div className="h-[calc(100%-45px)] space-y-2 overflow-auto p-2">
{executions.length === 0 ? (
<div className="rounded-xl border border-dashed p-3 text-xs text-muted-foreground">
<div className="rounded-xl border border-dashed border-border/60 p-3 text-xs text-muted-foreground">
No executions yet.
</div>
) : (
executions.map((execution) => (
<button
key={execution.id}
type="button"
onClick={() => onSelectExecution(execution.id)}
className={cn(
"mb-2 w-full rounded-xl corner-squircle border border-r-4 p-3 text-left",
selectedExecutionId === execution.id
? "border-primary/50 bg-primary/5"
: "hover:bg-muted/40",
statusRightBorder(execution.status),
)}
>
<div className="mb-2 flex items-center justify-between gap-2">
<p className="truncate text-sm font-medium capitalize">
{execution.kind}
</p>
<Badge
variant="secondary"
className={cn("capitalize", statusTone(execution.status))}
>
{formatStatus(execution.status)}
</Badge>
</div>
<p className="text-xs text-muted-foreground">{execution.rows} rows</p>
{isExecutionInProgress(execution.status) &&
typeof execution.batch?.total === "number" &&
execution.batch.total > 1 && (
<p className="text-xs text-muted-foreground">
Batch {execution.batch.idx ?? "--"}/{execution.batch.total}
</p>
executions.map((execution) => {
const title =
execution.kind === "full"
? (normalizeRunName(execution.run_name) ??
executionLabel(execution.kind))
: executionLabel(execution.kind);
return (
<button
key={execution.id}
type="button"
onClick={() => onSelectExecution(execution.id)}
className={cn(
"w-full rounded-xl corner-squircle border border-r-2 border-border/60 bg-card/60 p-3 text-left transition-colors",
selectedExecutionId === execution.id
? "border-primary/35 bg-primary/[0.045]"
: "hover:bg-muted/25",
statusRightBorder(execution.status),
)}
<p className="text-xs text-muted-foreground">
{formatTimestamp(execution.createdAt)}
</p>
</button>
))
>
<div className="mb-2 flex items-center justify-between gap-2">
<p className="truncate text-sm font-medium">
{title}
</p>
<Badge
variant="outline"
className={cn("capitalize text-[11px]", statusTone(execution.status))}
>
{formatStatus(execution.status)}
</Badge>
</div>
<p className="text-xs text-muted-foreground">{execution.rows} rows</p>
{isExecutionInProgress(execution.status) &&
typeof execution.batch?.total === "number" &&
execution.batch.total > 1 && (
<p className="text-xs text-muted-foreground">
Batch {execution.batch.idx ?? "--"}/{execution.batch.total}
</p>
)}
<p className="text-xs text-muted-foreground">
{formatTimestamp(execution.createdAt)}
</p>
</button>
);
})
)}
</div>
</aside>

View file

@ -3,6 +3,7 @@ import type {
RecipeExecutionStatus,
} from "../../execution-types";
import { isExecutionInProgress } from "../../executions/execution-helpers";
import { resolveImagePreview } from "../../utils/image-preview";
export type AnalysisColumnStat = {
column_name: string;
@ -55,6 +56,18 @@ export function truncateCellValue(value: string): string {
return `${value.slice(0, 180).trimEnd()}...`;
}
export function hasExpandableTextCell(
row: Record<string, unknown>,
visibleColumnNames: string[],
): boolean {
return visibleColumnNames.some((columnName) => {
if (resolveImagePreview(row[columnName])) {
return false;
}
return isExpandableCellValue(formatCellValue(row[columnName]));
});
}
function parseNumber(value: unknown): number | null {
return typeof value === "number" && Number.isFinite(value) ? value : null;
}
@ -90,28 +103,28 @@ export function parseAnalysisColumns(
export function statusTone(status: RecipeExecutionStatus): string {
if (status === "completed") {
return "bg-emerald-100 text-emerald-700";
return "border-emerald-500/30 text-emerald-700 dark:text-emerald-300";
}
if (status === "error" || status === "cancelled") {
return "bg-red-100 text-red-700";
return "border-red-500/30 text-red-700 dark:text-red-300";
}
if (isExecutionInProgress(status)) {
return "bg-amber-100 text-amber-700";
return "border-amber-500/30 text-amber-700 dark:text-amber-300";
}
return "bg-muted text-muted-foreground";
return "border-border/60 text-muted-foreground";
}
export function statusRightBorder(status: RecipeExecutionStatus): string {
if (status === "completed") {
return "border-r-emerald-500";
return "border-r-emerald-500/40";
}
if (status === "error" || status === "cancelled") {
return "border-r-red-500";
return "border-r-red-500/40";
}
if (isExecutionInProgress(status)) {
return "border-r-amber-500";
return "border-r-amber-500/40";
}
return "border-r-border";
return "border-r-border/50";
}
export function formatStatus(status: RecipeExecutionStatus): string {

View file

@ -10,6 +10,7 @@ import { Button } from "@/components/ui/button";
import { Progress } from "@/components/ui/progress";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { cn } from "@/lib/utils";
import { resolveImagePreview } from "../../utils/image-preview";
import type {
RecipeExecutionRecord,
} from "../../execution-types";
@ -25,12 +26,9 @@ import {
formatCellValue,
formatDuration,
formatPercent,
formatStatus,
formatTimestamp,
isExpandableCellValue,
hasExpandableTextCell,
parseAnalysisColumns,
parseModelUsageRows,
statusTone,
truncateCellValue,
} from "./executions-view-helpers";
@ -51,6 +49,10 @@ export function ExecutionsView({
onCancelExecution,
onLoadDatasetPage,
}: ExecutionsViewProps): ReactElement {
const formatEta = (value: number | null | undefined): string =>
typeof value === "number" && Number.isFinite(value)
? `${value.toLocaleString()} s`
: "--";
const [detailTab, setDetailTab] = useState("overview");
const [hiddenDatasetColumnsByExecution, setHiddenDatasetColumnsByExecution] = useState<
Record<string, string[]>
@ -119,10 +121,33 @@ export function ExecutionsView({
header: name,
cell: ({ getValue, row }) => {
const rawValue = getValue();
const imagePreview = resolveImagePreview(rawValue);
if (imagePreview?.kind === "ready") {
return (
<div className="max-w-[32rem]">
<img
src={imagePreview.src}
alt={`${name} preview`}
loading="lazy"
className="h-24 w-auto max-w-[260px] rounded-md border border-border/60 bg-muted/20 object-contain"
/>
</div>
);
}
if (imagePreview?.kind === "too_large") {
return (
<div className="max-w-[32rem]">
<p className="text-xs text-muted-foreground">
Image too large to preview
</p>
</div>
);
}
const value = formatCellValue(rawValue);
const rowExpanded = Boolean(expandedDatasetRows[row.id]);
const rowHasExpandableCell = visibleDatasetColumnNames.some((columnName) =>
isExpandableCellValue(formatCellValue(row.original[columnName])),
const rowHasExpandableCell = hasExpandableTextCell(
row.original,
visibleDatasetColumnNames,
);
const showTruncated = rowHasExpandableCell && !rowExpanded;
@ -259,9 +284,32 @@ export function ExecutionsView({
return formatDuration(selectedExecution.createdAt, selectedExecution.finishedAt);
}, [selectedExecution]);
const showSummaryCards = selectedExecution?.status === "completed";
const showProgressPanel =
selectedExecution?.status === "completed" ||
(selectedExecution ? isExecutionInProgress(selectedExecution.status) : false);
const hasProgressSnapshot = Boolean(
selectedExecution?.progress &&
(typeof selectedExecution.progress.done === "number" ||
typeof selectedExecution.progress.total === "number" ||
typeof selectedExecution.progress.percent === "number" ||
typeof selectedExecution.progress.rate === "number" ||
typeof selectedExecution.progress.eta_sec === "number"),
) || Boolean(
selectedExecution?.column_progress &&
(typeof selectedExecution.column_progress.done === "number" ||
typeof selectedExecution.column_progress.total === "number" ||
typeof selectedExecution.column_progress.percent === "number"),
) || Boolean(
selectedExecution?.batch &&
(typeof selectedExecution.batch.idx === "number" ||
typeof selectedExecution.batch.total === "number"),
);
const selectedStatus = selectedExecution?.status ?? null;
const isSelectedExecutionInProgress = selectedStatus
? isExecutionInProgress(selectedStatus)
: false;
const showProgressPanel = Boolean(selectedExecution) && (
selectedStatus === "completed" ||
isSelectedExecutionInProgress ||
hasProgressSnapshot
);
const progressComplete = selectedExecution?.status === "completed";
const progressPercent = selectedExecution?.progress?.percent ?? (progressComplete ? 100 : 0);
const batchTotal = selectedExecution?.batch?.total ?? null;
@ -305,49 +353,13 @@ export function ExecutionsView({
/>
<section className="min-w-0 flex-1 overflow-auto p-4">
{!selectedExecution ? (
<div className="rounded-xl border border-dashed p-4 text-sm text-muted-foreground">
<div className="rounded-xl border border-dashed border-border/60 p-4 text-sm text-muted-foreground">
Select an execution.
</div>
) : (
<div className="space-y-4">
<div className="flex flex-wrap items-center gap-2 text-xs text-muted-foreground">
<span className="font-medium capitalize">{selectedExecution.kind} execution</span>
<Badge
variant="secondary"
className={cn("capitalize", statusTone(selectedExecution.status))}
>
{formatStatus(selectedExecution.status)}
</Badge>
<span>{selectedExecution.rows} rows</span>
<span>Started {formatTimestamp(selectedExecution.createdAt)}</span>
<span>
Duration {formatDuration(selectedExecution.createdAt, selectedExecution.finishedAt)}
</span>
{selectedExecution.stage && (
<span>
Stage: {selectedExecution.stage}
{selectedExecution.current_column
? ` | Column: ${selectedExecution.current_column}`
: ""}
</span>
)}
{showBatchProgress && (
<span>
Batch {batchIdx ?? "--"}/{batchTotal}
</span>
)}
{isStale && <Badge variant="outline">Recipe changed since this run</Badge>}
</div>
{showProgressPanel && (
<div
className={cn(
"space-y-3 rounded-xl border p-3",
progressComplete
? "border-emerald-200 bg-emerald-50/50 dark:border-emerald-900/50 dark:bg-emerald-950/25"
: "border-amber-200 bg-amber-50/50 dark:border-amber-900/50 dark:bg-amber-950/25",
)}
>
<div className="space-y-3 rounded-2xl border shadow-border border-border/60 bg-card/55 p-3">
<div className="flex items-center justify-between">
<div className="flex items-center gap-2">
<HugeiconsIcon
@ -359,51 +371,29 @@ export function ExecutionsView({
: "text-amber-700 dark:text-amber-300",
)}
/>
<p
className={cn(
"text-sm font-semibold",
progressComplete
? "text-emerald-900 dark:text-emerald-100"
: "text-amber-900 dark:text-amber-100",
)}
>
{progressComplete ? "Run completed" : "Run in progress"}
<p className="text-sm font-semibold text-foreground">
Progress
</p>
</div>
<p
className={cn(
"text-xs",
progressComplete
? "text-emerald-800 dark:text-emerald-200"
: "text-amber-800 dark:text-amber-200",
)}
>
{formatPercent(progressPercent)}
<p className="text-xs text-muted-foreground">{formatPercent(progressPercent)}</p>
</div>
<Progress value={progressPercent} className="h-1" />
<div className="grid gap-2 text-xs md:grid-cols-4">
<p className="text-muted-foreground">
Done: <span className="text-foreground">{selectedExecution.progress?.done ?? "--"}</span>
</p>
<p className="text-muted-foreground">
Total: <span className="text-foreground">{selectedExecution.progress?.total ?? "--"}</span>
</p>
<p className="text-muted-foreground">
Rate: <span className="text-foreground">{selectedExecution.progress?.rate ?? "--"} rec/s</span>
</p>
<p className="text-muted-foreground">
ETA: <span className="text-foreground">{formatEta(selectedExecution.progress?.eta_sec)}</span>
</p>
</div>
<Progress value={progressPercent} />
<div
className={cn(
"grid gap-2 text-xs md:grid-cols-4",
progressComplete
? "text-emerald-900 dark:text-emerald-100"
: "text-amber-900 dark:text-amber-100",
)}
>
<p>Done: {selectedExecution.progress?.done ?? "--"}</p>
<p>Total: {selectedExecution.progress?.total ?? "--"}</p>
<p>Rate: {selectedExecution.progress?.rate ?? "--"} rec/s</p>
<p>ETA: {selectedExecution.progress?.eta_sec ?? "--"} s</p>
</div>
{selectedExecution.current_column && selectedExecution.column_progress && (
<p
className={cn(
"text-xs",
progressComplete
? "text-emerald-900 dark:text-emerald-100"
: "text-amber-900 dark:text-amber-100",
)}
>
<p className="text-xs text-muted-foreground">
Column {selectedExecution.current_column}:{" "}
{selectedExecution.column_progress.done ?? "--"}/
{selectedExecution.column_progress.total ?? "--"} (
@ -411,17 +401,11 @@ export function ExecutionsView({
</p>
)}
{showBatchProgress && (
<p
className={cn(
"text-xs",
progressComplete
? "text-emerald-900 dark:text-emerald-100"
: "text-amber-900 dark:text-amber-100",
)}
>
<p className="text-xs text-muted-foreground">
Processed batch: {batchIdx ?? "--"}/{batchTotal}
</p>
)}
{isStale && <Badge variant="outline">Recipe changed since this run</Badge>}
</div>
)}
@ -439,118 +423,115 @@ export function ExecutionsView({
</div>
)}
{(selectedExecution.status === "completed" ||
isExecutionInProgress(selectedExecution.status)) && (
<Tabs value={detailTab} onValueChange={setDetailTab}>
<div className="flex items-center justify-between gap-2">
<TabsList>
<TabsTrigger value="overview">Overview</TabsTrigger>
<TabsTrigger value="columns">Columns</TabsTrigger>
<TabsTrigger value="data">Data</TabsTrigger>
<TabsTrigger value="raw">Raw</TabsTrigger>
</TabsList>
{canCancel && (
<Button
type="button"
size="sm"
variant="outline"
onClick={() => onCancelExecution(selectedExecution.id)}
>
Cancel
</Button>
)}
</div>
<TabsContent value="overview">
<ExecutionOverviewTab
execution={selectedExecution}
showSummaryCards={showSummaryCards}
recordsMetric={recordsMetric}
totalMetric={totalMetric}
runDuration={runDuration}
columnCount={columnCount}
llmColumnCount={llmColumnCount}
nullRate={nullRate}
sideEffects={sideEffects}
lowUniquenessColumns={lowUniquenessColumns}
modelUsageRows={modelUsageRows}
terminalLines={terminalLines}
terminalRef={terminalRef}
onTerminalScroll={(event) => {
const element = event.currentTarget;
const distanceFromBottom =
element.scrollHeight - element.scrollTop - element.clientHeight;
shouldStickTerminalToBottomRef.current =
distanceFromBottom <= TERMINAL_STICKY_BOTTOM_THRESHOLD_PX;
}}
/>
</TabsContent>
<TabsContent value="columns">
<ExecutionColumnsTab analysisColumns={analysisColumns} />
</TabsContent>
<TabsContent value="data">
<ExecutionDataTab
execution={selectedExecution}
datasetColumnNames={datasetColumnNames}
hiddenDatasetColumns={hiddenDatasetColumns}
canPageDataset={canPageDataset}
currentDatasetPage={currentDatasetPage}
totalPages={totalPages}
tableColumns={tableColumns}
datasetRowsForTable={datasetRowsForTable}
visibleDatasetColumnNames={visibleDatasetColumnNames}
expandedDatasetRows={expandedDatasetRows}
selectedExecutionIdSafe={selectedExecutionIdSafe}
onSetHiddenColumns={(updater) => {
<Tabs value={detailTab} onValueChange={setDetailTab}>
<div className="flex items-center justify-between gap-2">
<TabsList className="border border-border/60 bg-card/40">
<TabsTrigger value="overview">Overview</TabsTrigger>
<TabsTrigger value="columns">Columns</TabsTrigger>
<TabsTrigger value="data">Data</TabsTrigger>
<TabsTrigger value="raw">Raw</TabsTrigger>
</TabsList>
{canCancel && (
<Button
type="button"
size="sm"
variant="outline"
onClick={() => onCancelExecution(selectedExecution.id)}
>
Cancel
</Button>
)}
</div>
<TabsContent value="overview">
<ExecutionOverviewTab
execution={selectedExecution}
showSummaryCards={showSummaryCards}
recordsMetric={recordsMetric}
totalMetric={totalMetric}
runDuration={runDuration}
columnCount={columnCount}
llmColumnCount={llmColumnCount}
nullRate={nullRate}
sideEffects={sideEffects}
lowUniquenessColumns={lowUniquenessColumns}
modelUsageRows={modelUsageRows}
terminalLines={terminalLines}
terminalRef={terminalRef}
onTerminalScroll={(event) => {
const element = event.currentTarget;
const distanceFromBottom =
element.scrollHeight - element.scrollTop - element.clientHeight;
shouldStickTerminalToBottomRef.current =
distanceFromBottom <= TERMINAL_STICKY_BOTTOM_THRESHOLD_PX;
}}
/>
</TabsContent>
<TabsContent value="columns">
<ExecutionColumnsTab analysisColumns={analysisColumns} />
</TabsContent>
<TabsContent value="data">
<ExecutionDataTab
execution={selectedExecution}
datasetColumnNames={datasetColumnNames}
hiddenDatasetColumns={hiddenDatasetColumns}
canPageDataset={canPageDataset}
currentDatasetPage={currentDatasetPage}
totalPages={totalPages}
tableColumns={tableColumns}
datasetRowsForTable={datasetRowsForTable}
visibleDatasetColumnNames={visibleDatasetColumnNames}
expandedDatasetRows={expandedDatasetRows}
selectedExecutionIdSafe={selectedExecutionIdSafe}
onSetHiddenColumns={(updater) => {
const selectedId = selectedExecution.id;
setHiddenDatasetColumnsByExecution((current) => {
const currentColumns = current[selectedId] ?? [];
return {
...current,
[selectedId]: updater(currentColumns),
};
});
}}
onPrevPage={() => {
if (selectedExecution.kind === "preview") {
const selectedId = selectedExecution.id;
setHiddenDatasetColumnsByExecution((current) => {
const currentColumns = current[selectedId] ?? [];
return {
...current,
[selectedId]: updater(currentColumns),
};
});
}}
onPrevPage={() => {
if (selectedExecution.kind === "preview") {
const selectedId = selectedExecution.id;
setPreviewDatasetPageByExecution((current) => ({
...current,
[selectedId]: Math.max(1, currentDatasetPage - 1),
}));
return;
}
onLoadDatasetPage(selectedExecution.id, currentDatasetPage - 1);
}}
onNextPage={() => {
if (selectedExecution.kind === "preview") {
const selectedId = selectedExecution.id;
setPreviewDatasetPageByExecution((current) => ({
...current,
[selectedId]: Math.min(totalPages, currentDatasetPage + 1),
}));
return;
}
onLoadDatasetPage(selectedExecution.id, currentDatasetPage + 1);
}}
onToggleRowExpanded={(rowId) => {
setExpandedDatasetRowsByExecution((current) => {
const rows = current[selectedExecution.id] ?? {};
return {
...current,
[selectedExecution.id]: {
...rows,
[rowId]: !rows[rowId],
},
};
});
}}
/>
</TabsContent>
<TabsContent value="raw">
<ExecutionRawTab rawExecution={rawExecution} />
</TabsContent>
</Tabs>
)}
setPreviewDatasetPageByExecution((current) => ({
...current,
[selectedId]: Math.max(1, currentDatasetPage - 1),
}));
return;
}
onLoadDatasetPage(selectedExecution.id, currentDatasetPage - 1);
}}
onNextPage={() => {
if (selectedExecution.kind === "preview") {
const selectedId = selectedExecution.id;
setPreviewDatasetPageByExecution((current) => ({
...current,
[selectedId]: Math.min(totalPages, currentDatasetPage + 1),
}));
return;
}
onLoadDatasetPage(selectedExecution.id, currentDatasetPage + 1);
}}
onToggleRowExpanded={(rowId) => {
setExpandedDatasetRowsByExecution((current) => {
const rows = current[selectedExecution.id] ?? {};
return {
...current,
[selectedExecution.id]: {
...rows,
[rowId]: !rows[rowId],
},
};
});
}}
/>
</TabsContent>
<TabsContent value="raw">
<ExecutionRawTab rawExecution={rawExecution} />
</TabsContent>
</Tabs>
</div>
)}
</section>

View file

@ -25,12 +25,14 @@ type PromptInputNodeData = {
llmId: string;
field: PromptField;
title: string;
executionLocked?: boolean;
};
type JudgeScoreNodeData = {
kind: "llm-judge-score";
llmId: string;
scoreIndex: number;
executionLocked?: boolean;
};
export type RecipeGraphAuxNodeData = PromptInputNodeData | JudgeScoreNodeData;
@ -81,6 +83,7 @@ function AuxNodeBase({
if (!(config && config.kind === "llm")) {
return null;
}
const executionLocked = Boolean(data.executionLocked);
const sourceHandles = (
<>
@ -135,6 +138,7 @@ function AuxNodeBase({
className="corner-squircle nodrag nowheel max-h-40 min-h-[88px] w-full resize-none overflow-y-auto text-xs"
aria-invalid={hasInvalidRefs}
value={value}
disabled={executionLocked}
onChange={(event) =>
updateConfig(data.llmId, {
[data.field]: event.target.value,
@ -193,7 +197,14 @@ function AuxNodeBase({
<BaseNodeHeaderTitle className="text-xs">
{score.name.trim() || `Scorer ${data.scoreIndex + 1}`}
</BaseNodeHeaderTitle>
<Button type="button" size="xs" variant="ghost" className="nodrag" onClick={removeScore}>
<Button
type="button"
size="xs"
variant="ghost"
className="nodrag"
disabled={executionLocked}
onClick={removeScore}
>
Remove
</Button>
</BaseNodeHeader>
@ -202,12 +213,14 @@ function AuxNodeBase({
className="nodrag h-7 w-full text-xs"
placeholder="Score name"
value={score.name}
disabled={executionLocked}
onChange={(event) => updateScore({ name: event.target.value })}
/>
<Textarea
className="corner-squircle nodrag nowheel max-h-32 min-h-[56px] w-full resize-none overflow-y-auto text-xs"
placeholder="Score description"
value={score.description}
disabled={executionLocked}
onChange={(event) => updateScore({ description: event.target.value })}
/>
<div className="space-y-1">
@ -217,6 +230,7 @@ function AuxNodeBase({
className="nodrag h-7 text-xs"
placeholder="Value"
value={option.value}
disabled={executionLocked}
onChange={(event) =>
updateOption(optionIndex, { value: event.target.value })
}
@ -225,6 +239,7 @@ function AuxNodeBase({
className="nodrag h-7 text-xs"
placeholder="Description"
value={option.description}
disabled={executionLocked}
onChange={(event) =>
updateOption(optionIndex, {
description: event.target.value,
@ -236,13 +251,21 @@ function AuxNodeBase({
size="xs"
variant="ghost"
className="nodrag"
disabled={executionLocked}
onClick={() => removeOption(optionIndex)}
>
x
</Button>
</div>
))}
<Button type="button" size="xs" variant="outline" className="nodrag mt-1" onClick={addOption}>
<Button
type="button"
size="xs"
variant="outline"
className="nodrag mt-1"
disabled={executionLocked}
onClick={addOption}
>
Add option
</Button>
</div>

View file

@ -81,6 +81,9 @@ const NODE_META = {
llm: {
tone: "bg-sky-50 text-sky-600 border-sky-100",
},
validator: {
tone: "bg-rose-50 text-rose-600 border-rose-100",
},
expression: {
tone: "bg-indigo-50 text-indigo-600 border-indigo-100",
},
@ -97,6 +100,8 @@ const NODE_META = {
tone: "bg-orange-50 text-orange-600 border-orange-100",
},
} as const;
const USER_NODE_TONE =
"bg-amber-50 text-amber-700 border-amber-100 dark:bg-amber-950/30 dark:text-amber-300 dark:border-amber-900/60";
const SAMPLER_ICONS: Record<SamplerType, IconType> = {
category: Tag01Icon,
@ -128,6 +133,9 @@ function resolveNodeIcon(
if (kind === "llm" && blockType in LLM_ICONS) {
return LLM_ICONS[blockType as LlmType];
}
if (kind === "validator") {
return Shield02Icon;
}
if (kind === "expression") {
return FunctionIcon;
}
@ -198,6 +206,14 @@ function getConfigSummary(config: NodeConfig | undefined): string {
return "Prompt/system via linked input nodes";
}
if (config.kind === "validator") {
const target = config.target_columns[0]?.trim();
if (target) {
return `Target: ${target}`;
}
return "Pick LLM code target";
}
if (config.kind === "seed") {
const seedSourceType = config.seed_source_type ?? "hf";
if (seedSourceType === "hf" && config.hf_repo_id.trim()) {
@ -288,6 +304,8 @@ function RecipeGraphNodeBase({
(state) => state.setLlmAuxVisibility,
);
const updateNodeInternals = useUpdateNodeInternals();
const executionLocked = Boolean(data.executionLocked);
const runtimeState = data.runtimeState ?? "idle";
useEffect(() => {
updateNodeInternals(id);
@ -329,12 +347,15 @@ function RecipeGraphNodeBase({
const showDataHandles =
data.kind === "llm" ||
data.kind === "validator" ||
data.kind === "expression" ||
data.kind === "sampler" ||
data.kind === "seed";
const showSemanticIn = data.kind === "model_config";
const showSemanticIn = data.kind === "model_config" || data.kind === "validator";
const showSemanticOut =
data.kind === "model_config" || data.kind === "model_provider";
data.kind === "model_config" ||
data.kind === "model_provider" ||
data.kind === "validator";
const summary = getConfigSummary(config);
const nodeBody = renderNodeBody(config, summary, updateConfig);
const canShowLlmAux =
@ -342,9 +363,34 @@ function RecipeGraphNodeBase({
(Boolean(config.prompt.trim()) ||
Boolean(config.system_prompt.trim()) ||
Boolean((config.scores?.length ?? 0) > 0));
const iconTone =
config?.kind === "sampler" &&
(config.sampler_type === "person" ||
config.sampler_type === "person_from_faker")
? USER_NODE_TONE
: meta.tone;
const runtimeNodeTone =
runtimeState === "running"
? "border-primary/70 ring-2 ring-primary/20 shadow-md"
: runtimeState === "done"
? "border-emerald-500/60 ring-1 ring-emerald-500/20"
: "";
return (
<BaseNode className="corner-squircle relative w-full min-w-0 overflow-visible rounded-lg border-border/60 shadow-sm">
<BaseNode
className={cn(
"corner-squircle relative w-full min-w-0 overflow-visible rounded-lg border-border/60 shadow-sm",
runtimeNodeTone,
)}
>
{runtimeState === "running" && config?.kind === "llm" && (
<div className="pointer-events-none absolute -top-7 right-2 z-20">
<span
className="block size-6 animate-spin rounded-full border-[3px] border-primary/90 border-t-transparent bg-background"
aria-label="Running"
/>
</div>
)}
<NodeResizer
isVisible={selected}
minWidth={MIN_NODE_WIDTH}
@ -362,7 +408,7 @@ function RecipeGraphNodeBase({
<div
className={cn(
"corner-squircle flex size-7 items-center justify-center rounded-md border",
meta.tone,
iconTone,
)}
>
<HugeiconsIcon icon={icon} className="size-3.5" />
@ -383,6 +429,7 @@ function RecipeGraphNodeBase({
size="xs"
variant="ghost"
className="nodrag"
disabled={executionLocked}
onClick={(event) => {
event.preventDefault();
event.stopPropagation();
@ -397,6 +444,7 @@ function RecipeGraphNodeBase({
size="xs"
variant="ghost"
className="nodrag"
disabled={executionLocked}
onClick={(event) => {
event.preventDefault();
event.stopPropagation();
@ -408,7 +456,12 @@ function RecipeGraphNodeBase({
</div>
</BaseNodeHeader>
<BaseNodeContent className="gap-2 px-3 py-2">
<BaseNodeContent
className={cn(
"gap-2 px-3 py-2",
executionLocked && "pointer-events-none opacity-85",
)}
>
{nodeBody}
</BaseNodeContent>

View file

@ -12,7 +12,9 @@ export const RecipeGraphSemanticEdge = memo(function RecipeGraphSemanticEdge({
style,
markerEnd,
selected,
data,
}: EdgeProps): ReactElement {
const isActive = Boolean((data as { active?: boolean } | undefined)?.active);
const [path] = getSmoothStepPath({
sourceX,
sourceY,
@ -30,12 +32,10 @@ export const RecipeGraphSemanticEdge = memo(function RecipeGraphSemanticEdge({
path={path}
markerEnd={markerEnd}
style={{
strokeDasharray: selected ? "7 5" : "6 5",
strokeWidth: selected ? 2.3 : 1.8,
stroke: selected
? "hsl(var(--primary) / 0.9)"
: "hsl(var(--foreground) / 0.38)",
opacity: selected ? 1 : 0.92,
strokeDasharray: isActive ? "8 6" : selected ? "7 5" : "6 5",
strokeWidth: isActive ? 2.4 : selected ? 2.3 : 1.8,
stroke: isActive || selected ? "var(--primary)" : "var(--muted-foreground)",
opacity: isActive ? 1 : selected ? 0.95 : 0.62,
...style,
}}
/>

View file

@ -11,6 +11,7 @@ import {
export type DataEdge = Edge<{
path?: "auto" | "bezier" | "smoothstep" | "step" | "straight";
active?: boolean;
}>;
export function DataEdge({
@ -29,6 +30,7 @@ export function DataEdge({
const resolvedPathType = resolvePathType({
type: data.path ?? "auto",
});
const isActive = Boolean(data.active);
const [edgePath] = getPath({
type: resolvedPathType,
sourceX,
@ -40,16 +42,20 @@ export function DataEdge({
});
const edgeStyle = {
stroke: selected
? "hsl(var(--primary) / 0.92)"
: "hsl(var(--foreground) / 0.42)",
strokeWidth: selected ? 2.6 : 2.1,
opacity: selected ? 1 : 0.92,
stroke: isActive || selected ? "var(--primary)" : "var(--muted-foreground)",
strokeWidth: isActive ? 2.6 : selected ? 2.6 : 2.1,
opacity: isActive ? 1 : selected ? 0.96 : 0.7,
strokeDasharray: isActive ? "8 6" : undefined,
...style,
};
return (
<BaseEdge id={id} path={edgePath} markerEnd={markerEnd} style={edgeStyle} />
<BaseEdge
id={id}
path={edgePath}
markerEnd={markerEnd}
style={edgeStyle}
/>
);
}

View file

@ -0,0 +1,146 @@
import {
ArrowDown01Icon,
ArrowUp01Icon,
CheckmarkCircle02Icon,
Flag02Icon,
} from "@hugeicons/core-free-icons";
import { HugeiconsIcon } from "@hugeicons/react";
import type { ReactElement } from "react";
import { Button } from "@/components/ui/button";
import { Progress } from "@/components/ui/progress";
import { Spinner } from "@/components/ui/spinner";
import { cn } from "@/lib/utils";
import type { RecipeExecutionRecord } from "../../execution-types";
import { isExecutionInProgress } from "../../executions/execution-helpers";
import {
formatMetricValue,
formatPercent,
} from "../executions/executions-view-helpers";
type ExecutionProgressIslandProps = {
execution: RecipeExecutionRecord;
currentColumnIcon: typeof Flag02Icon;
minimized: boolean;
onMinimizedChange: (value: boolean) => void;
onViewExecutions: () => void;
};
function formatEta(value: number | null | undefined): string {
const metric = formatMetricValue(value);
if (metric === "--") {
return "--";
}
return `${metric}s`;
}
function statusLabel(input: {
complete: boolean;
inProgress: boolean;
}): string {
if (input.complete) {
return "Run completed";
}
if (input.inProgress) {
return "Run in progress";
}
return "Run status";
}
export function ExecutionProgressIsland({
execution,
currentColumnIcon,
minimized,
onMinimizedChange,
onViewExecutions,
}: ExecutionProgressIslandProps): ReactElement {
const complete = execution.status === "completed";
const inProgress = isExecutionInProgress(execution.status);
const progressPercent = execution.progress?.percent ?? (complete ? 100 : 0);
const hasProgressSignal = Boolean(
execution.progress &&
(typeof execution.progress.done === "number" ||
typeof execution.progress.total === "number" ||
typeof execution.progress.percent === "number" ||
typeof execution.progress.rate === "number" ||
typeof execution.progress.eta_sec === "number"),
);
const showLoadingSpinner = inProgress && !hasProgressSignal;
const batchTotal = execution.batch?.total ?? null;
const showBatch = typeof batchTotal === "number" && batchTotal > 1;
return (
<div
className={cn(
"w-[clamp(15rem,26vw,20rem)] rounded-b-xl border-x border-b bg-card/96 shadow-sm backdrop-blur-sm transition-all",
minimized ? "min-h-[3rem]" : "min-h-[8.5rem]",
)}
>
<div className="flex items-center justify-between gap-2 px-3 py-2">
<div className="flex min-w-0 items-center gap-2">
<HugeiconsIcon
icon={complete ? CheckmarkCircle02Icon : Flag02Icon}
className={cn(
"size-3.5",
complete
? "text-emerald-700 dark:text-emerald-300"
: "text-amber-700 dark:text-amber-300",
)}
/>
<p className="truncate text-xs font-medium text-foreground">
{statusLabel({ complete, inProgress })}
</p>
</div>
<div className="flex items-center gap-2">
{showLoadingSpinner && (
<Spinner className="size-3.5 text-muted-foreground" />
)}
<span className="text-[11px] text-muted-foreground">{formatPercent(progressPercent)}</span>
<button
type="button"
onClick={() => onMinimizedChange(!minimized)}
className="inline-flex h-5 w-5 items-center justify-center rounded border border-border/70 text-muted-foreground transition hover:bg-muted/50"
aria-label={minimized ? "Expand progress" : "Minimize progress"}
title={minimized ? "Expand" : "Minimize"}
>
<HugeiconsIcon icon={minimized ? ArrowDown01Icon : ArrowUp01Icon} className="size-3" />
</button>
</div>
</div>
<div className="px-3">
<Progress value={progressPercent} className="h-1" />
</div>
{!minimized && (
<>
<div className="grid grid-cols-4 gap-2 px-3 pt-2 text-[11px] text-muted-foreground">
<p>Done: {formatMetricValue(execution.progress?.done)}</p>
<p>Total: {formatMetricValue(execution.progress?.total)}</p>
<p>Rate: {formatMetricValue(execution.progress?.rate)}</p>
<p>ETA: {formatEta(execution.progress?.eta_sec)}</p>
</div>
<div className="mt-1 flex items-center gap-1.5 px-3 text-[11px] text-muted-foreground">
<HugeiconsIcon icon={currentColumnIcon} className="size-3.5" />
<p className="truncate">Column: {execution.current_column ?? "--"}</p>
</div>
{showBatch && (
<div className="mt-1 px-3 text-[11px] text-muted-foreground">
Batch: {execution.batch?.idx ?? "--"}/{execution.batch?.total ?? "--"}
</div>
)}
<div className="px-3 pb-2 pt-2">
<Button
type="button"
variant="outline"
size="sm"
className="h-7 w-full text-[11px]"
onClick={onViewExecutions}
>
View more in executions view
</Button>
</div>
</>
)}
</div>
);
}

View file

@ -18,6 +18,7 @@ type ConfigDialogProps = {
datetimeOptions: string[];
onUpdate: (id: string, patch: Partial<NodeConfig>) => void;
container?: HTMLDivElement | null;
readOnly?: boolean;
};
export function ConfigDialog({
@ -30,11 +31,13 @@ export function ConfigDialog({
datetimeOptions,
onUpdate,
container,
readOnly = false,
}: ConfigDialogProps): ReactElement {
const blockDefinition = getBlockDefinitionForConfig(config);
const showDropToggle =
config?.kind === "sampler" ||
config?.kind === "llm" ||
config?.kind === "validator" ||
config?.kind === "expression" ||
(config?.kind === "seed" &&
(config.seed_source_type ?? "hf") === "unstructured");
@ -63,30 +66,38 @@ export function ConfigDialog({
)}
{config && (
<div className="space-y-4">
<ValidationBanner config={config} />
{showDropToggle && (
<div className="flex items-center corner-squircle justify-between gap-3 rounded-2xl border border-border/60 px-3 py-2">
<div>
<p className="text-sm font-semibold">Drop from final dataset</p>
<p className="text-xs text-muted-foreground">
Keep for generation but omit from exported rows.
</p>
</div>
<Switch
checked={config.drop ?? false}
onCheckedChange={(value) => onUpdate(config.id, { drop: value })}
/>
{readOnly && (
<div className="rounded-lg border border-amber-500/30 bg-amber-500/10 px-3 py-2 text-xs text-amber-700 dark:text-amber-300">
Recipe locked while execution is active.
</div>
)}
{renderBlockDialog(
config,
open,
categoryOptions,
modelConfigAliases,
modelProviderOptions,
datetimeOptions,
onUpdate,
)}
<ValidationBanner config={config} />
<div className={readOnly ? "pointer-events-none opacity-75" : undefined}>
{showDropToggle && (
<div className="mb-2 flex items-center corner-squircle justify-between gap-3 rounded-2xl border border-border/60 px-3 pt-2 pb-4">
<div>
<p className="text-sm font-semibold">Drop from final dataset</p>
<p className="text-xs text-muted-foreground">
Keep for generation but omit from exported rows.
</p>
</div>
<Switch
checked={config.drop ?? false}
disabled={readOnly}
onCheckedChange={(value) => onUpdate(config.id, { drop: value })}
/>
</div>
)}
{renderBlockDialog(
config,
open,
categoryOptions,
modelConfigAliases,
modelProviderOptions,
datetimeOptions,
onUpdate,
)}
</div>
</div>
)}
<DialogFooter>

View file

@ -1,3 +1,8 @@
import {
Collapsible,
CollapsibleContent,
CollapsibleTrigger,
} from "@/components/ui/collapsible";
import {
Combobox,
ComboboxContent,
@ -13,10 +18,12 @@ import {
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { Switch } from "@/components/ui/switch";
import { Textarea } from "@/components/ui/textarea";
import { type ReactElement, type RefObject, useMemo } from "react";
import { useRecipeStudioStore } from "../../stores/recipe-studio";
import type { LlmConfig } from "../../types";
import { isLikelyImageValue } from "../../utils/image-preview";
import { findInvalidJinjaReferences } from "../../utils/refs";
import { getAvailableVariables } from "../../utils/variables";
import { AvailableVariables } from "../shared/available-variables";
@ -42,6 +49,15 @@ const CODE_LANG_OPTIONS = [
"sql:ansi",
];
const TRACE_MODE_OPTIONS = ["none", "last_message", "all_messages"] as const;
function normalizeTraceMode(value: string): LlmConfig["with_trace"] {
if (value === "last_message" || value === "all_messages") {
return value;
}
return "none";
}
type LlmGeneralTabProps = {
config: LlmConfig;
modelConfigAliases: string[];
@ -85,17 +101,72 @@ export function LlmGeneralTab({
.slice(0, 3)
.map((ref) => `{{ ${ref} }}`)
.join(", ");
const seedConfig = useMemo(
() => Object.values(configs).find((item) => item.kind === "seed"),
[configs],
);
const hasHfSeed = Boolean(
seedConfig && (seedConfig.seed_source_type ?? "hf") === "hf",
);
const seedColumns = seedConfig?.seed_columns ?? [];
const seedPreviewRows = seedConfig?.seed_preview_rows ?? [];
const imageColumnOptions = useMemo(() => {
if (seedColumns.length === 0) {
return [];
}
const detected = seedColumns.filter((columnName) => {
const lower = columnName.toLowerCase();
if (
lower.includes("image") ||
lower.includes("img") ||
lower.includes("photo") ||
lower.includes("picture") ||
lower.includes("base64") ||
lower.includes("url")
) {
return true;
}
return seedPreviewRows.some((row) => isLikelyImageValue(row[columnName]));
});
return detected.length > 0 ? detected : seedColumns;
}, [seedColumns, seedPreviewRows]);
const imageContext = config.image_context ?? {
enabled: false,
// biome-ignore lint/style/useNamingConvention: api schema
column_name: "",
};
const imageContextToggleId = `${config.id}-image-context-enabled`;
const imageContextColumnId = `${config.id}-image-context-column`;
const imageContextColumnOptions = useMemo(() => {
const preferred =
imageColumnOptions.length > 0 ? imageColumnOptions : seedColumns;
const deduped = Array.from(
new Set(preferred.map((value) => value.trim()).filter(Boolean)),
);
const selected = imageContext.column_name.trim();
if (selected && !deduped.includes(selected)) {
deduped.unshift(selected);
}
return deduped;
}, [imageColumnOptions, imageContext.column_name, seedColumns]);
const traceModeId = `${config.id}-trace-mode`;
const reasoningToggleId = `${config.id}-reasoning-content`;
const advancedOpen = config.advancedOpen === true;
return (
<div className="space-y-4">
<AvailableVariables configId={config.id} />
<NameField value={config.name} onChange={(value) => onUpdate({ name: value })} />
<NameField
value={config.name}
onChange={(value) => onUpdate({ name: value })}
/>
{(!hasModelConfigs || !hasModelProviders) && (
<div className="rounded-2xl border border-border/60 bg-muted/20 px-3 py-2 text-xs text-muted-foreground">
<p className="font-semibold text-foreground">Setup hint</p>
<p>
{!hasModelProviders && "Add a Model Provider block. "}
{!hasModelConfigs && "Add a Model Config block and pick its alias here."}
{!hasModelConfigs &&
"Add a Model Config block and pick its alias here."}
</p>
</div>
)}
@ -185,6 +256,69 @@ export function LlmGeneralTab({
</p>
)}
</div>
{hasHfSeed && (
<div className="space-y-2">
<div className="flex items-center justify-between gap-3">
<FieldLabel
label="Use image context"
htmlFor={imageContextToggleId}
hint="Attach one seed image column to this LLM call."
/>
<Switch
id={imageContextToggleId}
checked={imageContext.enabled}
onCheckedChange={(checked) => {
onUpdate({
image_context: {
...imageContext,
enabled: checked,
// biome-ignore lint/style/useNamingConvention: api schema
column_name:
checked && !imageContext.column_name
? (imageContextColumnOptions[0] ?? "")
: imageContext.column_name,
},
});
}}
/>
</div>
{imageContext.enabled && (
<div className="grid gap-2">
<FieldLabel
label="Image column"
htmlFor={imageContextColumnId}
hint="Pick the seed column that contains image data."
/>
<Select
value={imageContext.column_name || undefined}
onValueChange={(value) =>
onUpdate({
image_context: {
...imageContext,
// biome-ignore lint/style/useNamingConvention: api schema
column_name: value,
},
})
}
>
<SelectTrigger
className="nodrag w-full"
id={imageContextColumnId}
>
<SelectValue placeholder="Select image column" />
</SelectTrigger>
<SelectContent>
{imageContextColumnOptions.map((columnName) => (
<SelectItem key={columnName} value={columnName}>
{columnName}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
)}
</div>
)}
{config.llm_type === "structured" && (
<div className="grid gap-2">
<FieldLabel
@ -224,6 +358,66 @@ export function LlmGeneralTab({
</p>
)}
</div>
<Collapsible
open={advancedOpen}
onOpenChange={(open) => onUpdate({ advancedOpen: open })}
>
<CollapsibleTrigger asChild={true}>
<button
type="button"
className="flex w-full items-center justify-between text-left text-xs text-muted-foreground"
>
<span className="font-semibold uppercase">Advanced</span>
<span>{advancedOpen ? "Hide" : "Show"}</span>
</button>
</CollapsibleTrigger>
<CollapsibleContent className="mt-3 space-y-4">
<div className="grid gap-2">
<FieldLabel
label="Trace capture"
htmlFor={traceModeId}
hint="Adds {column}__trace for debugging/replay."
/>
<Select
value={config.with_trace ?? "none"}
onValueChange={(value) =>
onUpdate({
// biome-ignore lint/style/useNamingConvention: api schema
with_trace: normalizeTraceMode(value),
})
}
>
<SelectTrigger className="nodrag w-full" id={traceModeId}>
<SelectValue placeholder="Select trace mode" />
</SelectTrigger>
<SelectContent>
{TRACE_MODE_OPTIONS.map((traceMode) => (
<SelectItem key={traceMode} value={traceMode}>
{traceMode}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
<div className="flex items-center justify-between gap-3">
<FieldLabel
label="Extract reasoning content"
htmlFor={reasoningToggleId}
hint="Adds {column}__reasoning_content when model provides it."
/>
<Switch
id={reasoningToggleId}
checked={config.extract_reasoning_content === true}
onCheckedChange={(checked) =>
onUpdate({
// biome-ignore lint/style/useNamingConvention: api schema
extract_reasoning_content: checked,
})
}
/>
</div>
</CollapsibleContent>
</Collapsible>
</div>
);
}

View file

@ -1,4 +1,11 @@
import { Button } from "@/components/ui/button";
import {
Empty,
EmptyContent,
EmptyDescription,
EmptyHeader,
EmptyTitle,
} from "@/components/ui/empty";
import { Input } from "@/components/ui/input";
import { Textarea } from "@/components/ui/textarea";
import { type ReactElement } from "react";
@ -91,14 +98,26 @@ export function LlmScoresTab({
label="Scorers"
hint="Rubrics used by LLM Judge to score each generated row."
/>
<Button type="button" size="xs" variant="outline" onClick={addScore}>
Add scorer
</Button>
{scores.length > 0 && (
<Button type="button" size="xs" variant="outline" onClick={addScore}>
Add scorer
</Button>
)}
</div>
{scores.length === 0 && (
<p className="text-xs text-muted-foreground">
Add at least one scorer.
</p>
<Empty className="rounded-xl border border-dashed border-border/70 p-5">
<EmptyHeader>
<EmptyTitle className="text-sm">No scorers yet</EmptyTitle>
<EmptyDescription className="text-xs">
Add a scorer rubric before running judge generation.
</EmptyDescription>
</EmptyHeader>
<EmptyContent className="max-w-none">
<Button type="button" size="sm" onClick={addScore}>
Add first scorer
</Button>
</EmptyContent>
</Empty>
)}
{scores.map((score, index) => (
<div

View file

@ -1,3 +1,8 @@
import {
Collapsible,
CollapsibleContent,
CollapsibleTrigger,
} from "@/components/ui/collapsible";
import { Checkbox } from "@/components/ui/checkbox";
import {
Combobox,
@ -8,7 +13,8 @@ import {
ComboboxList,
} from "@/components/ui/combobox";
import { Input } from "@/components/ui/input";
import { type ReactElement, useRef } from "react";
import { Textarea } from "@/components/ui/textarea";
import { type ReactElement, useRef, useState } from "react";
import type { ModelConfig } from "../../types";
import { FieldLabel } from "../shared/field-label";
import { NameField } from "../shared/name-field";
@ -24,11 +30,14 @@ export function ModelConfigDialog({
providerOptions,
onUpdate,
}: ModelConfigDialogProps): ReactElement {
const [optionalOpen, setOptionalOpen] = useState(false);
const modelId = `${config.id}-model`;
const providerId = `${config.id}-provider`;
const tempId = `${config.id}-temperature`;
const topPId = `${config.id}-top-p`;
const maxTokensId = `${config.id}-max-tokens`;
const timeoutId = `${config.id}-timeout`;
const extraBodyId = `${config.id}-inference-extra-body`;
const providerAnchorRef = useRef<HTMLDivElement>(null);
const providerInputRef = useRef(config.provider);
const lastProviderRef = useRef(config.provider);
@ -46,6 +55,7 @@ export function ModelConfigDialog({
return (
<div className="space-y-4">
<NameField
label="Model alias"
value={config.name}
onChange={(value) => onUpdate({ name: value })}
/>
@ -114,7 +124,7 @@ export function ModelConfigDialog({
label="Inference"
hint="Runtime generation params for this model alias."
/>
<div className="grid grid-cols-3 gap-2">
<div className="grid grid-cols-2 gap-2 sm:grid-cols-4">
<Input
id={tempId}
className="nodrag"
@ -142,17 +152,55 @@ export function ModelConfigDialog({
updateField("inference_max_tokens", event.target.value)
}
/>
<Input
id={timeoutId}
className="nodrag"
placeholder="Timeout (sec)"
value={config.inference_timeout ?? ""}
onChange={(event) =>
updateField("inference_timeout", event.target.value)
}
/>
</div>
</div>
<label className="flex items-center gap-2 text-xs font-semibold uppercase text-muted-foreground">
<Checkbox
checked={config.skip_health_check ?? false}
onCheckedChange={(value) =>
updateField("skip_health_check", Boolean(value))
}
/>
Skip health check
</label>
<Collapsible open={optionalOpen} onOpenChange={setOptionalOpen}>
<CollapsibleTrigger asChild={true}>
<button
type="button"
className="flex w-full items-center justify-between text-left text-xs text-muted-foreground"
>
<span className="font-semibold uppercase">Optional</span>
<span>{optionalOpen ? "Hide" : "Show"}</span>
</button>
</CollapsibleTrigger>
<CollapsibleContent className="mt-3 space-y-4">
<div className="grid gap-2">
<FieldLabel
label="Inference extra body (JSON)"
htmlFor={extraBodyId}
hint="Optional request fields merged into inference parameters."
/>
<Textarea
id={extraBodyId}
className="corner-squircle nodrag"
placeholder='{"top_k": 20, "min_p": 0.0}'
value={config.inference_extra_body ?? ""}
onChange={(event) =>
updateField("inference_extra_body", event.target.value)
}
/>
</div>
<label className="flex items-center gap-2 text-xs font-semibold uppercase text-muted-foreground">
<Checkbox
checked={config.skip_health_check ?? false}
onCheckedChange={(value) =>
updateField("skip_health_check", Boolean(value))
}
/>
Skip health check
</label>
</CollapsibleContent>
</Collapsible>
</div>
);
}

View file

@ -35,6 +35,7 @@ export function ModelProviderDialog({
return (
<div className="space-y-4">
<NameField
label="Provider name"
value={config.name}
onChange={(value) => onUpdate({ name: value })}
/>

View file

@ -13,7 +13,15 @@ import {
} from "@/components/ui/dialog";
import { Input } from "@/components/ui/input";
import { Switch } from "@/components/ui/switch";
import { CookBookIcon, TestTube01Icon } from "@hugeicons/core-free-icons";
import { Badge } from "@/components/ui/badge";
import { cn } from "@/lib/utils";
import {
AlertCircleIcon,
CheckmarkCircle02Icon,
CookBookIcon,
SparklesIcon,
TestTube01Icon,
} from "@hugeicons/core-free-icons";
import { HugeiconsIcon } from "@hugeicons/react";
import { type ReactElement, useEffect, useState } from "react";
import type { RecipeExecutionKind } from "../execution-types";
@ -26,6 +34,8 @@ type RunDialogProps = {
kind: RecipeExecutionKind;
onKindChange: (kind: RecipeExecutionKind) => void;
rows: number;
fullRunName: string;
onFullRunNameChange: (name: string) => void;
onRowsChange: (rows: number) => void;
settings: RecipeRunSettings;
onSettingsChange: (patch: Partial<RecipeRunSettings>) => void;
@ -170,21 +180,43 @@ function ValidationResultPanel({
return (
<div
className={
className={cn(
"space-y-3 rounded-2xl border p-4 shadow-border backdrop-blur-sm",
validateResult.valid
? "space-y-1 rounded-xl border border-emerald-300 bg-emerald-50 p-3"
: "space-y-1 rounded-xl border border-destructive/30 bg-destructive/5 p-3"
}
? "border-emerald-300/70 bg-emerald-50/80 dark:border-emerald-900/60 dark:bg-emerald-950/30"
: "border-destructive/30 bg-destructive/5",
)}
>
<p
className={
validateResult.valid
? "text-xs font-semibold uppercase text-emerald-700"
: "text-xs font-semibold uppercase text-destructive"
}
>
{validateResult.valid ? "Validation passed" : "Validation failed"}
</p>
<div className="flex items-start gap-3">
<div
className={cn(
"mt-0.5 flex size-8 shrink-0 items-center justify-center rounded-full border",
validateResult.valid
? "border-emerald-300/70 bg-emerald-500/10 text-emerald-700 dark:border-emerald-900/60 dark:text-emerald-300"
: "border-destructive/30 bg-destructive/10 text-destructive",
)}
>
<HugeiconsIcon
icon={validateResult.valid ? CheckmarkCircle02Icon : AlertCircleIcon}
className="size-4"
/>
</div>
<div className="min-w-0 flex-1 space-y-1">
<p
className={cn(
"text-sm font-semibold",
validateResult.valid ? "text-emerald-700 dark:text-emerald-300" : "text-destructive",
)}
>
{validateResult.valid ? "Recipe looks good" : "Recipe needs attention"}
</p>
<p className="text-xs text-muted-foreground">
{validateResult.valid
? "Validation passed. You can start the run when ready."
: "Fix the issues below, then validate again."}
</p>
</div>
</div>
{!validateResult.valid && validateResult.errors.length > 0 && (
<div className="space-y-1">
{validateResult.errors.map((error) => (
@ -207,6 +239,8 @@ export function RunDialog({
kind,
onKindChange,
rows,
fullRunName,
onFullRunNameChange,
onRowsChange,
settings,
onSettingsChange,
@ -220,6 +254,8 @@ export function RunDialog({
}: RunDialogProps): ReactElement {
const [advancedOpen, setAdvancedOpen] = useState(false);
const kindLabel = kind === "preview" ? "Preview" : "Full run";
const normalizedFullRunName = fullRunName.trim();
const isFullRunNameMissing = kind === "full" && normalizedFullRunName.length === 0;
const rowHint =
kind === "preview"
? "How many sample rows to generate for a quick check."
@ -249,6 +285,8 @@ export function RunDialog({
const [shutdownRateDraft, setShutdownRateDraft] = useState(
String(settings.shutdownErrorRate),
);
const showBatchingHint =
kind === "full" && rows >= 1000 && !settings.batchEnabled;
useEffect(() => {
if (!open) {
@ -285,17 +323,40 @@ export function RunDialog({
position="absolute"
overlayPosition="absolute"
overlayClassName="bg-transparent"
className="corner-squircle sm:max-w-2xl shadow-border"
className="corner-squircle border-border/70 bg-background/95 sm:max-w-2xl shadow-border backdrop-blur-xl"
>
<DialogHeader>
<DialogHeader className="space-y-2">
<DialogTitle>{kindLabel} settings</DialogTitle>
<p className="text-sm text-muted-foreground">
Configure run size and performance knobs for this execution.
</p>
</DialogHeader>
<div className="flex items-center justify-between text-sm">
<span className="font-medium text-foreground">Preview mode</span>
{showBatchingHint && (
<div className="flex items-start gap-3 rounded-2xl border border-amber-300/70 bg-amber-50/80 p-4 shadow-border dark:border-amber-900/60 dark:bg-amber-950/30">
<div className="flex size-8 shrink-0 items-center justify-center rounded-full border border-amber-300/70 bg-amber-500/10 text-amber-700 dark:border-amber-900/60 dark:text-amber-300">
<HugeiconsIcon icon={SparklesIcon} className="size-4" />
</div>
<div className="space-y-1">
<p className="text-sm font-semibold text-amber-800 dark:text-amber-200">
Bigger runs usually feel smoother with batching on
</p>
<p className="text-xs leading-relaxed text-amber-900/80 dark:text-amber-100/80">
You&apos;re generating {rows.toLocaleString()} records. Turning on batching
usually makes larger runs easier to manage and more resilient if
something goes wrong mid-run.
</p>
</div>
</div>
)}
<div className="flex items-center justify-between rounded-2xl border border-border/70 bg-card/60 px-4 py-3 text-sm shadow-border">
<div className="space-y-0.5">
<span className="font-medium text-foreground">Preview mode</span>
<p className="text-xs text-muted-foreground">
Turn this off for a full dataset run.
</p>
</div>
<Switch
checked={kind === "preview"}
onCheckedChange={(checked) =>
@ -304,6 +365,29 @@ export function RunDialog({
/>
</div>
{kind === "full" && (
<div className="grid gap-2">
<FieldLabel
label="Run name"
htmlFor="run-name"
hint="Optional label shown in executions list."
/>
<Input
id="run-name"
type="text"
value={fullRunName}
onChange={(event) => onFullRunNameChange(event.target.value)}
placeholder="Sprint dataset v2"
aria-invalid={isFullRunNameMissing}
/>
{isFullRunNameMissing ? (
<p className="text-xs text-destructive">
Run name is required before starting a full run.
</p>
) : null}
</div>
)}
<div className="grid gap-4 md:grid-cols-2">
<div className="grid gap-2">
<FieldLabel label="Records" htmlFor="run-rows" hint={rowHint} />
@ -363,9 +447,14 @@ export function RunDialog({
</div>
{kind === "full" && (
<div className="space-y-3">
<div className="space-y-3 rounded-2xl border border-border/70 bg-card/60 p-4 shadow-border">
<div className="flex items-center justify-between gap-3 text-sm">
<span className="font-medium">Enable batching</span>
<div className="space-y-0.5">
<span className="font-medium">Enable batching</span>
<p className="text-xs text-muted-foreground">
Split big runs into manageable chunks.
</p>
</div>
<Switch
checked={settings.batchEnabled}
onCheckedChange={(checked) =>
@ -395,9 +484,12 @@ export function RunDialog({
}
/>
<div className="flex items-center justify-between gap-3 text-sm">
<span className="font-medium">
Merge batches to one parquet
</span>
<div className="space-y-0.5">
<span className="font-medium">Merge batches to one parquet</span>
<p className="text-xs text-muted-foreground">
Combine chunk outputs into one final file when done.
</p>
</div>
<Switch
checked={settings.mergeBatches}
onCheckedChange={(checked) =>
@ -420,7 +512,7 @@ export function RunDialog({
</button>
</CollapsibleTrigger>
<CollapsibleContent className="mt-3 space-y-4">
<div className="grid gap-4 md:grid-cols-2">
<div className="grid gap-4 rounded-2xl border border-border/70 bg-card/60 p-4 shadow-border md:grid-cols-2">
<DraftInputField
id="run-non-inference-workers"
label="CPU workers"
@ -515,7 +607,13 @@ export function RunDialog({
)
}
/>
<div className="flex items-center gap-3 text-sm text-foreground">
<div className="flex items-center justify-between gap-3 rounded-xl border border-border/60 bg-background/60 px-3 py-2 text-sm text-foreground md:col-span-2">
<div className="space-y-0.5">
<p className="font-medium">Keep running through failures</p>
<p className="text-xs text-muted-foreground">
Recommended for longer runs when you want maximum output.
</p>
</div>
<Switch
checked={settings.disableEarlyShutdown}
onCheckedChange={(checked) =>
@ -524,17 +622,19 @@ export function RunDialog({
})
}
/>
Disable early shutdown
</div>
</div>
</CollapsibleContent>
</Collapsible>
{errors.length > 0 && (
<div className="max-h-44 space-y-1 overflow-y-auto rounded-xl border border-destructive/30 bg-destructive/5 p-3">
<p className="text-xs font-semibold uppercase text-destructive">
Run checks
</p>
<div className="max-h-44 space-y-2 overflow-y-auto rounded-2xl border border-destructive/30 bg-destructive/5 p-4 shadow-border">
<div className="flex items-center gap-2">
<HugeiconsIcon icon={AlertCircleIcon} className="size-4 text-destructive" />
<Badge variant="outline" className="rounded-full text-[10px] text-destructive">
Run checks
</Badge>
</div>
{errors.map((error) => (
<p key={error} className="text-xs text-destructive">
{error}
@ -551,6 +651,7 @@ export function RunDialog({
variant="outline"
onClick={() => onOpenChange(false)}
disabled={loading}
className="corner-squircle border-border/70 bg-card/70"
>
Cancel
</Button>
@ -559,11 +660,17 @@ export function RunDialog({
variant="outline"
onClick={onValidate}
disabled={loading || validateLoading}
className="corner-squircle border-border/70 bg-card/70"
>
<HugeiconsIcon icon={TestTube01Icon} className="size-3.5" />
{validateLoading ? "Validating..." : "Validate recipe"}
</Button>
<Button type="button" onClick={onRun} disabled={loading}>
<Button
type="button"
onClick={onRun}
disabled={loading || isFullRunNameMissing}
className="corner-squircle"
>
<HugeiconsIcon icon={CookBookIcon} className="size-3.5" />
{loading ? "Starting..." : `Start ${kindLabel.toLowerCase()}`}
</Button>

View file

@ -44,7 +44,7 @@ export function CategoryDialog({
onUpdate,
}: CategoryDialogProps): ReactElement {
const [conditionDraft, setConditionDraft] = useState("");
const [advancedOpen, setAdvancedOpen] = useState(false);
const advancedOpen = config.advancedOpen === true;
const conditionInputId = `${config.id}-conditional-rule`;
const conditional = config.conditional_params ?? {};
const conditionalCount = Object.keys(conditional).length;
@ -112,7 +112,10 @@ export function CategoryDialog({
/>
</div>
</div>
<Collapsible open={advancedOpen} onOpenChange={setAdvancedOpen}>
<Collapsible
open={advancedOpen}
onOpenChange={(open) => onUpdate({ advancedOpen: open })}
>
<CollapsibleTrigger asChild={true}>
<button
type="button"

View file

@ -34,12 +34,12 @@ import {
TabsList,
TabsTrigger,
} from "@/components/ui/tabs";
import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters";
import mammoth from "mammoth";
import { type ReactElement, useCallback, useEffect, useMemo, useRef, useState } from "react";
import { extractText, getDocumentProxy } from "unpdf";
import { cn } from "@/lib/utils";
import { inspectSeedDataset, inspectSeedUpload } from "../../api";
import { resolveImagePreview } from "../../utils/image-preview";
import type {
SeedConfig,
SeedSamplingStrategy,
@ -102,6 +102,29 @@ function truncatePreviewValue(value: string): string {
return `${value.slice(0, PREVIEW_TRUNCATE_AT)}`;
}
function getPreviewEmptyStateCopy(mode: SeedConfig["seed_source_type"]): {
title: string;
description: string;
} {
if (mode === "local") {
return {
title: "No local preview yet",
description: "Choose a CSV/JSON/JSONL file, then click Load to fetch 10 rows.",
};
}
if (mode === "unstructured") {
return {
title: "No chunk preview yet",
description:
"Choose a TXT/PDF/DOCX file, then click Load to extract + preview chunk_text rows.",
};
}
return {
title: "No dataset preview yet",
description: "Pick a Hugging Face dataset and click Load to fetch 10 sample rows.",
};
}
function parseChunkNumber(
value: string | undefined,
fallback: number,
@ -137,22 +160,6 @@ function resolveChunking(config: SeedConfig): {
return { chunkSize, chunkOverlap };
}
async function chunkText(
input: string,
chunkSize: number,
chunkOverlap: number,
): Promise<string[]> {
const text = input.replace(/\r\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
if (!text) return [];
const splitter = new RecursiveCharacterTextSplitter({
chunkSize,
chunkOverlap,
});
const chunks = await splitter.splitText(text);
return chunks.map((chunk) => chunk.trim()).filter(Boolean);
}
async function fileToBase64Payload(file: File): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
@ -185,16 +192,34 @@ async function extractUnstructuredText(file: File): Promise<string> {
throw new Error("Unsupported unstructured file type");
}
async function toUnstructuredUploadFile(file: File): Promise<File> {
const lower = file.name.toLowerCase();
if (lower.endsWith(".txt") || lower.endsWith(".md")) {
return file;
}
const text = (await extractUnstructuredText(file)).trim();
if (!text) {
throw new Error("No text found in file.");
}
const normalized = text.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
const stem = file.name.replace(/\.(pdf|docx)$/i, "") || "unstructured_seed";
return new File([normalized], `${stem}.txt`, {
type: "text/plain",
});
}
export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactElement {
const [inspectError, setInspectError] = useState<string | null>(null);
const [isInspecting, setIsInspecting] = useState(false);
const [advancedOpen, setAdvancedOpen] = useState(false);
const advancedOpen = config.advancedOpen === true;
const [previewRows, setPreviewRows] = useState<Record<string, unknown>[]>([]);
const [expandedPreviewRows, setExpandedPreviewRows] = useState<Record<number, boolean>>({});
const [localFile, setLocalFile] = useState<File | null>(null);
const [unstructuredFile, setUnstructuredFile] = useState<File | null>(null);
const mode = config.seed_source_type ?? "hf";
const previewEmpty = getPreviewEmptyStateCopy(mode);
useEffect(() => {
setInspectError(null);
@ -252,7 +277,8 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
const response = await inspectSeedDataset({
dataset_name: datasetName,
hf_token: config.hf_token?.trim() || undefined,
subset: undefined,
split: config.hf_split?.trim() || undefined,
subset: config.hf_subset?.trim() || undefined,
preview_size: 10,
});
onUpdate({
@ -262,8 +288,8 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
response.columns.includes(name),
),
seed_preview_rows: response.preview_rows ?? [],
hf_split: "",
hf_subset: "",
hf_split: response.split ?? "",
hf_subset: response.subset ?? "",
local_file_name: "",
unstructured_file_name: "",
});
@ -310,27 +336,19 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
throw new Error("File too large (max 50MB).");
}
const text = await extractUnstructuredText(unstructuredFile);
const { chunkSize, chunkOverlap } = resolveChunking(config);
const chunks = await chunkText(text, chunkSize, chunkOverlap);
if (chunks.length === 0) {
throw new Error("No text found in file.");
const uploadFile = await toUnstructuredUploadFile(unstructuredFile);
if (uploadFile.size > MAX_UPLOAD_BYTES) {
throw new Error("Processed text is too large (max 50MB).");
}
const jsonl = chunks
.map((chunk) => JSON.stringify({ chunk_text: chunk }))
.join("\n");
const stem =
unstructuredFile.name.replace(/\.(pdf|docx|txt)$/i, "") ||
"unstructured_seed";
const jsonlFile = new File([jsonl], `${stem}.jsonl`, {
type: "application/json",
});
const payload = await fileToBase64Payload(jsonlFile);
const payload = await fileToBase64Payload(uploadFile);
const response = await inspectSeedUpload({
filename: jsonlFile.name,
filename: uploadFile.name,
content_base64: payload,
preview_size: 10,
seed_source_type: "unstructured",
unstructured_chunk_size: chunkSize,
unstructured_chunk_overlap: chunkOverlap,
});
onUpdate({
hf_path: response.resolved_path,
@ -393,6 +411,16 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
() => new Set(selectedSeedDropColumns),
[selectedSeedDropColumns],
);
const rowHasExpandableText = useCallback(
(row: Record<string, unknown>): boolean =>
previewColumns.some((columnName) => {
if (resolveImagePreview(row[columnName])) {
return false;
}
return isExpandablePreviewValue(stringifyCell(row[columnName]));
}),
[previewColumns],
);
return (
<Tabs defaultValue="config" className="w-full">
@ -538,7 +566,7 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
</Button>
</div>
<p className="text-xs text-muted-foreground">
Chunking uses chunk_text only. Max 50MB.
File is converted to text, then chunked server-side into chunk_text rows. Max 50MB.
</p>
{(unstructuredFile?.name ||
config.unstructured_file_name?.trim()) && (
@ -590,7 +618,10 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
</div>
)}
<Collapsible open={advancedOpen} onOpenChange={setAdvancedOpen}>
<Collapsible
open={advancedOpen}
onOpenChange={(openState) => onUpdate({ advancedOpen: openState })}
>
<CollapsibleTrigger asChild={true}>
<button
type="button"
@ -749,12 +780,14 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
<div className="flex w-full items-center justify-center">
<Empty className="max-w-lg">
<EmptyHeader>
<EmptyTitle>Seed preview</EmptyTitle>
<EmptyTitle>{previewEmpty.title}</EmptyTitle>
<EmptyDescription>
Use the load button next to the source input to fetch 10 rows.
{previewEmpty.description}
</EmptyDescription>
</EmptyHeader>
<EmptyContent />
<EmptyContent className="text-xs text-muted-foreground">
Preview appears here after loading source metadata.
</EmptyContent>
</Empty>
</div>
) : (
@ -778,15 +811,11 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
<TableRow
key={`row-${rowIdx}`}
className={cn(
previewColumns.some((col) =>
isExpandablePreviewValue(stringifyCell(row[col])),
) && "cursor-pointer hover:bg-primary/[0.06]",
rowHasExpandableText(row) && "cursor-pointer hover:bg-primary/[0.06]",
expandedPreviewRows[rowIdx] && "bg-primary/[0.05]",
)}
onClick={() => {
const canExpand = previewColumns.some((col) =>
isExpandablePreviewValue(stringifyCell(row[col])),
);
const canExpand = rowHasExpandableText(row);
if (!canExpand) {
return;
}
@ -802,10 +831,22 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
className="max-w-[260px] whitespace-pre-wrap break-words text-xs"
>
{(() => {
const imagePreview = resolveImagePreview(row[col]);
if (imagePreview?.kind === "ready") {
return (
<img
src={imagePreview.src}
alt={`${col} preview`}
loading="lazy"
className="h-20 w-auto max-w-[220px] rounded-md border border-border/60 bg-muted/20 object-contain"
/>
);
}
if (imagePreview?.kind === "too_large") {
return "Image too large to preview";
}
const value = stringifyCell(row[col]);
const rowHasExpandableCell = previewColumns.some((columnName) =>
isExpandablePreviewValue(stringifyCell(row[columnName])),
);
const rowHasExpandableCell = rowHasExpandableText(row);
const rowExpanded = Boolean(expandedPreviewRows[rowIdx]);
return rowHasExpandableCell && !rowExpanded
? truncatePreviewValue(value)

View file

@ -1,5 +1,5 @@
import { Badge } from "@/components/ui/badge";
import type { ReactElement } from "react";
import { type ReactElement, useMemo, useState } from "react";
import { useRecipeStudioStore } from "../../stores/recipe-studio";
import { getAvailableVariableEntries } from "../../utils/variables";
@ -7,11 +7,33 @@ type AvailableVariablesProps = {
configId: string;
};
const USER_EXPANDED_FIELDS = [
"first_name",
"last_name",
"sex",
"city",
"state",
"age",
] as const;
const USER_BADGE_CLASS =
"corner-squircle border-amber-500/25 bg-amber-500/10 font-mono text-[11px] text-amber-700 dark:text-amber-300";
export function AvailableVariables({
configId,
}: AvailableVariablesProps): ReactElement | null {
const [showUserFields, setShowUserFields] = useState(false);
const configs = useRecipeStudioStore((state) => state.configs);
const vars = getAvailableVariableEntries(configs, configId);
const variableNames = useMemo(() => new Set(vars.map((entry) => entry.name)), [vars]);
const hasUserRoot = variableNames.has("user");
const userFieldEntries = useMemo(
() =>
USER_EXPANDED_FIELDS.map((field) => ({
source: "column" as const,
name: `user.${field}`,
})).filter((entry) => !variableNames.has(entry.name)),
[variableNames],
);
if (vars.length === 0) return null;
@ -21,19 +43,48 @@ export function AvailableVariables({
Available references
</p>
<div className="flex flex-wrap gap-1.5">
{vars.map((v) => (
<Badge
key={`${v.source}:${v.name}`}
variant="secondary"
className={
v.source === "seed"
? "corner-squircle border-blue-500/25 bg-blue-500/10 font-mono text-[11px] text-blue-700 dark:text-blue-300"
: "corner-squircle font-mono text-[11px]"
}
>
{`{{ ${v.name} }}`}
</Badge>
))}
{vars.map((v) => {
const className =
v.name === "user" || v.name.startsWith("user.")
? USER_BADGE_CLASS
: v.source === "seed"
? "corner-squircle border-blue-500/25 bg-blue-500/10 font-mono text-[11px] text-blue-700 dark:text-blue-300"
: "corner-squircle font-mono text-[11px]";
if (v.name !== "user") {
return (
<Badge
key={`${v.source}:${v.name}`}
variant="secondary"
className={className}
>
{`{{ ${v.name} }}`}
</Badge>
);
}
return (
<button
key={`${v.source}:${v.name}`}
type="button"
onClick={() => setShowUserFields((prev) => !prev)}
className="cursor-pointer"
aria-expanded={showUserFields}
>
<Badge variant="secondary" className={className}>
{`{{ ${v.name} }}`}
</Badge>
</button>
);
})}
{hasUserRoot && showUserFields &&
userFieldEntries.map((entry) => (
<Badge
key={`user-expanded:${entry.name}`}
variant="secondary"
className={USER_BADGE_CLASS}
>
{`{{ ${entry.name} }}`}
</Badge>
))}
</div>
</div>
);

View file

@ -6,6 +6,7 @@ type NameFieldProps = {
id?: string;
value: string;
onChange: (value: string) => void;
label?: string;
hint?: string;
};
@ -13,6 +14,7 @@ export function NameField({
id,
value,
onChange,
label,
hint,
}: NameFieldProps): ReactElement {
const fallbackId = useId();
@ -20,7 +22,7 @@ export function NameField({
return (
<div className="grid gap-2">
<FieldLabel
label="Column name"
label={label ?? "Column name"}
htmlFor={inputId}
hint={
hint ??

View file

@ -0,0 +1,266 @@
import {
Collapsible,
CollapsibleContent,
CollapsibleTrigger,
} from "@/components/ui/collapsible";
import {
Combobox,
ComboboxContent,
ComboboxEmpty,
ComboboxInput,
ComboboxItem,
ComboboxList,
} from "@/components/ui/combobox";
import { Input } from "@/components/ui/input";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { type ReactElement, useMemo, useRef } from "react";
import { useRecipeStudioStore } from "../../stores/recipe-studio";
import type { ValidatorConfig } from "../../types";
import {
isValidatorCodeLang,
VALIDATOR_OXC_CODE_LANGS,
VALIDATOR_SQL_CODE_LANGS,
} from "../../utils/validators/code-lang";
import {
OXC_CODE_SHAPES,
normalizeOxcCodeShape,
} from "../../utils/validators/oxc-code-shape";
import {
OXC_VALIDATION_MODES,
normalizeOxcValidationMode,
} from "../../utils/validators/oxc-mode";
import { FieldLabel } from "../shared/field-label";
import { NameField } from "../shared/name-field";
type ValidatorDialogProps = {
config: ValidatorConfig;
onUpdate: (patch: Partial<ValidatorConfig>) => void;
};
const NONE_VALUE = "__none__";
export function ValidatorDialog({
config,
onUpdate,
}: ValidatorDialogProps): ReactElement {
const configs = useRecipeStudioStore((state) => state.configs);
const targetColumnId = `${config.id}-target-column`;
const oxcModeId = `${config.id}-oxc-mode`;
const oxcCodeShapeId = `${config.id}-oxc-code-shape`;
const batchSizeId = `${config.id}-batch-size`;
const oxcModeAnchorRef = useRef<HTMLDivElement>(null);
const oxcCodeShapeAnchorRef = useRef<HTMLDivElement>(null);
const advancedOpen = config.advancedOpen === true;
const selectedOxcMode = normalizeOxcValidationMode(config.oxc_validation_mode);
const selectedOxcCodeShape = normalizeOxcCodeShape(config.oxc_code_shape);
const codeOptions = useMemo(
() =>
Object.values(configs)
.flatMap((item) => {
if (!(item.kind === "llm" && item.llm_type === "code")) {
return [];
}
if (config.validator_type === "oxc") {
const lang = item.code_lang?.trim() ?? "";
if (!VALIDATOR_OXC_CODE_LANGS.includes(lang as typeof config.code_lang)) {
return [];
}
} else {
const lang = item.code_lang?.trim() ?? "";
if (
!(
lang === "python" ||
VALIDATOR_SQL_CODE_LANGS.includes(lang as typeof config.code_lang)
)
) {
return [];
}
}
return [
{
name: item.name,
codeLang: item.code_lang?.trim() ?? "",
},
];
})
.filter((item) => item.name.trim())
.sort((a, b) => a.name.localeCompare(b.name)),
[configs],
);
const currentTarget = config.target_columns[0] ?? "";
return (
<div className="space-y-4">
<NameField
value={config.name}
onChange={(value) => onUpdate({ name: value })}
/>
<div className="grid gap-2">
<FieldLabel
label="Target code column"
htmlFor={targetColumnId}
hint="Must reference an LLM Code block."
/>
<Select
value={currentTarget || NONE_VALUE}
onValueChange={(value) => {
if (value === NONE_VALUE) {
onUpdate({
// biome-ignore lint/style/useNamingConvention: api schema
target_columns: [],
});
return;
}
const targetConfig = codeOptions.find((item) => item.name === value);
const nextCodeLang = targetConfig?.codeLang?.trim();
onUpdate({
// biome-ignore lint/style/useNamingConvention: api schema
target_columns: [value],
// biome-ignore lint/style/useNamingConvention: api schema
code_lang:
nextCodeLang && isValidatorCodeLang(nextCodeLang)
? nextCodeLang
: config.code_lang,
});
}}
>
<SelectTrigger className="nodrag w-full" id={targetColumnId}>
<SelectValue placeholder="Select code column" />
</SelectTrigger>
<SelectContent>
<SelectItem value={NONE_VALUE}>None</SelectItem>
{codeOptions.map((item) => (
<SelectItem key={item.name} value={item.name}>
{item.name}
</SelectItem>
))}
</SelectContent>
</Select>
{codeOptions.length === 0 && (
<p className="text-xs text-muted-foreground">
{config.validator_type === "oxc"
? "Add an LLM Code block with javascript/typescript first."
: "Add an LLM Code block first."}
</p>
)}
</div>
{config.validator_type === "oxc" && (
<div className="grid gap-3">
<div className="grid gap-2">
<FieldLabel
label="Validation mode"
htmlFor={oxcModeId}
hint="syntax: parser only. lint: oxlint only. syntax+lint: both."
/>
<div ref={oxcModeAnchorRef}>
<Combobox
items={OXC_VALIDATION_MODES}
filteredItems={OXC_VALIDATION_MODES}
filter={null}
value={selectedOxcMode}
onValueChange={(value) =>
onUpdate({
oxc_validation_mode: normalizeOxcValidationMode(value),
})
}
itemToStringValue={(value) => value}
autoHighlight={true}
>
<ComboboxInput
id={oxcModeId}
className="nodrag w-full"
placeholder="Select validation mode"
readOnly={true}
/>
<ComboboxContent anchor={oxcModeAnchorRef}>
<ComboboxEmpty>No modes available</ComboboxEmpty>
<ComboboxList>
{(mode: string) => (
<ComboboxItem key={mode} value={mode}>
{mode}
</ComboboxItem>
)}
</ComboboxList>
</ComboboxContent>
</Combobox>
</div>
</div>
<div className="grid gap-2">
<FieldLabel
label="Code shape"
htmlFor={oxcCodeShapeId}
hint="auto: detect module/snippet. module: strict file. snippet: wrapped fragment."
/>
<div ref={oxcCodeShapeAnchorRef}>
<Combobox
items={OXC_CODE_SHAPES}
filteredItems={OXC_CODE_SHAPES}
filter={null}
value={selectedOxcCodeShape}
onValueChange={(value) =>
onUpdate({
oxc_code_shape: normalizeOxcCodeShape(value),
})
}
itemToStringValue={(value) => value}
autoHighlight={true}
>
<ComboboxInput
id={oxcCodeShapeId}
className="nodrag w-full"
placeholder="Select code shape"
readOnly={true}
/>
<ComboboxContent anchor={oxcCodeShapeAnchorRef}>
<ComboboxEmpty>No code-shape options</ComboboxEmpty>
<ComboboxList>
{(shape: string) => (
<ComboboxItem key={shape} value={shape}>
{shape}
</ComboboxItem>
)}
</ComboboxList>
</ComboboxContent>
</Combobox>
</div>
</div>
</div>
)}
<Collapsible
open={advancedOpen}
onOpenChange={(open) => onUpdate({ advancedOpen: open })}
>
<CollapsibleTrigger asChild={true}>
<button
type="button"
className="flex w-full items-center justify-between text-left text-xs text-muted-foreground"
>
<span className="font-semibold uppercase">Advanced</span>
<span>{advancedOpen ? "Hide" : "Show"}</span>
</button>
</CollapsibleTrigger>
<CollapsibleContent className="mt-3">
<div className="grid gap-2">
<FieldLabel
label="Batch size"
htmlFor={batchSizeId}
hint="Records per validation batch."
/>
<Input
id={batchSizeId}
className="nodrag"
value={config.batch_size}
onChange={(event) => onUpdate({ batch_size: event.target.value })}
/>
</div>
</CollapsibleContent>
</Collapsible>
</div>
);
}

View file

@ -44,6 +44,9 @@ export type RecipeExecutionRecord = {
// biome-ignore lint/style/useNamingConvention: backend schema
jobId: string | null;
kind: RecipeExecutionKind;
// ui-only display label for full runs
// biome-ignore lint/style/useNamingConvention: ui schema
run_name: string | null;
status: RecipeExecutionStatus;
rows: number;
createdAt: number;
@ -52,6 +55,8 @@ export type RecipeExecutionRecord = {
stage: string | null;
// biome-ignore lint/style/useNamingConvention: backend schema
current_column: string | null;
// biome-ignore lint/style/useNamingConvention: backend schema
completed_columns: string[];
progress: RecipeExecutionProgress | null;
// biome-ignore lint/style/useNamingConvention: backend schema
column_progress: RecipeExecutionProgress | null;

View file

@ -89,6 +89,14 @@ export function executionLabel(kind: "preview" | "full"): string {
return kind === "preview" ? "Preview" : "Full run";
}
export function normalizeRunName(value: unknown): string | null {
if (typeof value !== "string") {
return null;
}
const trimmed = value.trim();
return trimmed.length > 0 ? trimmed : null;
}
function executionSortWeight(status: RecipeExecutionStatus): number {
if (isExecutionInProgress(status)) {
return 0;
@ -133,11 +141,17 @@ export function withExecutionDefaults(
return {
...record,
run_name: normalizeRunName(record.run_name),
dataset,
log_lines: logLines,
datasetTotal,
datasetPage,
datasetPageSize,
completed_columns: Array.isArray(record.completed_columns)
? record.completed_columns.filter(
(value): value is string => typeof value === "string" && value.trim().length > 0,
)
: [],
column_progress: record.column_progress ?? null,
batch: record.batch ?? null,
};

View file

@ -153,13 +153,13 @@ export function buildExecutionPayload(input: {
kind: RecipeExecutionKind;
rows: number;
settings: RecipeRunSettings;
runName?: string | null;
}): RecipePayload {
const normalizedSettings = normalizeRunSettings(input.settings);
const payloadWithParallelism = applyGlobalParallelismOverride(
input.payload,
normalizedSettings.llmParallelRequests,
);
return {
...payloadWithParallelism,
run: {
@ -174,6 +174,8 @@ export function buildExecutionPayload(input: {
input.kind === "full" &&
normalizedSettings.batchEnabled &&
normalizedSettings.mergeBatches,
// biome-ignore lint/style/useNamingConvention: backend schema
run_name: input.kind === "full" ? (input.runName ?? null) : null,
},
};
}

View file

@ -87,6 +87,11 @@ export function applyExecutionStatusSnapshot(
rows: status.rows ?? execution.rows,
stage: status.stage ?? execution.stage,
current_column: status.current_column ?? null,
completed_columns: Array.isArray(status.completed_columns)
? status.completed_columns.filter(
(value): value is string => typeof value === "string" && value.trim().length > 0,
)
: execution.completed_columns,
progress: (normalizeObject(status.progress) as RecipeExecutionRecord["progress"]) ?? null,
column_progress:
(normalizeObject(status.column_progress) as RecipeExecutionRecord["column_progress"]) ??
@ -109,6 +114,7 @@ export function createBaseExecutionRecord(input: {
kind: RecipeExecutionKind;
rows: number;
currentSignature: string;
runName?: string | null;
}): RecipeExecutionRecord {
const createdAt = Date.now();
return {
@ -116,6 +122,7 @@ export function createBaseExecutionRecord(input: {
recipeId: input.recipeId,
jobId: null,
kind: input.kind,
run_name: input.runName ?? null,
status: "pending",
rows: input.rows,
createdAt,
@ -123,6 +130,7 @@ export function createBaseExecutionRecord(input: {
recipeSignature: input.currentSignature,
stage: "pending",
current_column: null,
completed_columns: [],
progress: null,
column_progress: null,
batch: null,

View file

@ -232,6 +232,12 @@ export async function trackRecipeExecution({
const eventDataset = completedEventPayload
? completedEventPayload["dataset"]
: null;
const eventProcessorArtifacts =
completedEventPayload &&
typeof completedEventPayload["processor_artifacts"] === "object" &&
completedEventPayload["processor_artifacts"] !== null
? (completedEventPayload["processor_artifacts"] as Record<string, unknown>)
: null;
const shouldFetchPreviewDataset = kind === "preview" && !Array.isArray(eventDataset);
const shouldFetchAnalysis =
!completedEventPayload ||
@ -276,6 +282,7 @@ export async function trackRecipeExecution({
datasetPage: 1,
datasetPageSize: DATASET_PAGE_SIZE,
error: null,
processor_artifacts: eventProcessorArtifacts ?? latestExecution.processor_artifacts,
finishedAt: latestExecution.finishedAt ?? Date.now(),
};
onUpsert(latestExecution);

View file

@ -0,0 +1,320 @@
import type {
Edge,
EdgeChange,
Node,
NodeChange,
ReactFlowInstance,
XYPosition,
} from "@xyflow/react";
import {
type DragEvent as ReactDragEvent,
type RefObject,
useCallback,
useMemo,
} from "react";
import { RECIPE_BLOCK_DND_MIME, type RecipeBlockDragPayload } from "../components/block-sheet";
import type { SeedBlockType } from "../blocks/registry";
import type {
LlmType,
NodeConfig,
RecipeNode as RecipeBuilderNode,
RecipeNodeData,
SamplerType,
} from "../types";
import { applyAuxNodeChanges, filterEdgeChangesByIds, filterNodeChangesByIds } from "../utils/reactflow-changes";
import type { RecipeGraphAuxNodeData } from "../components/recipe-graph-aux-node";
const SUPPORTED_DRAG_KINDS: RecipeBlockDragPayload["kind"][] = [
"sampler",
"seed",
"llm",
"validator",
"expression",
"note",
];
function parseRecipeBlockDragPayload(raw: string): RecipeBlockDragPayload | null {
try {
const parsed = JSON.parse(raw) as {
kind?: RecipeBlockDragPayload["kind"];
type?: RecipeBlockDragPayload["type"];
};
if (!parsed.kind || !parsed.type || !SUPPORTED_DRAG_KINDS.includes(parsed.kind)) {
return null;
}
return {
kind: parsed.kind,
type: parsed.type,
};
} catch {
return null;
}
}
type UseRecipeEditorGraphArgs = {
nodes: RecipeBuilderNode[];
edges: Edge[];
configs: Record<string, NodeConfig>;
reactFlowInstance: ReactFlowInstance<Node<RecipeNodeData | RecipeGraphAuxNodeData>, Edge> | null;
flowContainerRef: RefObject<HTMLDivElement | null>;
selectConfig: (id: string) => void;
openConfig: (id: string) => void;
onNodesChange: (changes: NodeChange<RecipeBuilderNode>[]) => void;
onEdgesChange: (changes: EdgeChange<Edge>[]) => void;
setAuxNodePosition: (id: string, position: XYPosition) => void;
addSamplerNode: (type: SamplerType, position?: XYPosition, openDialog?: boolean) => void;
addSeedNode: (type: SeedBlockType, position?: XYPosition, openDialog?: boolean) => void;
addLlmNode: (type: LlmType, position?: XYPosition, openDialog?: boolean) => void;
addModelProviderNode: (position?: XYPosition, openDialog?: boolean) => void;
addModelConfigNode: (position?: XYPosition, openDialog?: boolean) => void;
addExpressionNode: (position?: XYPosition, openDialog?: boolean) => void;
addValidatorNode: (
type: "validator_python" | "validator_sql" | "validator_oxc",
position?: XYPosition,
openDialog?: boolean,
) => void;
addMarkdownNoteNode: (position?: XYPosition, openDialog?: boolean) => void;
};
type UseRecipeEditorGraphResult = {
handleNodeClick: (_: unknown, node: Node<RecipeNodeData | RecipeGraphAuxNodeData>) => void;
handleNodeDoubleClick: (_: unknown, node: Node<RecipeNodeData | RecipeGraphAuxNodeData>) => void;
handleNodesChange: (
changes: NodeChange<Node<RecipeNodeData | RecipeGraphAuxNodeData>>[],
) => void;
handleEdgesChange: (changes: EdgeChange<Edge>[]) => void;
handleDragOver: (event: ReactDragEvent<HTMLDivElement>) => void;
handleDrop: (event: ReactDragEvent<HTMLDivElement>) => void;
handleAddSamplerFromSheet: (type: SamplerType) => void;
handleAddSeedFromSheet: (type: SeedBlockType) => void;
handleAddLlmFromSheet: (type: LlmType) => void;
handleAddModelProviderFromSheet: () => void;
handleAddModelConfigFromSheet: () => void;
handleAddExpressionFromSheet: () => void;
handleAddValidatorFromSheet: (
type: "validator_python" | "validator_sql" | "validator_oxc",
) => void;
handleAddMarkdownNoteFromSheet: () => void;
};
export function useRecipeEditorGraph({
nodes,
edges,
configs,
reactFlowInstance,
flowContainerRef,
selectConfig,
openConfig,
onNodesChange,
onEdgesChange,
setAuxNodePosition,
addSamplerNode,
addSeedNode,
addLlmNode,
addModelProviderNode,
addModelConfigNode,
addExpressionNode,
addValidatorNode,
addMarkdownNoteNode,
}: UseRecipeEditorGraphArgs): UseRecipeEditorGraphResult {
const baseNodeIds = useMemo(() => new Set(nodes.map((node) => node.id)), [nodes]);
const baseEdgeIds = useMemo(() => new Set(edges.map((edge) => edge.id)), [edges]);
const handleNodeClick = useCallback(
(_: unknown, node: Node<RecipeNodeData | RecipeGraphAuxNodeData>) => {
if (node.type !== "builder") {
return;
}
selectConfig(node.id);
},
[selectConfig],
);
const handleNodeDoubleClick = useCallback(
(_: unknown, node: Node<RecipeNodeData | RecipeGraphAuxNodeData>) => {
if (node.type !== "builder") {
return;
}
const nodeConfig = configs[node.id];
if (nodeConfig?.kind === "markdown_note") {
openConfig(node.id);
}
},
[configs, openConfig],
);
const handleNodesChange = useCallback(
(changes: NodeChange<Node<RecipeNodeData | RecipeGraphAuxNodeData>>[]) => {
applyAuxNodeChanges(changes, { setAuxNodePosition });
const next = filterNodeChangesByIds(
changes as NodeChange<RecipeBuilderNode>[],
baseNodeIds,
);
if (next.length) {
onNodesChange(next);
}
},
[baseNodeIds, onNodesChange, setAuxNodePosition],
);
const handleEdgesChange = useCallback(
(changes: EdgeChange<Edge>[]) => {
const next = filterEdgeChangesByIds(changes, baseEdgeIds);
if (next.length) {
onEdgesChange(next);
}
},
[baseEdgeIds, onEdgesChange],
);
const handleDragOver = useCallback((event: ReactDragEvent<HTMLDivElement>) => {
if (
!event.dataTransfer.types.includes(RECIPE_BLOCK_DND_MIME) &&
!event.dataTransfer.types.includes("text/plain")
) {
return;
}
event.preventDefault();
event.dataTransfer.dropEffect = "copy";
}, []);
const handleDrop = useCallback(
(event: ReactDragEvent<HTMLDivElement>) => {
if (!reactFlowInstance) {
return;
}
const raw =
event.dataTransfer.getData(RECIPE_BLOCK_DND_MIME) ||
event.dataTransfer.getData("text/plain");
if (!raw) {
return;
}
const payload = parseRecipeBlockDragPayload(raw);
if (!payload) {
return;
}
event.preventDefault();
const position = reactFlowInstance.screenToFlowPosition({
x: event.clientX,
y: event.clientY,
});
if (payload.kind === "sampler") {
addSamplerNode(payload.type as SamplerType, position, false);
return;
}
if (payload.kind === "seed") {
addSeedNode(payload.type as SeedBlockType, position, false);
return;
}
if (payload.kind === "expression") {
addExpressionNode(position, false);
return;
}
if (payload.kind === "validator") {
addValidatorNode(
payload.type as "validator_python" | "validator_sql" | "validator_oxc",
position,
false,
);
return;
}
if (payload.kind === "note") {
addMarkdownNoteNode(position, false);
return;
}
if (payload.type === "model_provider") {
addModelProviderNode(position, false);
return;
}
if (payload.type === "model_config") {
addModelConfigNode(position, false);
return;
}
addLlmNode(payload.type as LlmType, position, false);
},
[
addExpressionNode,
addLlmNode,
addMarkdownNoteNode,
addModelConfigNode,
addModelProviderNode,
addSamplerNode,
addSeedNode,
addValidatorNode,
reactFlowInstance,
],
);
const getViewportCenterPosition = useCallback(() => {
if (!reactFlowInstance || !flowContainerRef.current) {
return undefined;
}
const rect = flowContainerRef.current.getBoundingClientRect();
return reactFlowInstance.screenToFlowPosition({
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2,
});
}, [flowContainerRef, reactFlowInstance]);
const handleAddSamplerFromSheet = useCallback(
(type: SamplerType) => {
addSamplerNode(type, getViewportCenterPosition());
},
[addSamplerNode, getViewportCenterPosition],
);
const handleAddSeedFromSheet = useCallback(
(type: SeedBlockType) => {
addSeedNode(type, getViewportCenterPosition());
},
[addSeedNode, getViewportCenterPosition],
);
const handleAddLlmFromSheet = useCallback(
(type: LlmType) => {
addLlmNode(type, getViewportCenterPosition());
},
[addLlmNode, getViewportCenterPosition],
);
const handleAddModelProviderFromSheet = useCallback(() => {
addModelProviderNode(getViewportCenterPosition());
}, [addModelProviderNode, getViewportCenterPosition]);
const handleAddModelConfigFromSheet = useCallback(() => {
addModelConfigNode(getViewportCenterPosition());
}, [addModelConfigNode, getViewportCenterPosition]);
const handleAddExpressionFromSheet = useCallback(() => {
addExpressionNode(getViewportCenterPosition());
}, [addExpressionNode, getViewportCenterPosition]);
const handleAddValidatorFromSheet = useCallback(
(type: "validator_python" | "validator_sql" | "validator_oxc") => {
addValidatorNode(type, getViewportCenterPosition());
},
[addValidatorNode, getViewportCenterPosition],
);
const handleAddMarkdownNoteFromSheet = useCallback(() => {
addMarkdownNoteNode(getViewportCenterPosition());
}, [addMarkdownNoteNode, getViewportCenterPosition]);
return {
handleNodeClick,
handleNodeDoubleClick,
handleNodesChange,
handleEdgesChange,
handleDragOver,
handleDrop,
handleAddSamplerFromSheet,
handleAddSeedFromSheet,
handleAddLlmFromSheet,
handleAddModelProviderFromSheet,
handleAddModelConfigFromSheet,
handleAddExpressionFromSheet,
handleAddValidatorFromSheet,
handleAddMarkdownNoteFromSheet,
};
}

View file

@ -15,6 +15,7 @@ import type {
import {
DATASET_PAGE_SIZE,
executionLabel,
normalizeRunName,
normalizeDatasetRows,
toErrorMessage,
withExecutionDefaults,
@ -50,8 +51,10 @@ type UseRecipeExecutionsResult = {
setRunDialogOpen: (open: boolean) => void;
previewRows: number;
fullRows: number;
fullRunName: string;
setPreviewRows: (rows: number) => void;
setFullRows: (rows: number) => void;
setFullRunName: (name: string) => void;
runErrors: string[];
runSettings: RecipeRunSettings;
setRunSettings: (patch: Partial<RecipeRunSettings>) => void;
@ -109,6 +112,7 @@ export function useRecipeExecutions({
runDialogKind,
previewRows,
fullRows,
fullRunName,
runErrors,
runSettings,
previewLoading,
@ -119,6 +123,7 @@ export function useRecipeExecutions({
setRunDialogKind,
setPreviewRows,
setFullRows,
setFullRunName,
setRunErrors,
setRunSettings,
setPreviewLoading,
@ -133,6 +138,7 @@ export function useRecipeExecutions({
runDialogKind: state.runDialogKind,
previewRows: state.previewRows,
fullRows: state.fullRows,
fullRunName: state.fullRunName,
runErrors: state.runErrors,
runSettings: state.runSettings,
previewLoading: state.previewLoading,
@ -143,6 +149,7 @@ export function useRecipeExecutions({
setRunDialogKind: state.setRunDialogKind,
setPreviewRows: state.setPreviewRows,
setFullRows: state.setFullRows,
setFullRunName: state.setFullRunName,
setRunErrors: state.setRunErrors,
setRunSettings: state.setRunSettings,
setPreviewLoading: state.setPreviewLoading,
@ -238,8 +245,9 @@ export function useRecipeExecutions({
payload: RecipePayload;
rows: number;
settings: RecipeRunSettings;
runName: string | null;
}): Promise<boolean> => {
const { kind, payload, rows, settings } = input;
const { kind, payload, rows, settings, runName } = input;
const setLoading = kind === "preview" ? setPreviewLoading : setFullLoading;
const label = executionLabel(kind);
@ -249,6 +257,7 @@ export function useRecipeExecutions({
kind,
rows,
currentSignature,
runName,
});
upsertAndPersist(baseExecution);
@ -261,6 +270,7 @@ export function useRecipeExecutions({
kind,
rows,
settings,
runName,
});
const createdJob = await createRecipeJob(jobPayload);
const executionWithJob = {
@ -309,7 +319,19 @@ export function useRecipeExecutions({
);
const runWithValidation = useCallback(
async (kind: RecipeExecutionKind, rows: number): Promise<boolean> => {
async (
kind: RecipeExecutionKind,
rows: number,
runName: string | null,
): Promise<boolean> => {
const trimmedRunName = typeof runName === "string" ? runName.trim() : "";
if (kind === "full" && !trimmedRunName) {
const message = "Run name required for full runs.";
setRunErrors([message]);
toastError("Run name required", message);
return false;
}
const payload = readExecutablePayload();
if (!payload) {
return false;
@ -321,6 +343,7 @@ export function useRecipeExecutions({
kind,
rows: normalizedRows,
settings: runSettings,
runName,
});
try {
@ -345,18 +368,19 @@ export function useRecipeExecutions({
payload,
rows: normalizedRows,
settings: runSettings,
runName,
});
},
[readExecutablePayload, runExecution, runSettings, setRunErrors],
);
const runPreview = useCallback(async (): Promise<boolean> => {
return runWithValidation("preview", previewRows);
return runWithValidation("preview", previewRows, null);
}, [previewRows, runWithValidation]);
const runFull = useCallback(async (): Promise<boolean> => {
return runWithValidation("full", fullRows);
}, [fullRows, runWithValidation]);
return runWithValidation("full", fullRows, fullRunName);
}, [fullRows, fullRunName, runWithValidation]);
const runFromDialog = useCallback(async (): Promise<boolean> => {
setValidateResult(null);
@ -388,6 +412,7 @@ export function useRecipeExecutions({
kind: runDialogKind,
rows: normalizedRows,
settings: runSettings,
runName: runDialogKind === "full" ? normalizeRunName(fullRunName) : null,
});
setValidateLoading(true);
@ -412,6 +437,7 @@ export function useRecipeExecutions({
setValidateLoading(false);
}
}, [
fullRunName,
fullRows,
payloadErrorMessage,
payloadResult.errors,
@ -419,6 +445,7 @@ export function useRecipeExecutions({
readPayload,
runDialogKind,
runSettings,
setRunErrors,
]);
const openRunDialog = useCallback(
@ -509,8 +536,10 @@ export function useRecipeExecutions({
setRunDialogOpen,
previewRows,
fullRows,
fullRunName,
setPreviewRows,
setFullRows,
setFullRunName,
runErrors,
runSettings,
setRunSettings,

View file

@ -64,6 +64,23 @@ function stripApiKeys(value: unknown): unknown {
return output;
}
function inferHfRepoIdFromPath(pathValue: unknown): string {
if (typeof pathValue !== "string") {
return "";
}
const parts = pathValue
.trim()
.split("/")
.filter(Boolean);
if (parts.length >= 3 && parts[0] === "datasets") {
return `${parts[1]}/${parts[2]}`;
}
if (parts.length >= 2) {
return `${parts[0]}/${parts[1]}`;
}
return "";
}
function sanitizeSeedForShare(payload: unknown): unknown {
if (!payload || typeof payload !== "object") {
return payload;
@ -95,11 +112,28 @@ function sanitizeSeedForShare(payload: unknown): unknown {
typeof ui?.seed_source_type === "string" ? ui.seed_source_type : null;
const sourceType =
typeof source?.seed_type === "string" ? source.seed_type : null;
const shouldResetHfState =
sourceType === "hf" || uiSourceType === "hf";
const shouldResetLocalState =
sourceType === "local" ||
sourceType === "unstructured" ||
uiSourceType === "local" ||
uiSourceType === "unstructured";
if (shouldResetHfState) {
const repoId = inferHfRepoIdFromPath(source?.path);
if (source && "path" in source) {
source.path = repoId;
}
if (ui) {
ui.seed_columns = [];
ui.seed_drop_columns = [];
ui.seed_preview_rows = [];
ui.local_file_name = "";
ui.unstructured_file_name = "";
}
}
if (shouldResetLocalState) {
if (source && "path" in source) {
source.path = "";

View file

@ -0,0 +1,178 @@
import {
BalanceScaleIcon,
Clock01Icon,
CodeIcon,
CodeSimpleIcon,
DiceFaces03Icon,
EqualSignIcon,
FingerPrintIcon,
FunctionIcon,
Parabola02Icon,
PencilEdit02Icon,
Plant01Icon,
Shield02Icon,
Tag01Icon,
TagsIcon,
UserAccountIcon,
} from "@hugeicons/core-free-icons";
import { useMemo } from "react";
import type { Edge } from "@xyflow/react";
import { deriveDisplayGraph } from "../utils/graph/derive-display-graph";
import {
deriveGraphRuntimeVisualState,
pickLatestActiveExecution,
} from "../utils/graph/runtime-visual-state";
import type {
LayoutDirection,
LlmType,
NodeConfig,
RecipeNode as RecipeBuilderNode,
SamplerType,
} from "../types";
import type { RecipeExecutionRecord } from "../execution-types";
type IconType = typeof CodeIcon;
const SAMPLER_ICONS: Record<SamplerType, IconType> = {
category: Tag01Icon,
subcategory: TagsIcon,
uniform: EqualSignIcon,
gaussian: Parabola02Icon,
bernoulli: EqualSignIcon,
datetime: Clock01Icon,
timedelta: Clock01Icon,
uuid: FingerPrintIcon,
person: UserAccountIcon,
person_from_faker: UserAccountIcon,
};
const LLM_ICONS: Record<LlmType, IconType> = {
text: PencilEdit02Icon,
structured: CodeIcon,
code: CodeSimpleIcon,
judge: BalanceScaleIcon,
};
function resolveExecutionColumnIcon(config: NodeConfig | null): IconType {
if (!config) {
return DiceFaces03Icon;
}
if (config.kind === "sampler") {
return SAMPLER_ICONS[config.sampler_type];
}
if (config.kind === "llm") {
return LLM_ICONS[config.llm_type];
}
if (config.kind === "expression") {
return FunctionIcon;
}
if (config.kind === "validator") {
return Shield02Icon;
}
if (config.kind === "seed") {
return Plant01Icon;
}
if (config.kind === "model_provider") {
return Shield02Icon;
}
if (config.kind === "model_config") {
return Plant01Icon;
}
return PencilEdit02Icon;
}
type UseRecipeRuntimeVisualsArgs = {
executions: RecipeExecutionRecord[];
configs: Record<string, NodeConfig>;
nodes: RecipeBuilderNode[];
edges: Edge[];
layoutDirection: LayoutDirection;
auxNodePositions: Record<string, { x: number; y: number }>;
llmAuxVisibility: Record<string, boolean>;
};
type UseRecipeRuntimeVisualsResult = {
activeExecution: RecipeExecutionRecord | null;
runtimeVisualState: ReturnType<typeof deriveGraphRuntimeVisualState>;
displayGraph: ReturnType<typeof deriveDisplayGraph>;
displayNodeIds: string[];
currentColumnIcon: IconType;
};
export function useRecipeRuntimeVisuals({
executions,
configs,
nodes,
edges,
layoutDirection,
auxNodePositions,
llmAuxVisibility,
}: UseRecipeRuntimeVisualsArgs): UseRecipeRuntimeVisualsResult {
const activeExecution = useMemo(
() => pickLatestActiveExecution(executions),
[executions],
);
const runtimeVisualState = useMemo(
() =>
deriveGraphRuntimeVisualState({
activeExecution,
configs,
edges,
}),
[activeExecution, configs, edges],
);
const displayGraph = useMemo(
() =>
deriveDisplayGraph({
nodes,
edges,
configs,
layoutDirection,
auxNodePositions,
llmAuxVisibility,
runtime: runtimeVisualState,
}),
[
auxNodePositions,
configs,
edges,
layoutDirection,
llmAuxVisibility,
nodes,
runtimeVisualState,
],
);
const currentColumnConfig = useMemo(() => {
const columnName = activeExecution?.current_column?.trim();
if (!columnName) {
return null;
}
for (const config of Object.values(configs)) {
if (config.name.trim() === columnName) {
return config;
}
}
return null;
}, [activeExecution?.current_column, configs]);
const currentColumnIcon = useMemo(
() => resolveExecutionColumnIcon(currentColumnConfig),
[currentColumnConfig],
);
const displayNodeIds = useMemo(
() => displayGraph.nodes.map((node) => node.id),
[displayGraph.nodes],
);
return {
activeExecution,
runtimeVisualState,
displayGraph,
displayNodeIds,
currentColumnIcon,
};
}

View file

@ -48,8 +48,10 @@ type UseRecipeStudioActionsResult = {
setRunDialogOpen: (open: boolean) => void;
previewRows: number;
fullRows: number;
fullRunName: string;
setPreviewRows: (rows: number) => void;
setFullRows: (rows: number) => void;
setFullRunName: (name: string) => void;
runErrors: string[];
runSettings: RecipeRunSettings;
setRunSettings: (patch: Partial<RecipeRunSettings>) => void;
@ -125,8 +127,10 @@ export function useRecipeStudioActions({
setRunDialogOpen: executions.setRunDialogOpen,
previewRows: executions.previewRows,
fullRows: executions.fullRows,
fullRunName: executions.fullRunName,
setPreviewRows: executions.setPreviewRows,
setFullRows: executions.setFullRows,
setFullRunName: executions.setFullRunName,
runErrors: executions.runErrors,
runSettings: executions.runSettings,
setRunSettings: executions.setRunSettings,

View file

@ -2,23 +2,16 @@ import {
Background,
BackgroundVariant,
type Edge,
type EdgeChange,
type EdgeTypes,
type Node,
type NodeChange,
type NodeTypes,
Panel,
ReactFlow,
type ReactFlowInstance,
} from "@xyflow/react";
import {
CookBookIcon,
PlusSignIcon,
TestTube01Icon,
} from "@hugeicons/core-free-icons";
import { PlusSignIcon } from "@hugeicons/core-free-icons";
import { HugeiconsIcon } from "@hugeicons/react";
import {
type DragEvent as ReactDragEvent,
type ReactElement,
useCallback,
useEffect,
@ -31,77 +24,39 @@ import "@xyflow/react/dist/style.css";
import { RecipeGraphAuxNode, type RecipeGraphAuxNodeData } from "./components/recipe-graph-aux-node";
import {
BlockSheet,
RECIPE_BLOCK_DND_MIME,
type RecipeBlockDragPayload,
} from "./components/block-sheet";
import { LayoutControls } from "./components/controls/layout-controls";
import { RunValidateFloatingControls } from "./components/controls/run-validate-floating-controls";
import { ViewportControls } from "./components/controls/viewport-controls";
import { ExecutionsView } from "./components/executions/executions-view";
import { InternalsSync } from "./components/graph/internals-sync";
import { ExecutionProgressIsland } from "./components/runtime/execution-progress-island";
import { RecipeStudioHeader } from "./components/recipe-studio-header";
import { RecipeNode } from "./components/recipe-graph-node";
import { RecipeGraphSemanticEdge } from "./components/recipe-graph-semantic-edge";
import { DataEdge } from "./components/rf-ui/data-edge";
import { Button } from "@/components/ui/button";
import { ConfigDialog } from "./dialogs/config-dialog";
import { ImportDialog } from "./dialogs/import-dialog";
import { RunDialog } from "./dialogs/preview-dialog";
import { ProcessorsDialog } from "./dialogs/processors-dialog";
import { useRecipeEditorGraph } from "./hooks/use-recipe-editor-graph";
import { useRecipeRuntimeVisuals } from "./hooks/use-recipe-runtime-visuals";
import { useRecipeStudioActions } from "./hooks/use-recipe-studio-actions";
import { useRecipeStudioStore } from "./stores/recipe-studio";
import type {
LlmType,
RecipeNode as RecipeBuilderNode,
RecipeNodeData,
SamplerType,
} from "./types";
import type { SeedBlockType } from "./blocks/registry";
import { deriveDisplayGraph } from "./utils/graph/derive-display-graph";
import { isExecutionInProgress } from "./executions/execution-helpers";
import type { RecipeNodeData } from "./types";
import { getFitNodeIdsIgnoringNotes } from "./utils/graph/fit-view";
import { buildRecipePayload } from "./utils/payload";
import type { RecipePayload } from "./utils/payload/types";
import { buildDefaultSchemaTransform } from "./utils/processors";
import {
applyAuxNodeChanges,
filterEdgeChangesByIds,
filterNodeChangesByIds,
} from "./utils/reactflow-changes";
import {
buildDialogOptions,
} from "./utils/recipe-studio-view";
import type { RecipeStudioView } from "./execution-types";
import { buildDialogOptions } from "./utils/recipe-studio-view";
import type { RecipeExecutionRecord, RecipeStudioView } from "./execution-types";
const NODE_TYPES: NodeTypes = { builder: RecipeNode, aux: RecipeGraphAuxNode };
const EDGE_TYPES: EdgeTypes = { canvas: DataEdge, semantic: RecipeGraphSemanticEdge };
const SUPPORTED_DRAG_KINDS: RecipeBlockDragPayload["kind"][] = [
"sampler",
"seed",
"llm",
"expression",
"note",
];
function parseRecipeBlockDragPayload(raw: string): RecipeBlockDragPayload | null {
try {
const parsed = JSON.parse(raw) as {
kind?: RecipeBlockDragPayload["kind"];
type?: RecipeBlockDragPayload["type"];
};
if (
!parsed.kind ||
!parsed.type ||
!SUPPORTED_DRAG_KINDS.includes(parsed.kind)
) {
return null;
}
return {
kind: parsed.kind,
type: parsed.type,
};
} catch {
return null;
}
}
const COMPLETE_ISLAND_VISIBLE_MS = 7_000;
const TAB_SWITCH_FIT_DELAY_MS = 110;
const FIT_ANIMATION_MS = 340;
export type PersistRecipeInput = {
id: string | null;
@ -150,6 +105,7 @@ export function RecipeStudioPage({
addModelProviderNode,
addModelConfigNode,
addExpressionNode,
addValidatorNode,
addMarkdownNoteNode,
selectConfig,
openConfig,
@ -163,6 +119,7 @@ export function RecipeStudioPage({
setLayoutDirection,
applyLayout,
setAuxNodePosition,
setExecutionLocked,
} = useRecipeStudioStore(
useShallow((state) => ({
nodes: state.nodes,
@ -185,6 +142,7 @@ export function RecipeStudioPage({
addModelProviderNode: state.addModelProviderNode,
addModelConfigNode: state.addModelConfigNode,
addExpressionNode: state.addExpressionNode,
addValidatorNode: state.addValidatorNode,
addMarkdownNoteNode: state.addMarkdownNoteNode,
selectConfig: state.selectConfig,
openConfig: state.openConfig,
@ -198,6 +156,7 @@ export function RecipeStudioPage({
setLayoutDirection: state.setLayoutDirection,
applyLayout: state.applyLayout,
setAuxNodePosition: state.setAuxNodePosition,
setExecutionLocked: state.setExecutionLocked,
})),
);
const [sheetContainer, setSheetContainer] = useState<HTMLDivElement | null>(
@ -208,204 +167,53 @@ export function RecipeStudioPage({
const [activeView, setActiveView] = useState<RecipeStudioView>("editor");
const [processorsOpen, setProcessorsOpen] = useState(false);
const [interactive, setInteractive] = useState(true);
const [runtimeIslandMinimized, setRuntimeIslandMinimized] = useState(false);
const [recentCompletedExecution, setRecentCompletedExecution] =
useState<RecipeExecutionRecord | null>(null);
const [reactFlowInstance, setReactFlowInstance] = useState<
ReactFlowInstance<Node<RecipeNodeData | RecipeGraphAuxNodeData>, Edge> | null
>(null);
const lastProcessedFitTickRef = useRef(0);
const handleExecutionStart = useCallback(() => {
setActiveView("executions");
}, []);
const handlePreviewSuccess = useCallback(() => {
setActiveView("executions");
}, []);
const baseNodeIds = useMemo(
() => new Set(nodes.map((node) => node.id)),
[nodes],
);
const baseEdgeIds = useMemo(
() => new Set(edges.map((edge) => edge.id)),
[edges],
);
const displayGraph = useMemo(() => {
return deriveDisplayGraph({
nodes,
edges,
configs,
layoutDirection,
auxNodePositions,
llmAuxVisibility,
});
}, [
auxNodePositions,
configs,
edges,
layoutDirection,
llmAuxVisibility,
const previousActiveViewRef = useRef<RecipeStudioView>("editor");
const previousActiveExecutionIdRef = useRef<string | null>(null);
const pendingEditorTabFitRef = useRef(false);
const forceEditorTabFitRef = useRef(false);
const viewportMovedSinceAutoFitRef = useRef(true);
const {
handleNodeClick,
handleNodeDoubleClick,
handleNodesChange,
handleEdgesChange,
handleDragOver,
handleDrop,
handleAddSamplerFromSheet,
handleAddSeedFromSheet,
handleAddLlmFromSheet,
handleAddModelProviderFromSheet,
handleAddModelConfigFromSheet,
handleAddExpressionFromSheet,
handleAddValidatorFromSheet,
handleAddMarkdownNoteFromSheet,
} = useRecipeEditorGraph({
nodes,
]);
const displayNodeIds = useMemo(
() => displayGraph.nodes.map((node) => node.id),
[displayGraph.nodes],
);
const handleNodeClick = useCallback(
(_: unknown, node: Node<RecipeNodeData | RecipeGraphAuxNodeData>) => {
if (node.type !== "builder") {
return;
}
selectConfig(node.id);
},
[selectConfig],
);
const handleNodeDoubleClick = useCallback(
(_: unknown, node: Node<RecipeNodeData | RecipeGraphAuxNodeData>) => {
if (node.type !== "builder") {
return;
}
const nodeConfig = configs[node.id];
if (nodeConfig?.kind === "markdown_note") {
openConfig(node.id);
}
},
[configs, openConfig],
);
const handleNodesChange = useCallback(
(changes: NodeChange<Node<RecipeNodeData | RecipeGraphAuxNodeData>>[]) => {
applyAuxNodeChanges(changes, { setAuxNodePosition });
const next = filterNodeChangesByIds(
changes as NodeChange<RecipeBuilderNode>[],
baseNodeIds,
);
if (next.length) {
onNodesChange(next);
}
},
[baseNodeIds, onNodesChange, setAuxNodePosition],
);
const handleEdgesChange = useCallback(
(changes: EdgeChange<Edge>[]) => {
const next = filterEdgeChangesByIds(changes, baseEdgeIds);
if (next.length) {
onEdgesChange(next);
}
},
[baseEdgeIds, onEdgesChange],
);
const handleDragOver = useCallback((event: ReactDragEvent<HTMLDivElement>) => {
if (
!event.dataTransfer.types.includes(RECIPE_BLOCK_DND_MIME) &&
!event.dataTransfer.types.includes("text/plain")
) {
return;
}
event.preventDefault();
event.dataTransfer.dropEffect = "copy";
}, []);
const handleDrop = useCallback(
(event: ReactDragEvent<HTMLDivElement>) => {
if (!reactFlowInstance) {
return;
}
const raw =
event.dataTransfer.getData(RECIPE_BLOCK_DND_MIME) ||
event.dataTransfer.getData("text/plain");
if (!raw) {
return;
}
const payload = parseRecipeBlockDragPayload(raw);
if (!payload) {
return;
}
event.preventDefault();
const position = reactFlowInstance.screenToFlowPosition({
x: event.clientX,
y: event.clientY,
});
if (payload.kind === "sampler") {
addSamplerNode(payload.type as SamplerType, position, false);
return;
}
if (payload.kind === "seed") {
addSeedNode(payload.type as SeedBlockType, position, false);
return;
}
if (payload.kind === "expression") {
addExpressionNode(position, false);
return;
}
if (payload.kind === "note") {
addMarkdownNoteNode(position, false);
return;
}
if (payload.type === "model_provider") {
addModelProviderNode(position, false);
return;
}
if (payload.type === "model_config") {
addModelConfigNode(position, false);
return;
}
addLlmNode(payload.type as LlmType, position, false);
},
[
addExpressionNode,
addLlmNode,
addMarkdownNoteNode,
addModelConfigNode,
addModelProviderNode,
addSamplerNode,
addSeedNode,
reactFlowInstance,
],
);
const getViewportCenterPosition = useCallback(() => {
if (!reactFlowInstance || !flowContainerRef.current) {
return undefined;
}
const rect = flowContainerRef.current.getBoundingClientRect();
return reactFlowInstance.screenToFlowPosition({
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2,
});
}, [reactFlowInstance]);
const handleAddSamplerFromSheet = useCallback(
(type: SamplerType) => {
addSamplerNode(type, getViewportCenterPosition());
},
[addSamplerNode, getViewportCenterPosition],
);
const handleAddSeedFromSheet = useCallback(
(type: SeedBlockType) => {
addSeedNode(type, getViewportCenterPosition());
},
[addSeedNode, getViewportCenterPosition],
);
const handleAddLlmFromSheet = useCallback(
(type: LlmType) => {
addLlmNode(type, getViewportCenterPosition());
},
[addLlmNode, getViewportCenterPosition],
);
const handleAddModelProviderFromSheet = useCallback(() => {
addModelProviderNode(getViewportCenterPosition());
}, [addModelProviderNode, getViewportCenterPosition]);
const handleAddModelConfigFromSheet = useCallback(() => {
addModelConfigNode(getViewportCenterPosition());
}, [addModelConfigNode, getViewportCenterPosition]);
const handleAddExpressionFromSheet = useCallback(() => {
addExpressionNode(getViewportCenterPosition());
}, [addExpressionNode, getViewportCenterPosition]);
const handleAddMarkdownNoteFromSheet = useCallback(() => {
addMarkdownNoteNode(getViewportCenterPosition());
}, [addMarkdownNoteNode, getViewportCenterPosition]);
edges,
configs,
reactFlowInstance,
flowContainerRef,
selectConfig,
openConfig,
onNodesChange,
onEdgesChange,
setAuxNodePosition,
addSamplerNode,
addSeedNode,
addLlmNode,
addModelProviderNode,
addModelConfigNode,
addExpressionNode,
addValidatorNode,
addMarkdownNoteNode,
});
const configList = useMemo(() => Object.values(configs), [configs]);
const config = activeConfigId ? configs[activeConfigId] : null;
@ -418,10 +226,6 @@ export function RecipeStudioPage({
setLayoutDirection(layoutDirection === "LR" ? "TB" : "LR");
}, [layoutDirection, setLayoutDirection]);
const toggleInteractive = useCallback(() => {
setInteractive((value) => !value);
}, []);
const payloadResult = useMemo(
() =>
buildRecipePayload(
@ -460,8 +264,10 @@ export function RecipeStudioPage({
setRunDialogOpen,
previewRows,
fullRows,
fullRunName,
setPreviewRows,
setFullRows,
setFullRunName,
runErrors,
runSettings,
setRunSettings,
@ -491,9 +297,78 @@ export function RecipeStudioPage({
resetRecipe,
loadRecipe,
getCurrentPayloadFromStore,
onExecutionStart: handleExecutionStart,
onPreviewSuccess: handlePreviewSuccess,
});
const {
activeExecution,
runtimeVisualState,
displayGraph,
displayNodeIds,
currentColumnIcon,
} = useRecipeRuntimeVisuals({
executions,
configs,
nodes,
edges,
layoutDirection,
auxNodePositions,
llmAuxVisibility,
});
const executionLocked = runtimeVisualState.executionLocked;
const canvasInteractive = interactive && !executionLocked;
const runBusy = previewLoading || fullLoading || executionLocked;
const islandExecution = activeExecution ?? recentCompletedExecution;
const toggleInteractive = useCallback(() => {
if (executionLocked) {
return;
}
setInteractive((value) => !value);
}, [executionLocked]);
useEffect(() => {
setExecutionLocked(executionLocked);
}, [executionLocked, setExecutionLocked]);
useEffect(() => {
const activeExecutionId = activeExecution?.id ?? null;
if (
activeExecutionId &&
activeExecutionId !== previousActiveExecutionIdRef.current
) {
setRuntimeIslandMinimized(false);
}
previousActiveExecutionIdRef.current = activeExecutionId;
}, [activeExecution?.id]);
useEffect(() => {
if (activeExecution) {
setRecentCompletedExecution(null);
return;
}
const latestCompleted = executions.find(
(execution) =>
execution.status === "completed" && typeof execution.finishedAt === "number",
);
if (!latestCompleted || typeof latestCompleted.finishedAt !== "number") {
setRecentCompletedExecution(null);
return;
}
const elapsedMs = Date.now() - latestCompleted.finishedAt;
if (elapsedMs >= COMPLETE_ISLAND_VISIBLE_MS) {
setRecentCompletedExecution(null);
return;
}
setRecentCompletedExecution(latestCompleted);
const hideTimer = window.setTimeout(() => {
setRecentCompletedExecution(null);
setActiveView((currentView) =>
currentView === "editor" ? "executions" : currentView,
);
}, COMPLETE_ISLAND_VISIBLE_MS - elapsedMs);
return () => {
window.clearTimeout(hideTimer);
};
}, [activeExecution, executions]);
const openProcessorsFromSheet = useCallback(() => {
if (
@ -514,6 +389,95 @@ export function RecipeStudioPage({
const runDialogLoading =
runDialogKind === "preview" ? previewLoading : fullLoading;
const scheduleFitView = useCallback(
({ delayMs = 0 }: { delayMs?: number } = {}) => {
if (!reactFlowInstance) {
return () => {};
}
let timeoutId = 0;
let frameId = 0;
let retryFrameId = 0;
const fitWithCurrentNodes = () => {
const targetNodes = getFitNodeIdsIgnoringNotes(reactFlowInstance.getNodes());
if (targetNodes.length === 0) {
return false;
}
viewportMovedSinceAutoFitRef.current = false;
reactFlowInstance.fitView({
duration: FIT_ANIMATION_MS,
nodes: targetNodes,
});
return true;
};
const runFit = () => {
if (fitWithCurrentNodes()) {
return;
}
retryFrameId = window.requestAnimationFrame(() => {
fitWithCurrentNodes();
});
};
const start = () => {
frameId = window.requestAnimationFrame(runFit);
};
if (delayMs > 0) {
timeoutId = window.setTimeout(start, delayMs);
} else {
start();
}
return () => {
if (timeoutId) {
window.clearTimeout(timeoutId);
}
if (frameId) {
window.cancelAnimationFrame(frameId);
}
if (retryFrameId) {
window.cancelAnimationFrame(retryFrameId);
}
};
},
[reactFlowInstance],
);
useEffect(() => {
if (previousActiveViewRef.current !== activeView && activeView === "editor") {
pendingEditorTabFitRef.current = true;
forceEditorTabFitRef.current = previousActiveViewRef.current === "executions";
}
previousActiveViewRef.current = activeView;
}, [activeView]);
useEffect(() => {
if (activeView !== "editor" && reactFlowInstance) {
setReactFlowInstance(null);
}
}, [activeView, reactFlowInstance]);
useEffect(() => {
if (
!reactFlowInstance ||
activeView !== "editor" ||
!pendingEditorTabFitRef.current
) {
return;
}
pendingEditorTabFitRef.current = false;
const forceFit = forceEditorTabFitRef.current;
forceEditorTabFitRef.current = false;
if (!forceFit && !viewportMovedSinceAutoFitRef.current) {
return;
}
return scheduleFitView({ delayMs: TAB_SWITCH_FIT_DELAY_MS });
}, [activeView, reactFlowInstance, scheduleFitView]);
useEffect(() => {
if (!reactFlowInstance || fitViewTick === 0 || activeView !== "editor") {
return;
@ -522,28 +486,8 @@ export function RecipeStudioPage({
return;
}
lastProcessedFitTickRef.current = fitViewTick;
let frame2 = 0;
let frame3 = 0;
const frame1 = window.requestAnimationFrame(() => {
frame2 = window.requestAnimationFrame(() => {
frame3 = window.requestAnimationFrame(() => {
reactFlowInstance.fitView({
duration: 320,
nodes: getFitNodeIdsIgnoringNotes(reactFlowInstance.getNodes()),
});
});
});
});
return () => {
window.cancelAnimationFrame(frame1);
if (frame2) {
window.cancelAnimationFrame(frame2);
}
if (frame3) {
window.cancelAnimationFrame(frame3);
}
};
}, [activeView, fitViewTick, reactFlowInstance]);
return scheduleFitView();
}, [activeView, fitViewTick, reactFlowInstance, scheduleFitView]);
return (
<div className="min-h-screen bg-background">
@ -584,9 +528,14 @@ export function RecipeStudioPage({
onNodeClick={handleNodeClick}
onNodeDoubleClick={handleNodeDoubleClick}
isValidConnection={isValidConnection}
nodesDraggable={interactive}
nodesConnectable={interactive}
elementsSelectable={interactive}
onMoveEnd={(event) => {
if (event) {
viewportMovedSinceAutoFitRef.current = true;
}
}}
nodesDraggable={canvasInteractive}
nodesConnectable={canvasInteractive}
elementsSelectable={canvasInteractive}
fitView={false}
className="h-full w-full rounded-t-none"
>
@ -639,6 +588,7 @@ export function RecipeStudioPage({
onAddModelProvider={handleAddModelProviderFromSheet}
onAddModelConfig={handleAddModelConfigFromSheet}
onAddExpression={handleAddExpressionFromSheet}
onAddValidator={handleAddValidatorFromSheet}
onAddMarkdownNote={handleAddMarkdownNoteFromSheet}
onOpenProcessors={openProcessorsFromSheet}
copied={copied}
@ -647,35 +597,34 @@ export function RecipeStudioPage({
/>
</Panel>
<ViewportControls
interactive={interactive}
interactive={canvasInteractive}
lockDisabled={executionLocked}
onToggleInteractive={toggleInteractive}
/>
<div className="pointer-events-none absolute inset-x-0 bottom-3 z-20 flex justify-center">
<div className="pointer-events-auto flex items-center gap-2">
<Button
type="button"
className="h-11 px-5"
onClick={() => openRunDialog(runDialogKind)}
disabled={previewLoading || fullLoading}
>
<HugeiconsIcon icon={CookBookIcon} className="size-4" />
{previewLoading || fullLoading ? "Running..." : "Run"}
</Button>
<Button
type="button"
variant="outline"
className="h-11 px-5"
onClick={() => {
openRunDialog(runDialogKind);
void validateFromDialog();
}}
disabled={validateLoading}
>
<HugeiconsIcon icon={TestTube01Icon} className="size-4" />
{validateLoading ? "Validating..." : "Validate"}
</Button>
</div>
</div>
{islandExecution &&
(isExecutionInProgress(islandExecution.status) ||
islandExecution.status === "completed") && (
<Panel position="top-center" className="!m-0">
<ExecutionProgressIsland
execution={islandExecution}
currentColumnIcon={currentColumnIcon}
minimized={runtimeIslandMinimized}
onMinimizedChange={setRuntimeIslandMinimized}
onViewExecutions={() => setActiveView("executions")}
/>
</Panel>
)}
<RunValidateFloatingControls
runBusy={runBusy}
runDialogKind={runDialogKind}
validateLoading={validateLoading}
executionLocked={executionLocked}
onOpenRunDialog={openRunDialog}
onValidate={() => {
openRunDialog(runDialogKind);
void validateFromDialog();
}}
/>
</ReactFlow>
) : (
<ExecutionsView
@ -698,6 +647,7 @@ export function RecipeStudioPage({
open={dialogOpen}
onOpenChange={setDialogOpen}
config={config}
readOnly={executionLocked}
categoryOptions={dialogOptions.categoryOptions}
modelConfigAliases={dialogOptions.modelConfigAliases}
modelProviderOptions={dialogOptions.modelProviderOptions}
@ -724,6 +674,8 @@ export function RecipeStudioPage({
kind={runDialogKind}
onKindChange={setRunDialogKind}
rows={runDialogRows}
fullRunName={fullRunName}
onFullRunNameChange={setFullRunName}
onRowsChange={(rows) => {
if (runDialogKind === "preview") {
setPreviewRows(rows);

View file

@ -1,9 +1,12 @@
import { type Edge, addEdge } from "@xyflow/react";
import type {
LayoutDirection,
ModelConfig,
NodeConfig,
SamplerConfig,
ValidatorConfig,
} from "../../types";
import { applyRecipeConnection } from "../../utils/graph";
import { isCategoryConfig, isSubcategoryConfig } from "../../utils";
import { HANDLE_IDS } from "../../utils/handles";
@ -30,13 +33,17 @@ function addRecipeEdge(edges: Edge[], source: string, target: string): Edge[] {
);
}
function addSemanticEdge(edges: Edge[], source: string, target: string): Edge[] {
function addValidatorSemanticEdge(
edges: Edge[],
source: string,
target: string,
): Edge[] {
return addEdge(
{
source,
target,
sourceHandle: HANDLE_IDS.semanticOut,
targetHandle: HANDLE_IDS.semanticIn,
sourceHandle: HANDLE_IDS.dataOut,
targetHandle: HANDLE_IDS.dataIn,
type: "semantic",
},
edges,
@ -66,6 +73,7 @@ export function syncEdgesForConfigPatch(
patch: Partial<NodeConfig>,
configs: Record<string, NodeConfig>,
edges: Edge[],
layoutDirection: LayoutDirection,
): Edge[] {
let nextEdges = edges;
@ -88,6 +96,9 @@ export function syncEdgesForConfigPatch(
);
if (current.kind === "model_config" && hasProviderPatch) {
const nextProvider = (patch as Partial<ModelConfig>).provider ?? "";
if (nextProvider.trim() === current.provider.trim()) {
return nextEdges;
}
nextEdges = removeTargetEdgesBySource(
nextEdges,
configs,
@ -97,7 +108,18 @@ export function syncEdgesForConfigPatch(
if (nextProvider) {
const providerId = findNodeIdByName(configs, nextProvider);
if (providerId) {
nextEdges = addSemanticEdge(nextEdges, providerId, current.id);
const result = applyRecipeConnection(
{
source: providerId,
sourceHandle: HANDLE_IDS.semanticOut,
target: current.id,
targetHandle: HANDLE_IDS.semanticIn,
},
configs,
nextEdges,
layoutDirection,
);
nextEdges = result.edges;
}
}
}
@ -145,6 +167,9 @@ export function syncEdgesForConfigPatch(
if (current.kind === "llm" && hasModelAliasPatch) {
const nextAlias =
(patch as Partial<NodeConfig> & { model_alias?: string }).model_alias ?? "";
if (nextAlias.trim() === current.model_alias.trim()) {
return nextEdges;
}
nextEdges = removeTargetEdgesBySource(
nextEdges,
configs,
@ -154,7 +179,54 @@ export function syncEdgesForConfigPatch(
if (nextAlias) {
const modelConfigId = findNodeIdByName(configs, nextAlias);
if (modelConfigId) {
nextEdges = addSemanticEdge(nextEdges, modelConfigId, current.id);
const result = applyRecipeConnection(
{
source: modelConfigId,
sourceHandle: HANDLE_IDS.semanticOut,
target: current.id,
targetHandle: HANDLE_IDS.semanticIn,
},
configs,
nextEdges,
layoutDirection,
);
nextEdges = result.edges;
}
}
}
const hasValidatorTargetsPatch = Object.prototype.hasOwnProperty.call(
patch,
"target_columns",
);
if (current.kind === "validator" && hasValidatorTargetsPatch) {
const nextTargets =
((patch as Partial<ValidatorConfig>).target_columns ?? [])
.map((value) => value.trim())
.filter(Boolean);
nextEdges = nextEdges.filter((edge) => {
if (edge.source !== current.id && edge.target !== current.id) {
return true;
}
const otherId = edge.source === current.id ? edge.target : edge.source;
const other = configs[otherId];
return !(
other &&
other.kind === "llm" &&
other.llm_type === "code"
);
});
const nextTargetName = nextTargets[0];
if (nextTargetName) {
const targetId = findNodeIdByName(configs, nextTargetName);
const target = targetId ? configs[targetId] : null;
if (
targetId &&
target &&
target.kind === "llm" &&
target.llm_type === "code"
) {
nextEdges = addValidatorSemanticEdge(nextEdges, targetId, current.id);
}
}
}

View file

@ -68,13 +68,19 @@ function findNonOverlappingPosition(
return preferred;
}
function isProviderToConfigEdge(edge: Edge, configs: Record<string, NodeConfig>): boolean {
function isProviderToConfigEdge(
edge: Edge,
configs: Record<string, NodeConfig>,
): boolean {
const source = configs[edge.source];
const target = configs[edge.target];
return source?.kind === "model_provider" && target?.kind === "model_config";
}
function isConfigToLlmEdge(edge: Edge, configs: Record<string, NodeConfig>): boolean {
function isConfigToLlmEdge(
edge: Edge,
configs: Record<string, NodeConfig>,
): boolean {
const source = configs[edge.source];
const target = configs[edge.target];
return source?.kind === "model_config" && target?.kind === "llm";
@ -84,17 +90,29 @@ function usageKey(nodeId: string, handleId: string): string {
return `${nodeId}::${handleId}`;
}
function incrementUsage(map: Map<string, number>, nodeId: string, handleId: string): void {
function incrementUsage(
map: Map<string, number>,
nodeId: string,
handleId: string,
): void {
const key = usageKey(nodeId, handleId);
map.set(key, (map.get(key) ?? 0) + 1);
}
function decrementUsage(map: Map<string, number>, nodeId: string, handleId: string): void {
function decrementUsage(
map: Map<string, number>,
nodeId: string,
handleId: string,
): void {
const key = usageKey(nodeId, handleId);
map.set(key, Math.max(0, (map.get(key) ?? 0) - 1));
}
function getUsage(map: Map<string, number>, nodeId: string, handleId: string): number {
function getUsage(
map: Map<string, number>,
nodeId: string,
handleId: string,
): number {
return map.get(usageKey(nodeId, handleId)) ?? 0;
}
@ -103,7 +121,9 @@ function pickHandleByUsage(
nodeId: string,
usageMap: Map<string, number>,
): string {
const free = candidates.filter((handleId) => getUsage(usageMap, nodeId, handleId) === 0);
const free = candidates.filter(
(handleId) => getUsage(usageMap, nodeId, handleId) === 0,
);
if (free.length > 0) {
return free[0];
}
@ -140,7 +160,10 @@ function getNodeCenter(node: RecipeNode): { x: number; y: number } {
};
}
function collectBounds(ids: string[], nodesById: Map<string, RecipeNode>): Bounds | null {
function collectBounds(
ids: string[],
nodesById: Map<string, RecipeNode>,
): Bounds | null {
const rects = ids
.map((id) => nodesById.get(id))
.flatMap((node) => (node ? [toRect(node)] : []));
@ -186,20 +209,26 @@ function sortPreferredLlmTargetHandles(
return [...verticalFirst, HANDLE_IDS.dataIn, HANDLE_IDS.dataInRight];
}
function getProviderSourceHandleCandidates(direction: LayoutDirection): string[] {
function getProviderSourceHandleCandidates(
direction: LayoutDirection,
): string[] {
return direction === "TB"
? [HANDLE_IDS.semanticOut, HANDLE_IDS.semanticOutBottom]
: [HANDLE_IDS.semanticOutBottom, HANDLE_IDS.semanticOut];
}
function getProviderTargetHandleCandidates(direction: LayoutDirection): string[] {
function getProviderTargetHandleCandidates(
direction: LayoutDirection,
): string[] {
return direction === "TB"
? [HANDLE_IDS.semanticIn, HANDLE_IDS.semanticInTop]
: [HANDLE_IDS.semanticInTop, HANDLE_IDS.semanticIn];
}
function getConfigSourceHandleCandidates(direction: LayoutDirection): string[] {
return direction === "TB" ? [HANDLE_IDS.semanticOut] : [HANDLE_IDS.semanticOutBottom];
return direction === "TB"
? [HANDLE_IDS.semanticOut]
: [HANDLE_IDS.semanticOutBottom];
}
export function optimizeModelInfraEdgeHandles(
@ -251,8 +280,16 @@ export function optimizeModelInfraEdgeHandles(
if (isProviderToConfigEdge(edge, configs)) {
const sourceCandidates = getProviderSourceHandleCandidates(direction);
const targetCandidates = getProviderTargetHandleCandidates(direction);
const sourceHandle = pickHandleByUsage(sourceCandidates, edge.source, sourceUsage);
const targetHandle = pickHandleByUsage(targetCandidates, edge.target, targetUsage);
const sourceHandle = pickHandleByUsage(
sourceCandidates,
edge.source,
sourceUsage,
);
const targetHandle = pickHandleByUsage(
targetCandidates,
edge.target,
targetUsage,
);
nextEdges.push(
applyEdgeWithHandles(
edge,
@ -265,14 +302,22 @@ export function optimizeModelInfraEdgeHandles(
continue;
}
const sourceCandidates = getConfigSourceHandleCandidates(direction);
const targetCandidates = sortPreferredLlmTargetHandles(
direction,
nodesById.get(edge.source),
nodesById.get(edge.target),
);
const sourceHandle = pickHandleByUsage(sourceCandidates, edge.source, sourceUsage);
const targetHandle = pickHandleByUsage(targetCandidates, edge.target, targetUsage);
const sourceCandidates = getConfigSourceHandleCandidates(direction);
const sourceHandle = pickHandleByUsage(
sourceCandidates,
edge.source,
sourceUsage,
);
const targetHandle = pickHandleByUsage(
targetCandidates,
edge.target,
targetUsage,
);
nextEdges.push(
applyEdgeWithHandles(
edge,
@ -316,13 +361,19 @@ export function centerModelInfraNodes(
}
const modelConfigIds = Object.values(configs)
.filter((config) => config.kind === "model_config" && nodesById.has(config.id))
.filter(
(config) => config.kind === "model_config" && nodesById.has(config.id),
)
.map((config) => config.id);
const modelProviderIds = Object.values(configs)
.filter((config) => config.kind === "model_provider" && nodesById.has(config.id))
.filter(
(config) => config.kind === "model_provider" && nodesById.has(config.id),
)
.map((config) => config.id);
const occupiedById = new Map(nodes.map((node) => [node.id, toRect(node)] as const));
const occupiedById = new Map(
nodes.map((node) => [node.id, toRect(node)] as const),
);
const clusterGap = 72;
const placeNode = (nodeId: string, preferred: XYPosition): void => {

View file

@ -83,6 +83,17 @@ export function applyRenameToConfig(
const base = next as LlmConfig;
next = { ...base, model_alias: to };
}
if (config.kind === "validator") {
const targets = config.target_columns ?? [];
if (targets.includes(from)) {
const base = next as typeof config;
next = {
...base,
// biome-ignore lint/style/useNamingConvention: api schema
target_columns: targets.map((target) => (target === from ? to : target)),
};
}
}
return next;
}
@ -125,6 +136,17 @@ export function applyRemovalToConfig(
const base = next as LlmConfig;
next = { ...base, model_alias: "" };
}
if (config.kind === "validator") {
const targets = (config.target_columns ?? []).filter((target) => target !== ref);
if (targets.length !== (config.target_columns ?? []).length) {
const base = next as typeof config;
next = {
...base,
// biome-ignore lint/style/useNamingConvention: api schema
target_columns: targets,
};
}
}
return next;
}

View file

@ -73,6 +73,19 @@ export function applyEdgeRemovals(
}
next[target.id] = updated;
}
if (
source.kind === "validator" &&
target.kind === "llm" &&
target.llm_type === "code"
) {
const sourceUpdated = applyRemovalToConfig(source, target.name);
if (sourceUpdated !== source) {
if (next === configs) {
next = { ...configs };
}
next[source.id] = sourceUpdated;
}
}
}
return next;
}

View file

@ -18,13 +18,13 @@ export type RecipeRunSettings = {
const DEFAULT_RUN_SETTINGS: RecipeRunSettings = {
batchSize: 1000,
batchEnabled: true,
batchEnabled: false,
mergeBatches: false,
llmParallelRequests: null,
nonInferenceWorkers: 4,
maxConversationRestarts: 5,
maxConversationCorrectionSteps: 0,
disableEarlyShutdown: false,
disableEarlyShutdown: true,
shutdownErrorRate: 0.5,
shutdownErrorWindow: 10,
};
@ -34,6 +34,7 @@ type RecipeExecutionsState = {
runDialogKind: RecipeExecutionKind;
previewRows: number;
fullRows: number;
fullRunName: string;
runErrors: string[];
runSettings: RecipeRunSettings;
previewLoading: boolean;
@ -44,6 +45,7 @@ type RecipeExecutionsState = {
setRunDialogKind: (kind: RecipeExecutionKind) => void;
setPreviewRows: (rows: number) => void;
setFullRows: (rows: number) => void;
setFullRunName: (name: string) => void;
setRunErrors: (errors: string[]) => void;
setRunSettings: (patch: Partial<RecipeRunSettings>) => void;
setPreviewLoading: (loading: boolean) => void;
@ -58,7 +60,8 @@ const INITIAL_STATE = {
runDialogOpen: false,
runDialogKind: "preview",
previewRows: 5,
fullRows: 1000,
fullRows: 100,
fullRunName: "",
runErrors: [],
runSettings: DEFAULT_RUN_SETTINGS,
previewLoading: false,
@ -71,6 +74,7 @@ const INITIAL_STATE = {
| "runDialogKind"
| "previewRows"
| "fullRows"
| "fullRunName"
| "runErrors"
| "runSettings"
| "previewLoading"
@ -82,11 +86,25 @@ const INITIAL_STATE = {
export const useRecipeExecutionsStore = create<RecipeExecutionsState>((set) => ({
...INITIAL_STATE,
setRunDialogOpen: (open) => set({ runDialogOpen: open }),
setRunDialogKind: (kind) => set({ runDialogKind: kind }),
setRunDialogKind: (kind) =>
set((state) => {
if (state.runDialogKind === "preview" && kind === "full") {
return {
runDialogKind: kind,
fullRows: 100,
runSettings: {
...state.runSettings,
batchEnabled: false,
},
};
}
return { runDialogKind: kind };
}),
setPreviewRows: (rows) =>
set({ previewRows: Number.isFinite(rows) && rows > 0 ? Math.floor(rows) : 1 }),
setFullRows: (rows) =>
set({ fullRows: Number.isFinite(rows) && rows > 0 ? Math.floor(rows) : 1 }),
setFullRunName: (name) => set({ fullRunName: name }),
setRunErrors: (errors) => set({ runErrors: errors }),
setRunSettings: (patch) =>
set((state) => ({

View file

@ -52,6 +52,7 @@ type SheetView =
| "sampler"
| "seed"
| "llm"
| "validator"
| "expression"
| "note"
| "processor";
@ -67,12 +68,14 @@ type RecipeStudioState = {
activeConfigId: string | null;
dialogOpen: boolean;
layoutDirection: LayoutDirection;
executionLocked: boolean;
nextId: number;
nextY: number;
fitViewTick: number;
setSheetView: (view: SheetView) => void;
setProcessors: (processors: RecipeProcessorConfig[]) => void;
setDialogOpen: (open: boolean) => void;
setExecutionLocked: (locked: boolean) => void;
resetRecipe: () => void;
selectConfig: (id: string) => void;
openConfig: (id: string) => void;
@ -93,6 +96,11 @@ type RecipeStudioState = {
addModelProviderNode: (position?: XYPosition, openDialog?: boolean) => void;
addModelConfigNode: (position?: XYPosition, openDialog?: boolean) => void;
addExpressionNode: (position?: XYPosition, openDialog?: boolean) => void;
addValidatorNode: (
type: "validator_python" | "validator_sql" | "validator_oxc",
position?: XYPosition,
openDialog?: boolean,
) => void;
addMarkdownNoteNode: (position?: XYPosition, openDialog?: boolean) => void;
updateConfig: (id: string, patch: Partial<NodeConfig>) => void;
loadRecipe: (snapshot: RecipeSnapshot) => void;
@ -114,6 +122,7 @@ const INITIAL_STATE = {
activeConfigId: null,
dialogOpen: false,
layoutDirection: "LR",
executionLocked: false,
nextId: 3,
nextY: 280,
fitViewTick: 0,
@ -129,6 +138,7 @@ const INITIAL_STATE = {
| "activeConfigId"
| "dialogOpen"
| "layoutDirection"
| "executionLocked"
| "nextId"
| "nextY"
| "fitViewTick"
@ -213,6 +223,7 @@ function connectSemantic(
configs: Record<string, NodeConfig>,
sourceId: string,
targetId: string,
layoutDirection: LayoutDirection,
): { edges: Edge[]; configs: Record<string, NodeConfig> } {
const result = applyRecipeConnection(
{
@ -223,6 +234,7 @@ function connectSemantic(
},
configs,
edges,
layoutDirection,
);
return {
edges: result.edges,
@ -244,35 +256,45 @@ function isModelSemanticEdge(edge: Edge, configs: Record<string, NodeConfig>): b
export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
...INITIAL_STATE,
setSheetView: (view) => set({ sheetView: view }),
setProcessors: (processors) => set({ processors }),
setProcessors: (processors) =>
set((state) => (state.executionLocked ? state : { processors })),
setDialogOpen: (open) => set({ dialogOpen: open }),
setExecutionLocked: (locked) => set({ executionLocked: locked }),
resetRecipe: () => set(INITIAL_STATE),
selectConfig: (id) => set({ activeConfigId: id, dialogOpen: false }),
openConfig: (id) => set({ activeConfigId: id, dialogOpen: true }),
setLayoutDirection: (direction) =>
set((state) => ({
layoutDirection: direction,
edges: state.edges.map((edge) => {
if (isModelSemanticEdge(edge, state.configs)) {
set((state) => {
if (state.executionLocked) {
return state;
}
return {
layoutDirection: direction,
edges: state.edges.map((edge) => {
if (isModelSemanticEdge(edge, state.configs)) {
return {
...edge,
sourceHandle: normalizeRecipeHandleId(edge.sourceHandle),
targetHandle: normalizeRecipeHandleId(edge.targetHandle),
};
}
return {
...edge,
sourceHandle: normalizeRecipeHandleId(edge.sourceHandle),
targetHandle: normalizeRecipeHandleId(edge.targetHandle),
...remapRecipeEdgeHandlesForLayout(edge, direction),
};
}
return {
...edge,
...remapRecipeEdgeHandlesForLayout(edge, direction),
};
}),
nodes: applyLayoutDirectionToNodes(
state.nodes,
state.configs,
direction,
),
})),
}),
nodes: applyLayoutDirectionToNodes(
state.nodes,
state.configs,
direction,
),
};
}),
applyLayout: () =>
set((state) => {
if (state.executionLocked) {
return state;
}
const isTopBottom = state.layoutDirection === "TB";
const displayGraph = deriveDisplayGraph({
@ -333,11 +355,17 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
};
}),
addSamplerNode: (type, position, openDialog = true) =>
set((state) =>
buildAddedNodeState(state, "sampler", type, position, openDialog),
),
set((state) => {
if (state.executionLocked) {
return state;
}
return buildAddedNodeState(state, "sampler", type, position, openDialog);
}),
addSeedNode: (type, position, openDialog = true) =>
set((state) => {
if (state.executionLocked) {
return state;
}
const existing = Object.values(state.configs).find(
(config) => config.kind === "seed",
);
@ -390,11 +418,17 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
};
}),
addLlmNode: (type, position, openDialog = true) =>
set((state) =>
buildAddedNodeState(state, "llm", type, position, openDialog),
),
set((state) => {
if (state.executionLocked) {
return state;
}
return buildAddedNodeState(state, "llm", type, position, openDialog);
}),
addModelProviderNode: (position, openDialog = true) =>
set((state) => {
if (state.executionLocked) {
return state;
}
const added = buildAddedNodeState(
state,
"llm",
@ -428,6 +462,7 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
configs,
context.newNodeId,
unboundModelConfigs[0].id,
state.layoutDirection,
);
edges = next.edges;
configs = next.configs;
@ -436,6 +471,9 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
}),
addModelConfigNode: (position, openDialog = true) =>
set((state) => {
if (state.executionLocked) {
return state;
}
const added = buildAddedNodeState(
state,
"llm",
@ -478,6 +516,7 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
configs,
providers[0].id,
context.newNodeId,
state.layoutDirection,
);
edges = next.edges;
configs = next.configs;
@ -488,6 +527,7 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
configs,
context.newNodeId,
unboundLlms[0].id,
state.layoutDirection,
);
edges = next.edges;
configs = next.configs;
@ -495,25 +535,44 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
return { ...added, nodes, edges, configs };
}),
addExpressionNode: (position, openDialog = true) =>
set((state) =>
buildAddedNodeState(
set((state) => {
if (state.executionLocked) {
return state;
}
return buildAddedNodeState(
state,
"expression",
"expression",
position,
openDialog,
),
),
);
}),
addValidatorNode: (type, position, openDialog = true) =>
set((state) => {
if (state.executionLocked) {
return state;
}
return buildAddedNodeState(
state,
"validator",
type,
position,
openDialog,
);
}),
addMarkdownNoteNode: (position, openDialog = true) =>
set((state) =>
buildAddedNodeState(
set((state) => {
if (state.executionLocked) {
return state;
}
return buildAddedNodeState(
state,
"note",
"markdown_note",
position,
openDialog,
),
),
);
}),
loadRecipe: (snapshot) =>
set((state) => ({
configs: snapshot.configs,
@ -549,6 +608,9 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
}),
updateConfig: (id, patch) => {
const applyUpdate = (state: RecipeStudioState) => {
if (state.executionLocked) {
return state;
}
const current = state.configs[id];
if (!current) {
return state;
@ -567,7 +629,13 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
next,
state.layoutDirection,
);
const edges = syncEdgesForConfigPatch(current, patch, configs, state.edges);
const edges = syncEdgesForConfigPatch(
current,
patch,
configs,
state.edges,
state.layoutDirection,
);
configs = syncSubcategoryConfigsForCategoryUpdate(
current,
next,
@ -587,6 +655,9 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
},
onNodesChange: (changes) => {
const applyNodesChange = (state: RecipeStudioState) => {
if (state.executionLocked) {
return state;
}
const removedIds = changes
.filter((change) => change.type === "remove")
.map((change) => change.id);
@ -615,6 +686,9 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
},
onEdgesChange: (changes) => {
set((state) => {
if (state.executionLocked) {
return state;
}
const removedEdges = changes
.filter((change) => change.type === "remove")
.map((change) => state.edges.find((edge) => edge.id === change.id))
@ -628,10 +702,14 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
},
onConnect: (connection) => {
set((state) => {
if (state.executionLocked) {
return state;
}
const result = applyRecipeConnection(
connection,
state.configs,
state.edges,
state.layoutDirection,
);
return result.configs
? { edges: result.edges, configs: result.configs }

View file

@ -13,6 +13,21 @@ export type SamplerType =
| "person_from_faker";
export type LlmType = "text" | "structured" | "code" | "judge";
export type ValidatorCodeLang =
| "javascript"
| "typescript"
| "jsx"
| "tsx"
| "python"
| "sql:sqlite"
| "sql:postgres"
| "sql:mysql"
| "sql:tsql"
| "sql:bigquery"
| "sql:ansi";
export type ValidatorType = "code" | "oxc";
export type OxcValidationMode = "syntax" | "lint" | "syntax+lint";
export type OxcCodeShape = "auto" | "module" | "snippet";
export type ExpressionDtype = "str" | "int" | "float" | "bool";
@ -28,6 +43,7 @@ export type RecipeNodeData = {
kind:
| "sampler"
| "llm"
| "validator"
| "expression"
| "seed"
| "note"
@ -37,12 +53,17 @@ export type RecipeNodeData = {
blockType:
| SamplerType
| LlmType
| "validator_python"
| "validator_sql"
| "validator_oxc"
| "expression"
| "seed"
| "markdown_note"
| "model_provider"
| "model_config";
layoutDirection?: LayoutDirection;
runtimeState?: "idle" | "running" | "done";
executionLocked?: boolean;
};
export type RecipeNode = Node<RecipeNodeData, "builder">;
@ -57,6 +78,8 @@ export type CategoryConditionalParams = {
export type SamplerConfig = {
id: string;
kind: "sampler";
// ui-only
advancedOpen?: boolean;
// biome-ignore lint/style/useNamingConvention: api schema
sampler_type: SamplerType;
name: string;
@ -150,9 +173,19 @@ export type LlmToolConfig = {
timeout_sec?: string;
};
export type LlmImageContextConfig = {
enabled: boolean;
// biome-ignore lint/style/useNamingConvention: api schema
column_name: string;
};
export type LlmTraceType = "none" | "last_message" | "all_messages";
export type LlmConfig = {
id: string;
kind: "llm";
// ui-only
advancedOpen?: boolean;
// biome-ignore lint/style/useNamingConvention: api schema
llm_type: LlmType;
name: string;
@ -173,6 +206,13 @@ export type LlmConfig = {
// biome-ignore lint/style/useNamingConvention: ui schema
mcp_providers?: LlmMcpProviderConfig[];
scores?: Score[];
// ui-only, serialized into multi_modal_context for DataDesigner
// biome-ignore lint/style/useNamingConvention: ui schema
image_context?: LlmImageContextConfig;
// biome-ignore lint/style/useNamingConvention: api schema
with_trace?: LlmTraceType;
// biome-ignore lint/style/useNamingConvention: api schema
extract_reasoning_content?: boolean;
};
export type ModelProviderConfig = {
@ -205,6 +245,10 @@ export type ModelConfig = {
// biome-ignore lint/style/useNamingConvention: api schema
inference_max_tokens?: string;
// biome-ignore lint/style/useNamingConvention: api schema
inference_timeout?: string;
// biome-ignore lint/style/useNamingConvention: api schema
inference_extra_body?: string;
// biome-ignore lint/style/useNamingConvention: api schema
skip_health_check?: boolean;
};
@ -217,6 +261,27 @@ export type ExpressionConfig = {
dtype: ExpressionDtype;
};
export type ValidatorConfig = {
id: string;
kind: "validator";
// ui-only
advancedOpen?: boolean;
name: string;
drop?: boolean;
// biome-ignore lint/style/useNamingConvention: api schema
target_columns: string[];
// ui-only
validator_type: ValidatorType;
// biome-ignore lint/style/useNamingConvention: api schema
code_lang: ValidatorCodeLang;
// ui-only (used for OXC validators)
oxc_validation_mode: OxcValidationMode;
// ui-only (used for OXC validators)
oxc_code_shape: OxcCodeShape;
// ui ergonomics (serialized to int in payload)
batch_size: string;
};
export type MarkdownNoteConfig = {
id: string;
kind: "markdown_note";
@ -231,6 +296,8 @@ export type MarkdownNoteConfig = {
export type SeedConfig = {
id: string;
kind: "seed";
// ui-only
advancedOpen?: boolean;
name: string;
drop?: boolean;
// ui-only: explicit per-column drop for structured seed sources (hf/local)
@ -277,6 +344,7 @@ export type RecipeProcessorConfig = SchemaTransformProcessorConfig;
export type NodeConfig =
| SamplerConfig
| LlmConfig
| ValidatorConfig
| ExpressionConfig
| MarkdownNoteConfig
| SeedConfig

View file

@ -10,6 +10,9 @@ import type {
SeedSourceType,
SamplerConfig,
SamplerType,
ValidatorCodeLang,
ValidatorType,
ValidatorConfig,
} from "../types";
import { nextName } from "./naming";
@ -204,20 +207,17 @@ export function makeLlmConfig(
tool_configs: [],
// biome-ignore lint/style/useNamingConvention: ui schema
mcp_providers: [],
scores:
llmType === "judge"
? [
{
name: "Quality",
description: "Overall quality based on the criteria.",
options: [
{ value: "1", description: "Poor" },
{ value: "3", description: "Acceptable" },
{ value: "5", description: "Excellent" },
],
},
]
: undefined,
// biome-ignore lint/style/useNamingConvention: ui schema
image_context: {
enabled: false,
// biome-ignore lint/style/useNamingConvention: api schema
column_name: "",
},
// biome-ignore lint/style/useNamingConvention: api schema
with_trace: "none",
// biome-ignore lint/style/useNamingConvention: api schema
extract_reasoning_content: false,
scores: llmType === "judge" ? [] : undefined,
};
}
@ -256,10 +256,14 @@ export function makeModelConfig(
// biome-ignore lint/style/useNamingConvention: api schema
inference_temperature: "0.7",
// biome-ignore lint/style/useNamingConvention: api schema
inference_max_tokens: "256",
inference_max_tokens: "",
// biome-ignore lint/style/useNamingConvention: api schema
inference_top_p: "",
// biome-ignore lint/style/useNamingConvention: api schema
inference_timeout: "",
// biome-ignore lint/style/useNamingConvention: api schema
inference_extra_body: "",
// biome-ignore lint/style/useNamingConvention: api schema
skip_health_check: false,
};
}
@ -278,6 +282,36 @@ export function makeExpressionConfig(
};
}
export function makeValidatorConfig(
id: string,
validatorType: ValidatorType,
codeLang: ValidatorCodeLang,
existing: NodeConfig[],
): ValidatorConfig {
const isSql = validatorType === "code" && codeLang.startsWith("sql:");
const isOxc = validatorType === "oxc";
let namePrefix = "validator_python";
if (isSql) {
namePrefix = "validator_sql";
} else if (isOxc) {
namePrefix = "validator_oxc";
}
return {
id,
kind: "validator",
name: nextName(existing, namePrefix),
drop: false,
// biome-ignore lint/style/useNamingConvention: api schema
target_columns: [],
validator_type: validatorType,
// biome-ignore lint/style/useNamingConvention: api schema
code_lang: codeLang,
oxc_validation_mode: "syntax",
oxc_code_shape: "auto",
batch_size: "10",
};
}
export function makeMarkdownNoteConfig(
id: string,
existing: NodeConfig[],

View file

@ -3,6 +3,7 @@ import type {
LlmConfig,
NodeConfig,
SamplerConfig,
ValidatorConfig,
} from "../types";
export function isSamplerConfig(
@ -40,3 +41,9 @@ export function isExpressionConfig(
): config is ExpressionConfig {
return Boolean(config && config.kind === "expression");
}
export function isValidatorConfig(
config: NodeConfig | null | undefined,
): config is ValidatorConfig {
return Boolean(config && config.kind === "validator");
}

View file

@ -24,6 +24,12 @@ type DisplayGraphInput = {
layoutDirection: LayoutDirection;
auxNodePositions: Record<string, XYPosition>;
llmAuxVisibility: Record<string, boolean>;
runtime?: {
runningNodeId: string | null;
doneNodeIds: Set<string>;
activeEdgeIds: Set<string>;
executionLocked: boolean;
};
};
export type DisplayGraph = {
@ -31,27 +37,55 @@ export type DisplayGraph = {
edges: Edge[];
};
function isAuxEdge(edge: Edge): boolean {
return edge.source.startsWith("aux-") || edge.target.startsWith("aux-");
}
function normalizeEdge(
edge: Edge,
configs: Record<string, NodeConfig>,
layoutDirection: LayoutDirection,
activeEdgeIds: Set<string>,
runningNodeId: string | null,
doneNodeIds: Set<string>,
): Edge {
const baseStyle = { stroke: "var(--foreground)", strokeWidth: 2 };
const isAux = edge.source.startsWith("aux-") || edge.target.startsWith("aux-");
const isActiveByRuntimeTarget =
Boolean(runningNodeId) &&
edge.target === runningNodeId &&
!isAuxEdge(edge);
const isActiveEdge = activeEdgeIds.has(edge.id) || isActiveByRuntimeTarget;
const isAux = isAuxEdge(edge);
if (isAux) {
return {
...edge,
type: "canvas",
data: { ...(edge.data ?? {}), path: "smoothstep" },
style: { ...baseStyle, ...(edge.style ?? {}) },
data: { ...(edge.data ?? {}), path: "smoothstep", active: isActiveEdge },
animated: isActiveEdge,
};
}
const source = configs[edge.source];
const target = configs[edge.target];
const semantic = Boolean(source && target) && isSemanticRelation(source, target);
const sourceHandleNormalized = normalizeRecipeHandleId(edge.sourceHandle);
const targetHandleNormalized = normalizeRecipeHandleId(edge.targetHandle);
const isActiveReversedRuntimeEdge =
Boolean(runningNodeId) &&
isActiveEdge &&
edge.source === runningNodeId &&
doneNodeIds.has(edge.target);
const displayEdge = isActiveReversedRuntimeEdge
? {
...edge,
source: edge.target,
target: edge.source,
sourceHandle: getDefaultDataSourceHandle(layoutDirection),
targetHandle: getDefaultDataTargetHandle(layoutDirection),
}
: edge;
const source = configs[displayEdge.source];
const target = configs[displayEdge.target];
const semantic =
displayEdge.type === "semantic" ||
(Boolean(source && target) && isSemanticRelation(source, target));
const sourceHandleNormalized = normalizeRecipeHandleId(displayEdge.sourceHandle);
const targetHandleNormalized = normalizeRecipeHandleId(displayEdge.targetHandle);
const semanticSourceDefault =
source?.kind === "llm"
? getDefaultDataSourceHandle(layoutDirection)
@ -74,6 +108,13 @@ function normalizeEdge(
isDataTargetHandle(targetHandleNormalized)
? targetHandleNormalized ?? semanticTargetDefault
: semanticTargetDefault;
// LLM nodes only expose data lane handles; coerce legacy semantic handles.
if (source?.kind === "llm" && isSemanticSourceHandle(sourceHandle)) {
sourceHandle = semanticSourceDefault;
}
if (target?.kind === "llm" && isSemanticTargetHandle(targetHandle)) {
targetHandle = semanticTargetDefault;
}
} else {
sourceHandle = isDataSourceHandle(sourceHandleNormalized)
? sourceHandleNormalized ?? getDefaultDataSourceHandle(layoutDirection)
@ -84,12 +125,14 @@ function normalizeEdge(
}
return {
...edge,
...displayEdge,
type: semantic ? "semantic" : "canvas",
data: semantic ? edge.data : { ...(edge.data ?? {}), path: "smoothstep" },
data: semantic
? { ...(displayEdge.data ?? {}), active: isActiveEdge }
: { ...(displayEdge.data ?? {}), path: "smoothstep", active: isActiveEdge },
sourceHandle,
targetHandle,
style: { ...baseStyle, ...(edge.style ?? {}) },
animated: isActiveEdge,
};
}
@ -228,7 +271,7 @@ function pickAuxTargetHandle(
): string {
const occupied = new Set<HandleSide>();
for (const edge of edges) {
if (edge.source.startsWith("aux-") || edge.target.startsWith("aux-")) {
if (isAuxEdge(edge)) {
continue;
}
if (edge.target === llmId) {
@ -361,18 +404,41 @@ export function deriveDisplayGraph({
layoutDirection,
auxNodePositions,
llmAuxVisibility,
runtime,
}: DisplayGraphInput): DisplayGraph {
const executionLocked = runtime?.executionLocked ?? false;
const runningNodeId = runtime?.runningNodeId ?? null;
const doneNodeIds = runtime?.doneNodeIds ?? new Set<string>();
const activeEdgeIds = runtime?.activeEdgeIds ?? new Set<string>();
const displayNodes = nodes.map((node) => {
const hasWidth =
typeof node.width === "number" ||
typeof node.style?.width === "number" ||
(typeof node.style?.width === "string" &&
Number.isFinite(Number.parseFloat(node.style.width)));
const runtimeState: "idle" | "running" | "done" =
node.id === runningNodeId
? "running"
: doneNodeIds.has(node.id)
? "done"
: "idle";
if (hasWidth) {
return node;
return {
...node,
data: {
...node.data,
runtimeState,
executionLocked,
},
};
}
return {
...node,
data: {
...node.data,
runtimeState,
executionLocked,
},
style: { ...node.style, width: DEFAULT_NODE_WIDTH },
};
});
@ -407,6 +473,7 @@ export function deriveDisplayGraph({
llmId: config.id,
field: "system_prompt",
title: "System Prompt",
executionLocked,
},
});
}
@ -419,6 +486,7 @@ export function deriveDisplayGraph({
llmId: config.id,
field: "prompt",
title: "Prompt",
executionLocked,
},
});
}
@ -431,6 +499,7 @@ export function deriveDisplayGraph({
kind: "llm-judge-score",
llmId: config.id,
scoreIndex,
executionLocked,
},
});
});
@ -537,7 +606,14 @@ export function deriveDisplayGraph({
return {
nodes: [...displayNodes, ...auxNodes],
edges: [...edges, ...auxEdges].map((edge) =>
normalizeEdge(edge, configs, layoutDirection),
normalizeEdge(
edge,
configs,
layoutDirection,
activeEdgeIds,
runningNodeId,
doneNodeIds,
),
),
};
}

View file

@ -1,5 +1,5 @@
import { type Connection, type Edge, addEdge } from "@xyflow/react";
import type { NodeConfig, SamplerConfig } from "../../types";
import type { LayoutDirection, NodeConfig, SamplerConfig } from "../../types";
import {
HANDLE_IDS,
isDataSourceHandle,
@ -12,9 +12,12 @@ import { isSemanticRelation } from "./relations";
import {
isCategoryConfig,
isExpressionConfig,
isLlmConfig,
isSubcategoryConfig,
} from "../index";
import {
VALIDATOR_OXC_CODE_LANGS,
VALIDATOR_SQL_CODE_LANGS,
} from "../validators/code-lang";
function buildTemplateWithRef(template: string, ref: string): string {
if (template.includes(ref)) {
@ -78,7 +81,8 @@ type SingleRefRelation =
| "provider"
| "model_alias"
| "reference_column_name"
| "subcategory_parent";
| "subcategory_parent"
| "validator_target_columns";
function getSingleRefRelation(
source: NodeConfig,
@ -101,6 +105,13 @@ function getSingleRefRelation(
if (isCategoryConfig(source) && isSubcategoryConfig(target)) {
return "subcategory_parent";
}
if (
source.kind === "llm" &&
source.llm_type === "code" &&
target.kind === "validator"
) {
return "validator_target_columns";
}
return null;
}
@ -126,6 +137,9 @@ function isCompetingIncomingEdge(
if (relation === "subcategory_parent") {
return isCategoryConfig(source);
}
if (relation === "validator_target_columns") {
return source.kind === "llm" && source.llm_type === "code";
}
return source.kind === "sampler" && source.sampler_type === "datetime";
}
@ -136,6 +150,25 @@ function isModelSemanticRelation(source: NodeConfig, target: NodeConfig): boolea
);
}
function canApplyCodeLangToValidator(
validator: Extract<NodeConfig, { kind: "validator" }>,
codeLang: string,
): boolean {
const normalized = codeLang.trim();
if (!normalized) {
return false;
}
if (validator.validator_type === "oxc") {
return VALIDATOR_OXC_CODE_LANGS.includes(
normalized as typeof validator.code_lang,
);
}
if (normalized === "python") {
return true;
}
return VALIDATOR_SQL_CODE_LANGS.includes(normalized as typeof validator.code_lang);
}
function countHandleUsage(
edges: Edge[],
nodeId: string,
@ -186,20 +219,30 @@ function chooseModelSemanticHandles(
source: NodeConfig,
target: NodeConfig,
edges: Edge[],
layoutDirection: LayoutDirection,
): Connection {
if (!isModelSemanticRelation(source, target)) {
return connection;
}
const sourceCandidates = [HANDLE_IDS.semanticOut, HANDLE_IDS.semanticOutBottom];
const sourceCandidates =
source.kind === "model_config" && target.kind === "llm"
? layoutDirection === "TB"
? [HANDLE_IDS.semanticOut]
: [HANDLE_IDS.semanticOutBottom]
: layoutDirection === "TB"
? [HANDLE_IDS.semanticOut, HANDLE_IDS.semanticOutBottom]
: [HANDLE_IDS.semanticOutBottom, HANDLE_IDS.semanticOut];
const targetCandidates =
target.kind === "model_config"
? [HANDLE_IDS.semanticIn, HANDLE_IDS.semanticInTop]
? layoutDirection === "TB"
? [HANDLE_IDS.semanticIn, HANDLE_IDS.semanticInTop]
: [HANDLE_IDS.semanticInTop, HANDLE_IDS.semanticIn]
: [
HANDLE_IDS.dataIn,
HANDLE_IDS.dataInTop,
HANDLE_IDS.dataInRight,
HANDLE_IDS.dataInBottom,
HANDLE_IDS.dataIn,
HANDLE_IDS.dataInRight,
];
const sourceHandle = pickLeastUsedHandle(
@ -220,6 +263,27 @@ function chooseModelSemanticHandles(
};
}
function normalizeValidatorSemanticConnection(
connection: Connection,
source: NodeConfig,
target: NodeConfig,
): Connection {
if (
source.kind === "validator" &&
target.kind === "llm" &&
target.llm_type === "code"
) {
return {
...connection,
source: target.id,
target: source.id,
sourceHandle: HANDLE_IDS.dataOut,
targetHandle: HANDLE_IDS.dataIn,
};
}
return connection;
}
export function isValidRecipeConnection(
connection: Connection,
configs: Record<string, NodeConfig>,
@ -249,21 +313,46 @@ export function applyRecipeConnection(
connection: Connection,
configs: Record<string, NodeConfig>,
edges: Edge[],
layoutDirection: LayoutDirection = "LR",
): { edges: Edge[]; configs?: Record<string, NodeConfig> } {
if (!isValidRecipeConnection(connection, configs)) {
return { edges };
}
const source = connection.source
const initialSource = connection.source
? configs[connection.source]
: null;
const target = connection.target
const initialTarget = connection.target
? configs[connection.target]
: null;
if (!(initialSource && initialTarget)) {
return { edges };
}
const normalizedConnection = normalizeValidatorSemanticConnection(
connection,
initialSource,
initialTarget,
);
const source = normalizedConnection.source
? configs[normalizedConnection.source]
: null;
const target = normalizedConnection.target
? configs[normalizedConnection.target]
: null;
if (!(source && target)) {
return { edges };
}
const semanticRelation = isSemanticRelation(source, target);
const singleRefRelation = getSingleRefRelation(source, target);
if (
singleRefRelation === "subcategory_parent" &&
isSubcategoryConfig(target)
) {
const currentParent = target.subcategory_parent?.trim() ?? "";
if (currentParent && currentParent !== source.name) {
return { edges };
}
}
const nextBaseEdges = singleRefRelation
? edges.filter(
(edge) =>
@ -271,10 +360,11 @@ export function applyRecipeConnection(
)
: edges;
const resolvedConnection = chooseModelSemanticHandles(
connection,
normalizedConnection,
source,
target,
nextBaseEdges,
layoutDirection,
);
const nextEdges = addEdge(
{ ...resolvedConnection, type: semanticRelation ? "semantic" : "canvas" },
@ -302,23 +392,34 @@ export function applyRecipeConnection(
return { edges: nextEdges, configs: { ...configs, [target.id]: next } };
}
if (
isLlmConfig(target) &&
source.kind !== "seed" &&
source.kind !== "model_provider" &&
source.kind !== "model_config"
source.kind === "llm" &&
source.llm_type === "code" &&
target.kind === "validator"
) {
const ref = `{{ ${source.name} }}`;
const nextCodeLang = (source.code_lang ?? "").trim();
const canUseCodeLangForTarget = canApplyCodeLangToValidator(
target,
nextCodeLang,
);
const next = {
...target,
prompt: buildTemplateWithRef(target.prompt ?? "", ref),
// biome-ignore lint/style/useNamingConvention: api schema
target_columns: [source.name],
// biome-ignore lint/style/useNamingConvention: api schema
code_lang:
(
canUseCodeLangForTarget ? nextCodeLang : target.code_lang
) as typeof target.code_lang,
};
return { edges: nextEdges, configs: { ...configs, [target.id]: next } };
}
if (
isExpressionConfig(target) &&
!semanticRelation &&
source.kind !== "seed" &&
source.kind !== "model_provider" &&
source.kind !== "model_config"
source.kind !== "model_config" &&
source.kind !== "validator"
) {
const ref = `{{ ${source.name} }}`;
const next = {

View file

@ -7,6 +7,19 @@ export function isSemanticRelation(
if (source.kind === "model_provider" && target.kind === "model_config") {
return true;
}
return source.kind === "model_config" && target.kind === "llm";
if (source.kind === "model_config" && target.kind === "llm") {
return true;
}
if (
source.kind === "llm" &&
source.llm_type === "code" &&
target.kind === "validator"
) {
return true;
}
return (
source.kind === "validator" &&
target.kind === "llm" &&
target.llm_type === "code"
);
}

View file

@ -0,0 +1,258 @@
import type { Edge } from "@xyflow/react";
import type {
RecipeExecutionBatch,
RecipeExecutionRecord,
RecipeExecutionStatus,
} from "../../execution-types";
import type { NodeConfig } from "../../types";
import { extractRefs } from "../refs";
const ACTIVE_STATUSES: ReadonlySet<RecipeExecutionStatus> = new Set([
"pending",
"running",
"active",
"cancelling",
]);
const FRESH_PENDING_WINDOW_MS = 60_000;
const DONE_UPSTREAM_KINDS: ReadonlySet<NodeConfig["kind"]> = new Set([
"sampler",
"seed",
"expression",
"llm",
"model_config",
"model_provider",
]);
export type GraphRuntimeVisualState = {
executionLocked: boolean;
runningNodeId: string | null;
doneNodeIds: Set<string>;
activeEdgeIds: Set<string>;
batch: RecipeExecutionBatch | null;
};
function isAuxEdge(edge: Edge): boolean {
return edge.source.startsWith("aux-") || edge.target.startsWith("aux-");
}
function collectTemplateRefs(config: NodeConfig | null): Set<string> {
if (!config) {
return new Set();
}
const refs = new Set<string>();
if (config.kind === "llm") {
for (const ref of extractRefs(config.prompt ?? "")) {
refs.add(ref.trim());
}
for (const ref of extractRefs(config.system_prompt ?? "")) {
refs.add(ref.trim());
}
if (typeof config.output_format === "string") {
for (const ref of extractRefs(config.output_format)) {
refs.add(ref.trim());
}
}
return refs;
}
if (config.kind === "expression") {
for (const ref of extractRefs(config.expr ?? "")) {
refs.add(ref.trim());
}
}
return refs;
}
function isReversedRuntimeReferenceEdge(input: {
edge: Edge;
runningNodeId: string;
runningTemplateRefs: Set<string>;
configs: Record<string, NodeConfig>;
}): boolean {
const { edge, runningNodeId, runningTemplateRefs, configs } = input;
if (edge.source !== runningNodeId) {
return false;
}
const targetName = configs[edge.target]?.name?.trim() ?? "";
return Boolean(targetName && runningTemplateRefs.has(targetName));
}
function hasLiveExecutionSignal(execution: RecipeExecutionRecord): boolean {
if (execution.lastEventId !== null) {
return true;
}
if (execution.current_column !== null) {
return true;
}
if (execution.progress !== null || execution.column_progress !== null) {
return true;
}
return Boolean(execution.batch?.idx ?? execution.batch?.total);
}
export function pickLatestActiveExecution(
executions: RecipeExecutionRecord[],
): RecipeExecutionRecord | null {
const now = Date.now();
for (const execution of executions) {
if (!ACTIVE_STATUSES.has(execution.status)) {
continue;
}
if (!execution.jobId) {
continue;
}
if (execution.finishedAt !== null) {
continue;
}
const liveSignal = hasLiveExecutionSignal(execution);
if (!liveSignal && execution.status === "pending") {
const ageMs = Math.max(0, now - execution.createdAt);
if (ageMs > FRESH_PENDING_WINDOW_MS) {
continue;
}
}
if (!liveSignal && execution.status !== "pending") {
continue;
}
return execution;
}
return null;
}
export function deriveGraphRuntimeVisualState(input: {
activeExecution: RecipeExecutionRecord | null;
configs: Record<string, NodeConfig>;
edges: Edge[];
}): GraphRuntimeVisualState {
const { activeExecution, configs, edges } = input;
if (!activeExecution) {
return {
executionLocked: false,
runningNodeId: null,
doneNodeIds: new Set(),
activeEdgeIds: new Set(),
batch: null,
};
}
const nameToNodeId = new Map<string, string>();
for (const config of Object.values(configs)) {
const name = config.name.trim();
if (!name) {
continue;
}
nameToNodeId.set(name, config.id);
}
const doneNodeIds = new Set<string>();
for (const columnName of activeExecution.completed_columns) {
const nodeId = nameToNodeId.get(columnName.trim());
if (nodeId) {
doneNodeIds.add(nodeId);
}
}
const runningNodeId = activeExecution.current_column
? nameToNodeId.get(activeExecution.current_column.trim()) ?? null
: null;
if (runningNodeId) {
doneNodeIds.delete(runningNodeId);
}
const activeEdgeIds = new Set<string>();
if (runningNodeId) {
const runningConfig = configs[runningNodeId] ?? null;
const runningTemplateRefs = collectTemplateRefs(runningConfig);
for (const ref of runningTemplateRefs) {
const refNodeId = nameToNodeId.get(ref);
if (refNodeId && refNodeId !== runningNodeId) {
doneNodeIds.add(refNodeId);
}
}
for (const upstreamNodeId of collectUpstreamDoneNodeIds({
rootNodeId: runningNodeId,
edges,
configs,
})) {
doneNodeIds.add(upstreamNodeId);
}
for (const edge of edges) {
if (isAuxEdge(edge)) {
continue;
}
if (edge.target === runningNodeId) {
activeEdgeIds.add(edge.id);
continue;
}
if (
isReversedRuntimeReferenceEdge({
edge,
runningNodeId,
runningTemplateRefs,
configs,
})
) {
activeEdgeIds.add(edge.id);
}
}
}
const batch =
activeExecution.batch &&
typeof activeExecution.batch.total === "number" &&
activeExecution.batch.total > 1
? activeExecution.batch
: null;
return {
executionLocked: true,
runningNodeId,
doneNodeIds,
activeEdgeIds,
batch,
};
}
function collectUpstreamDoneNodeIds(input: {
rootNodeId: string;
edges: Edge[];
configs: Record<string, NodeConfig>;
}): Set<string> {
const { rootNodeId, edges, configs } = input;
const incoming = new Map<string, string[]>();
for (const edge of edges) {
if (isAuxEdge(edge)) {
continue;
}
const list = incoming.get(edge.target) ?? [];
list.push(edge.source);
incoming.set(edge.target, list);
}
const visited = new Set<string>();
const queue = [rootNodeId];
let queueIndex = 0;
const doneNodeIds = new Set<string>();
while (queueIndex < queue.length) {
const current = queue[queueIndex];
queueIndex += 1;
if (!current || visited.has(current)) {
continue;
}
visited.add(current);
const sources = incoming.get(current) ?? [];
for (const sourceId of sources) {
if (!visited.has(sourceId)) {
queue.push(sourceId);
}
const config = configs[sourceId];
if (config && DONE_UPSTREAM_KINDS.has(config.kind)) {
doneNodeIds.add(sourceId);
}
}
}
return doneNodeIds;
}

View file

@ -0,0 +1,205 @@
export const MAX_IMAGE_PREVIEW_BYTES = 200 * 1024;
type PreviewImagePayload = {
type?: unknown;
mime?: unknown;
data?: unknown;
};
type UnknownRecord = Record<string, unknown>;
export type ImagePreviewResult =
| { kind: "ready"; src: string }
| { kind: "too_large"; estimatedBytes: number };
function normalizeBase64(value: string): string {
return value.replace(/\s+/g, "");
}
function estimateBase64Bytes(base64: string): number {
const normalized = normalizeBase64(base64);
const padding = normalized.endsWith("==")
? 2
: normalized.endsWith("=")
? 1
: 0;
return Math.max(0, Math.floor((normalized.length * 3) / 4) - padding);
}
function inferMimeFromBase64(base64: string): string | null {
const normalized = normalizeBase64(base64);
if (normalized.startsWith("iVBORw0KGgo")) {
return "image/png";
}
if (normalized.startsWith("/9j/")) {
return "image/jpeg";
}
if (normalized.startsWith("R0lGOD")) {
return "image/gif";
}
if (normalized.startsWith("UklGR")) {
return "image/webp";
}
return null;
}
function isLikelyRawBase64Image(value: string): boolean {
const normalized = normalizeBase64(value);
if (normalized.length < 64) {
return false;
}
if (!/^[A-Za-z0-9+/=]+$/.test(normalized)) {
return false;
}
return inferMimeFromBase64(normalized) !== null;
}
function toDataUrlFromBase64(base64: string, mime: string): string {
return `data:${mime};base64,${normalizeBase64(base64)}`;
}
function isRecord(value: unknown): value is UnknownRecord {
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
}
function isByteArray(value: unknown): value is number[] {
if (!Array.isArray(value) || value.length === 0) {
return false;
}
return value.every(
(item) => typeof item === "number" && Number.isInteger(item) && item >= 0 && item <= 255,
);
}
function byteArrayToBase64(bytes: number[]): string {
let binary = "";
const chunkSize = 0x8000;
for (let idx = 0; idx < bytes.length; idx += chunkSize) {
const chunk = bytes.slice(idx, idx + chunkSize);
binary += String.fromCharCode(...chunk);
}
return btoa(binary);
}
function resolveStringCandidate(
value: unknown,
maxBytes: number,
): ImagePreviewResult | null {
if (typeof value !== "string") {
return null;
}
return resolveImagePreviewFromString(value, maxBytes);
}
function resolveImagePreviewFromString(
value: string,
maxBytes: number,
): ImagePreviewResult | null {
const trimmed = value.trim();
if (!trimmed) {
return null;
}
if (trimmed.startsWith("http://") || trimmed.startsWith("https://")) {
return { kind: "ready", src: trimmed };
}
if (trimmed.startsWith("data:image/")) {
const marker = "base64,";
const markerIdx = trimmed.indexOf(marker);
if (markerIdx < 0) {
return { kind: "ready", src: trimmed };
}
const encoded = trimmed.slice(markerIdx + marker.length);
const estimatedBytes = estimateBase64Bytes(encoded);
if (estimatedBytes > maxBytes) {
return { kind: "too_large", estimatedBytes };
}
return { kind: "ready", src: trimmed };
}
if (isLikelyRawBase64Image(trimmed)) {
const estimatedBytes = estimateBase64Bytes(trimmed);
if (estimatedBytes > maxBytes) {
return { kind: "too_large", estimatedBytes };
}
const mime = inferMimeFromBase64(trimmed) ?? "image/png";
return { kind: "ready", src: toDataUrlFromBase64(trimmed, mime) };
}
return null;
}
function resolveImagePayloadObject(value: unknown, maxBytes: number): ImagePreviewResult | null {
if (!isRecord(value)) {
return null;
}
const payload = value as PreviewImagePayload;
if (payload.type === "image" && typeof payload.data === "string") {
const mime = typeof payload.mime === "string" ? payload.mime : "image/jpeg";
const estimatedBytes = estimateBase64Bytes(payload.data);
if (estimatedBytes > maxBytes) {
return { kind: "too_large", estimatedBytes };
}
return {
kind: "ready",
src: toDataUrlFromBase64(payload.data, mime),
};
}
const imageUrl = value.image_url;
const directImageUrl = resolveStringCandidate(imageUrl, maxBytes);
if (directImageUrl !== null) {
return directImageUrl;
}
if (isRecord(imageUrl)) {
const nestedImageUrl = resolveStringCandidate(imageUrl.url, maxBytes);
if (nestedImageUrl !== null) {
return nestedImageUrl;
}
}
const scalarCandidates = [
value.url,
value.data,
value.bytes,
value.base64,
value.base64_image,
value.image,
value.path,
];
for (const candidate of scalarCandidates) {
const resolved = resolveStringCandidate(candidate, maxBytes);
if (resolved !== null) {
return resolved;
}
}
if (isByteArray(value.bytes)) {
const resolved = resolveStringCandidate(byteArrayToBase64(value.bytes), maxBytes);
if (resolved !== null) {
return resolved;
}
}
if (isRecord(value.image)) {
return resolveImagePayloadObject(value.image, maxBytes);
}
return null;
}
export function resolveImagePreview(
value: unknown,
maxBytes = MAX_IMAGE_PREVIEW_BYTES,
): ImagePreviewResult | null {
const payloadPreview = resolveImagePayloadObject(value, maxBytes);
if (payloadPreview) {
return payloadPreview;
}
if (typeof value !== "string") {
return null;
}
return resolveImagePreviewFromString(value, maxBytes);
}
export function isLikelyImageValue(value: unknown): boolean {
return resolveImagePreview(value, Number.POSITIVE_INFINITY) !== null;
}

View file

@ -17,7 +17,21 @@ function isSemanticConnection(source: NodeConfig, target: NodeConfig): boolean {
if (source.kind === "model_provider" && target.kind === "model_config") {
return true;
}
return source.kind === "model_config" && target.kind === "llm";
if (source.kind === "model_config" && target.kind === "llm") {
return true;
}
if (
source.kind === "llm" &&
source.llm_type === "code" &&
target.kind === "validator"
) {
return true;
}
return (
source.kind === "validator" &&
target.kind === "llm" &&
target.llm_type === "code"
);
}
export function buildEdges(
@ -149,6 +163,13 @@ export function buildEdges(
if (config.kind === "llm" && config.model_alias) {
addEdgeByName(config.model_alias, config.name);
}
if (config.kind === "validator") {
for (const targetColumn of config.target_columns ?? []) {
if (targetColumn.trim()) {
addEdgeByName(targetColumn, config.name);
}
}
}
}
return edges;

View file

@ -5,7 +5,10 @@ import type {
MarkdownNoteConfig,
NodeConfig,
RecipeProcessorConfig,
SeedConfig,
SamplerConfig,
SeedSourceType,
ValidatorConfig,
} from "../../types";
import { buildEdges } from "./edges";
import { isRecord, parseJson, readString } from "./helpers";
@ -39,6 +42,7 @@ type UiInput = {
unstructured_file_name?: unknown;
unstructured_chunk_size?: unknown;
unstructured_chunk_overlap?: unknown;
advanced_open_by_node?: unknown;
};
type UiMarkdownNoteNode = {
@ -256,6 +260,42 @@ function parseUiMarkdownNoteNodes(input: unknown): UiMarkdownNoteNode[] {
return noteNodes;
}
function parseAdvancedOpenByNode(input: unknown): Record<string, boolean> {
if (!isRecord(input)) {
return {};
}
const out: Record<string, boolean> = {};
for (const [nameRaw, value] of Object.entries(input)) {
const name = nameRaw.trim();
if (!name || typeof value !== "boolean") {
continue;
}
out[name] = value;
}
return out;
}
type AdvancedOpenConfig = LlmConfig | SamplerConfig | SeedConfig | ValidatorConfig;
function isAdvancedOpenConfig(config: NodeConfig): config is AdvancedOpenConfig {
return (
config.kind === "llm" ||
config.kind === "sampler" ||
config.kind === "seed" ||
config.kind === "validator"
);
}
function applyAdvancedOpen(
config: NodeConfig,
advancedOpenByNode: Record<string, boolean>,
): void {
if (!isAdvancedOpenConfig(config)) {
return;
}
config.advancedOpen = advancedOpenByNode[config.name] === true;
}
function attachLlmTooling(
config: LlmConfig,
toolConfigsByAlias: Map<string, LlmToolConfig>,
@ -336,6 +376,7 @@ export function importRecipePayload(input: string): ImportResult {
const uiUnstructuredChunkOverlap = readStringNumber(
ui?.unstructured_chunk_overlap,
);
const uiAdvancedOpenByNode = parseAdvancedOpenByNode(ui?.advanced_open_by_node);
const uiMarkdownNotes = parseUiMarkdownNoteNodes(ui?.nodes);
for (const note of uiMarkdownNotes) {
@ -364,7 +405,9 @@ export function importRecipePayload(input: string): ImportResult {
preferredSourceType: uiSeedSourceType,
seed_columns: uiSeedColumns,
seed_drop_columns:
uiSeedDropColumns ?? payloadSeedDropColumns,
uiSeedDropColumns && uiSeedDropColumns.length > 0
? uiSeedDropColumns
: payloadSeedDropColumns,
seed_preview_rows: uiSeedPreviewRows,
local_file_name: uiLocalFileName,
unstructured_file_name: uiUnstructuredFileName,
@ -372,6 +415,7 @@ export function importRecipePayload(input: string): ImportResult {
unstructured_chunk_overlap: uiUnstructuredChunkOverlap,
});
if (seedConfig) {
applyAdvancedOpen(seedConfig, uiAdvancedOpenByNode);
if (nameToId.has(seedConfig.name)) {
errors.push(`Duplicate column name: ${seedConfig.name}.`);
} else {
@ -441,6 +485,7 @@ export function importRecipePayload(input: string): ImportResult {
if (config.kind === "llm") {
attachLlmTooling(config, toolConfigsByAlias, mcpProvidersByName);
}
applyAdvancedOpen(config, uiAdvancedOpenByNode);
if (nameToId.has(config.name)) {
errors.push(`Duplicate column name: ${config.name}.`);
return;

Some files were not shown because too many files have changed in this diff Show more