From 552eb06bed4ed19a7321ab2b6ced950e7f8d0217 Mon Sep 17 00:00:00 2001 From: Shine1i Date: Thu, 5 Mar 2026 19:48:26 +0100 Subject: [PATCH] feat(data-recipes, validators): add OXC validator runtime and integration with recipe studio --- .gitignore | 1 + setup.sh | 2 + .../data_recipe/local_callable_validators.py | 301 ++++++++++++ .../oxc-validator/package-lock.json | 445 ++++++++++++++++++ .../data_recipe/oxc-validator/package.json | 9 + .../data_recipe/oxc-validator/validate.mjs | 149 ++++++ studio/backend/core/data_recipe/service.py | 11 + .../recipe-studio/blocks/definitions.ts | 19 +- .../recipe-studio/components/block-sheet.tsx | 8 +- .../dialogs/validators/validator-dialog.tsx | 26 +- .../hooks/use-recipe-editor-graph.ts | 8 +- .../recipe-studio/stores/recipe-studio.ts | 2 +- .../src/features/recipe-studio/types/index.ts | 8 + .../recipe-studio/utils/config-factories.ts | 14 +- .../utils/graph/recipe-graph-connection.ts | 31 +- .../utils/import/parsers/validator-parser.ts | 43 +- .../features/recipe-studio/utils/node-data.ts | 17 +- .../utils/payload/builders-validator.ts | 23 + .../recipe-studio/utils/payload/validate.ts | 13 + .../recipe-studio/utils/validation.ts | 10 + .../utils/validators/code-lang.ts | 11 + 21 files changed, 1132 insertions(+), 19 deletions(-) create mode 100644 studio/backend/core/data_recipe/local_callable_validators.py create mode 100644 studio/backend/core/data_recipe/oxc-validator/package-lock.json create mode 100644 studio/backend/core/data_recipe/oxc-validator/package.json create mode 100644 studio/backend/core/data_recipe/oxc-validator/validate.mjs diff --git a/.gitignore b/.gitignore index 0d0e7d1133..f7f914fb96 100755 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,7 @@ firebase-debug.log # Other resources/ tmp/ +**/node_modules/ auth.db studio/frontend/package-lock.json diff --git a/setup.sh b/setup.sh index 07ea131125..899677444b 100755 --- a/setup.sh +++ b/setup.sh @@ -99,6 +99,8 @@ echo "Building frontend..." cd "$SCRIPT_DIR/studio/frontend" run_quiet "npm install" npm install run_quiet "npm run build" npm run build +cd "$SCRIPT_DIR/studio/backend/core/data_recipe/oxc-validator" +run_quiet "npm install (oxc validator runtime)" npm install cd "$SCRIPT_DIR" echo "✅ Frontend built to studio/frontend/dist" diff --git a/studio/backend/core/data_recipe/local_callable_validators.py b/studio/backend/core/data_recipe/local_callable_validators.py new file mode 100644 index 0000000000..b9804e4b28 --- /dev/null +++ b/studio/backend/core/data_recipe/local_callable_validators.py @@ -0,0 +1,301 @@ +from __future__ import annotations + +import json +import logging +import subprocess +from copy import deepcopy +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +OXC_VALIDATION_FN_MARKER = "unsloth_oxc_validator" + +_OXC_LANG_TO_NODE_LANG = { + "javascript": "js", + "typescript": "ts", + "jsx": "jsx", + "tsx": "tsx", +} + +_OXC_TOOL_DIR = Path(__file__).resolve().parent / "oxc-validator" +_OXC_RUNNER_PATH = _OXC_TOOL_DIR / "validate.mjs" + + +@dataclass(frozen=True) +class OxcLocalCallableValidatorSpec: + name: str + drop: bool + target_columns: list[str] + batch_size: int + code_lang: str + + +def split_oxc_local_callable_validators( + recipe_core: dict[str, Any], +) -> tuple[dict[str, Any], list[OxcLocalCallableValidatorSpec]]: + columns = recipe_core.get("columns") + if not isinstance(columns, list): + return recipe_core, [] + + sanitized = deepcopy(recipe_core) + sanitized_columns = sanitized.get("columns") + if not isinstance(sanitized_columns, list): + return sanitized, [] + + llm_code_lang_by_name = _extract_llm_code_lang_by_name(sanitized_columns) + kept_columns: list[Any] = [] + oxc_specs: list[OxcLocalCallableValidatorSpec] = [] + + for column in sanitized_columns: + if not isinstance(column, dict): + kept_columns.append(column) + continue + + maybe_spec = _parse_oxc_spec( + column=column, + llm_code_lang_by_name=llm_code_lang_by_name, + ) + if maybe_spec is None: + kept_columns.append(column) + continue + oxc_specs.append(maybe_spec) + + sanitized["columns"] = kept_columns + return sanitized, oxc_specs + + +def register_oxc_local_callable_validators( + *, + builder, + specs: list[OxcLocalCallableValidatorSpec], +) -> None: + if not specs: + return + + from data_designer.config.column_configs import ValidationColumnConfig + from data_designer.config.validator_params import ( + LocalCallableValidatorParams, + ValidatorType, + ) + + for spec in specs: + validation_function = _build_oxc_validation_function(spec.code_lang) + builder.add_column( + ValidationColumnConfig( + name=spec.name, + drop=spec.drop, + target_columns=spec.target_columns, + validator_type=ValidatorType.LOCAL_CALLABLE, + validator_params=LocalCallableValidatorParams( + validation_function=validation_function, + ), + batch_size=spec.batch_size, + ) + ) + + +def _parse_oxc_spec( + *, + column: dict[str, Any], + llm_code_lang_by_name: dict[str, str], +) -> OxcLocalCallableValidatorSpec | None: + if str(column.get("column_type") or "").strip() != "validation": + return None + if str(column.get("validator_type") or "").strip() != "local_callable": + return None + + params = column.get("validator_params") + if not isinstance(params, dict): + return None + + fn_raw = params.get("validation_function") + fn_name = fn_raw.strip() if isinstance(fn_raw, str) else "" + if not fn_name.startswith(OXC_VALIDATION_FN_MARKER): + return None + + name = str(column.get("name") or "").strip() + if not name: + return None + + target_columns_raw = column.get("target_columns") + target_columns = ( + [value.strip() for value in target_columns_raw if isinstance(value, str) and value.strip()] + if isinstance(target_columns_raw, list) + else [] + ) + if not target_columns: + return None + + code_lang = _resolve_oxc_lang( + fn_name=fn_name, + target_columns=target_columns, + llm_code_lang_by_name=llm_code_lang_by_name, + ) + batch_size = _parse_batch_size(column.get("batch_size")) + drop = bool(column.get("drop") is True) + + return OxcLocalCallableValidatorSpec( + name=name, + drop=drop, + target_columns=target_columns, + batch_size=batch_size, + code_lang=code_lang, + ) + + +def _parse_batch_size(value: Any) -> int: + try: + parsed = int(value) + except (TypeError, ValueError): + return 10 + return parsed if parsed >= 1 else 10 + + +def _extract_llm_code_lang_by_name(columns: list[Any]) -> dict[str, str]: + out: dict[str, str] = {} + for column in columns: + if not isinstance(column, dict): + continue + if str(column.get("column_type") or "").strip() != "llm-code": + continue + name = str(column.get("name") or "").strip() + code_lang = str(column.get("code_lang") or "").strip() + if name and code_lang: + out[name] = code_lang + return out + + +def _resolve_oxc_lang( + *, + fn_name: str, + target_columns: list[str], + llm_code_lang_by_name: dict[str, str], +) -> str: + _, _, marker_lang = fn_name.partition(":") + marker_lang = marker_lang.strip() + if marker_lang in _OXC_LANG_TO_NODE_LANG: + return marker_lang + + first_target = target_columns[0] + target_lang = llm_code_lang_by_name.get(first_target, "").strip() + if target_lang in _OXC_LANG_TO_NODE_LANG: + return target_lang + return "javascript" + + +@lru_cache(maxsize=8) +def _build_oxc_validation_function(lang: str): + node_lang = _OXC_LANG_TO_NODE_LANG.get(lang, "js") + + def _validator(df): + import pandas as pd # imported lazily for local callable runtime + + row_count = int(len(df.index)) + if row_count == 0: + return pd.DataFrame({"is_valid": []}) + + code_column = str(df.columns[0]) if len(df.columns) > 0 else "" + code_values = ( + ["" for _ in range(row_count)] + if not code_column + else ["" if value is None else str(value) for value in df[code_column].tolist()] + ) + + results = _run_oxc_batch(node_lang=node_lang, code_values=code_values) + if len(results) != row_count: + results = _fallback_results( + row_count, + "OXC validator returned mismatched result size.", + ) + return pd.DataFrame(results) + + _validator.__name__ = f"{OXC_VALIDATION_FN_MARKER}_{node_lang}" + return _validator + + +def _run_oxc_batch(*, node_lang: str, code_values: list[str]) -> list[dict[str, Any]]: + if not _OXC_RUNNER_PATH.exists(): + return _fallback_results( + len(code_values), + f"OXC runner missing at {_OXC_RUNNER_PATH}", + ) + + payload = { + "lang": node_lang, + "codes": code_values, + } + try: + proc = subprocess.run( + ["node", str(_OXC_RUNNER_PATH)], + cwd=str(_OXC_TOOL_DIR), + input=json.dumps(payload), + text=True, + capture_output=True, + check=False, + ) + except (OSError, ValueError) as exc: + logger.warning("OXC subprocess launch failed: %s", exc) + return _fallback_results(len(code_values), f"OXC launch failed: {exc}") + + if proc.returncode != 0: + message = (proc.stderr or proc.stdout or "unknown error").strip() + if len(message) > 300: + message = f"{message[:300]}..." + return _fallback_results(len(code_values), f"OXC failed: {message}") + + try: + raw = json.loads(proc.stdout) + except json.JSONDecodeError: + return _fallback_results(len(code_values), "OXC output parse failed.") + + if not isinstance(raw, list): + return _fallback_results(len(code_values), "OXC output must be an array.") + + out: list[dict[str, Any]] = [] + for item in raw: + if not isinstance(item, dict): + out.append( + { + "is_valid": False, + "error_count": 1, + "error_message": "Invalid OXC result entry.", + "severity": None, + "labels": [], + "codeframe": None, + } + ) + continue + is_valid_raw = item.get("is_valid") + error_count_raw = item.get("error_count") + message_raw = item.get("error_message") + severity_raw = item.get("severity") + labels_raw = item.get("labels") + codeframe_raw = item.get("codeframe") + out.append( + { + "is_valid": bool(is_valid_raw) if isinstance(is_valid_raw, bool) else False, + "error_count": int(error_count_raw) if isinstance(error_count_raw, int) else 0, + "error_message": str(message_raw or ""), + "severity": str(severity_raw) if isinstance(severity_raw, str) else None, + "labels": labels_raw if isinstance(labels_raw, list) else [], + "codeframe": str(codeframe_raw) if isinstance(codeframe_raw, str) else None, + } + ) + return out + + +def _fallback_results(row_count: int, message: str) -> list[dict[str, Any]]: + return [ + { + "is_valid": False, + "error_count": 1, + "error_message": message, + "severity": None, + "labels": [], + "codeframe": None, + } + for _ in range(row_count) + ] diff --git a/studio/backend/core/data_recipe/oxc-validator/package-lock.json b/studio/backend/core/data_recipe/oxc-validator/package-lock.json new file mode 100644 index 0000000000..2a97d4514e --- /dev/null +++ b/studio/backend/core/data_recipe/oxc-validator/package-lock.json @@ -0,0 +1,445 @@ +{ + "name": "unsloth-oxc-validator-runtime", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "unsloth-oxc-validator-runtime", + "version": "0.0.1", + "dependencies": { + "oxc-parser": "^0.116.0" + } + }, + "node_modules/@emnapi/core": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.8.1.tgz", + "integrity": "sha512-AvT9QFpxK0Zd8J0jopedNm+w/2fIzvtPKPjqyw9jwvBaReTTqPBk9Hixaz7KbjimP+QNz605/XnjFcDAL2pqBg==", + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.1.0", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.1.tgz", + "integrity": "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz", + "integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.1.tgz", + "integrity": "sha512-p64ah1M1ld8xjWv3qbvFwHiFVWrq1yFvV4f7w+mzaqiR4IlSgkqhcRdHwsGgomwzBH51sRY4NEowLxnaBjcW/A==", + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1", + "@tybys/wasm-util": "^0.10.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@oxc-parser/binding-android-arm-eabi": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm-eabi/-/binding-android-arm-eabi-0.116.0.tgz", + "integrity": "sha512-AOET7YIOU3+ANO/3xQeRVGN5Xx6+JGXaIwlqkcHSfxJ/zzw2B6jb0YaLhX45SeRluKVTU8rka4N/tHtNoJjoCg==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-android-arm64": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm64/-/binding-android-arm64-0.116.0.tgz", + "integrity": "sha512-yh0Zvth5cQ6XZkP3QF9MDrXf695zr5XxXq/wBQqpZb0uAgI9wpr98/Hx2RZITMfnNjkIq2VcyU44o3A0bdEmlQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-darwin-arm64": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-arm64/-/binding-darwin-arm64-0.116.0.tgz", + "integrity": "sha512-plcTd/Jska55dToZz6XdRBPRVsj+asjD8QCpQFvt3Wj8pY+10D1pE53Mei3POAS/wSRSy7HiQ2twrm7H2A0CjA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-darwin-x64": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-x64/-/binding-darwin-x64-0.116.0.tgz", + "integrity": "sha512-ahqcF3e3x5Z2ZepzXpZ8ugREdmxvBL+g1nQ0SxO11pIZfck6UtbOtwtdAAxnQXBHHtidu7lPcrBq1SEx26t1PQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-freebsd-x64": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-freebsd-x64/-/binding-freebsd-x64-0.116.0.tgz", + "integrity": "sha512-yo2/LaSXtlzKBurvNbwun/sN/RJwW3XhbMr069FwNVtft7GBnaLLdPIz/sf47icxw/BPViEX6wFvzeD12mtrAg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-arm-gnueabihf": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-0.116.0.tgz", + "integrity": "sha512-EiZeliIPPdFsuaPx8PzDMVijD/4YaUxO46/eYPk5raRocJqjjxOG6GAacQ8UrG3fbrgYjaEChfYL1e8DyE445A==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-arm-musleabihf": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-0.116.0.tgz", + "integrity": "sha512-Nf7hnKRYRSIgglQcLAqE2St4b/Yr6dh+Z7in8mxol065Knevw71XZAiV1fmPSojq6uKPLV9eoH/wFrgr4TnZXw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-arm64-gnu": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-0.116.0.tgz", + "integrity": "sha512-9SJI0S4Qggn3QHpT8Y1jtZceA0m4BlpvO3ne2Wxd33UdTHMmelAnrXryjWutHWQtjCzOwSnFBEoQAdNNyt1u3A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-arm64-musl": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-musl/-/binding-linux-arm64-musl-0.116.0.tgz", + "integrity": "sha512-wMZ6//GI+q1JwO7G2OR51+eA5P8Gr3BobU8RAzCGJptvyGMkWb7KQ1E8s8naVZRr6bSGWAL2p3mCzKOxmEPmrA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-ppc64-gnu": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-0.116.0.tgz", + "integrity": "sha512-5BO0KCzTG2HZTnp3r6SCAOeCs/GwFBQJ1WAOG/ROfDf1fVVEy6hrtLKTLCuUMaamH38v+1+RVEmzRkzBj+rMDQ==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-riscv64-gnu": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-0.116.0.tgz", + "integrity": "sha512-M24gYb/ocVMnLwnH2wY5sLt4sRBkAUHDmfiYtyUYdKTkfPOKtpopd5otsL/BPLnIhpMD8zby4uXVvw7BU0UIlw==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-riscv64-musl": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-0.116.0.tgz", + "integrity": "sha512-LHLXTHCH0bdvGjlitwr1ngeh32GAgq9HYzQ5VAgt0k0UT84AS8AkXj9Spoa9l20fXkVgSvAKcCEkydi4Ol23Dw==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-s390x-gnu": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-0.116.0.tgz", + "integrity": "sha512-VE+XsztuE5jdHvLIDIQMuyDpz5NJGq1Vx/8EXYF0sS/gehlv9GhDpGVWU0SCZ/LjzIy4io/Z0W84UudqufvP3g==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-x64-gnu": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-gnu/-/binding-linux-x64-gnu-0.116.0.tgz", + "integrity": "sha512-rxUkauyjjCmgA7BoR63ogRGEtgubROnCm8AXE9ydg+p42jCGLLqG05mFcS2eC+FYyAU58ZFJNXXeqFW1iCyTGQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-x64-musl": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-musl/-/binding-linux-x64-musl-0.116.0.tgz", + "integrity": "sha512-0zoZlk9MmXe6oTgSh5lT1D51SDC1bfwC96JmE1amMFAPdEbJk5MFRisfTN9TFBpBigQua65842tjaxqMiorAYw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-openharmony-arm64": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-openharmony-arm64/-/binding-openharmony-arm64-0.116.0.tgz", + "integrity": "sha512-PGS7Xqik77U9WMyW626gAD5A2rSN629UvyYJKAl/tgpT+KqZI4+56pJfExhv8IW/PpSHjYHwjmakwobLikz8ww==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-wasm32-wasi": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-wasm32-wasi/-/binding-wasm32-wasi-0.116.0.tgz", + "integrity": "sha512-lGNf/9PU8XxB4Gt1Gr1AKwSrjxGYa6os0PlrT4bpoQsfE3gaZonQTKwJyKhiQdgy7pBCI+ed1LB1NNib1FYULw==", + "cpu": [ + "wasm32" + ], + "license": "MIT", + "optional": true, + "dependencies": { + "@napi-rs/wasm-runtime": "^1.1.1" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@oxc-parser/binding-win32-arm64-msvc": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-0.116.0.tgz", + "integrity": "sha512-tcsOHE31duBSRQXZ7NfdtjmMKZwQYlS00PwAMJ4w5oXs3iPCvisUuIXP7Ko4FzeOBTRvkd64btxtQ6cRM0Kwlw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-win32-ia32-msvc": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-0.116.0.tgz", + "integrity": "sha512-higCz/x+dOQ264YEk22hnu4RDqvjhfehjFORpxoh42QyUxsP6eIembYesBUu5ilALWo0HvRD+m89az2BSTwqpQ==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-win32-x64-msvc": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.116.0.tgz", + "integrity": "sha512-Lg2SRmVHpGG85knDVLbv44r1bYn0OpIV0vg9jVmoEIpDj3Q4kwXuQ6MWVtuslwHR8o2CSiqdBeEn1n1URrs6Eg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.116.0.tgz", + "integrity": "sha512-uOT8S1tlPmDckNxMNtIudN/yXpLdnhlJMX2oLS7cxCd7L0sUF09A/EbSVMWT3Y/iT44IwXCJSJfgfSxXAqWf9Q==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", + "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/oxc-parser": { + "version": "0.116.0", + "resolved": "https://registry.npmjs.org/oxc-parser/-/oxc-parser-0.116.0.tgz", + "integrity": "sha512-ugEo6wwqaqCGcpi7GsLCwSkoD7gIXzvtdaTxE+mbrXFYazU5Q9YdpZdAj9z2b79i/xlv+uW2aAvyzGAlpUzhKQ==", + "license": "MIT", + "dependencies": { + "@oxc-project/types": "^0.116.0" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/sponsors/Boshen" + }, + "optionalDependencies": { + "@oxc-parser/binding-android-arm-eabi": "0.116.0", + "@oxc-parser/binding-android-arm64": "0.116.0", + "@oxc-parser/binding-darwin-arm64": "0.116.0", + "@oxc-parser/binding-darwin-x64": "0.116.0", + "@oxc-parser/binding-freebsd-x64": "0.116.0", + "@oxc-parser/binding-linux-arm-gnueabihf": "0.116.0", + "@oxc-parser/binding-linux-arm-musleabihf": "0.116.0", + "@oxc-parser/binding-linux-arm64-gnu": "0.116.0", + "@oxc-parser/binding-linux-arm64-musl": "0.116.0", + "@oxc-parser/binding-linux-ppc64-gnu": "0.116.0", + "@oxc-parser/binding-linux-riscv64-gnu": "0.116.0", + "@oxc-parser/binding-linux-riscv64-musl": "0.116.0", + "@oxc-parser/binding-linux-s390x-gnu": "0.116.0", + "@oxc-parser/binding-linux-x64-gnu": "0.116.0", + "@oxc-parser/binding-linux-x64-musl": "0.116.0", + "@oxc-parser/binding-openharmony-arm64": "0.116.0", + "@oxc-parser/binding-wasm32-wasi": "0.116.0", + "@oxc-parser/binding-win32-arm64-msvc": "0.116.0", + "@oxc-parser/binding-win32-ia32-msvc": "0.116.0", + "@oxc-parser/binding-win32-x64-msvc": "0.116.0" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD", + "optional": true + } + } +} diff --git a/studio/backend/core/data_recipe/oxc-validator/package.json b/studio/backend/core/data_recipe/oxc-validator/package.json new file mode 100644 index 0000000000..2d1618162d --- /dev/null +++ b/studio/backend/core/data_recipe/oxc-validator/package.json @@ -0,0 +1,9 @@ +{ + "name": "unsloth-oxc-validator-runtime", + "private": true, + "version": "0.0.1", + "type": "module", + "dependencies": { + "oxc-parser": "^0.116.0" + } +} diff --git a/studio/backend/core/data_recipe/oxc-validator/validate.mjs b/studio/backend/core/data_recipe/oxc-validator/validate.mjs new file mode 100644 index 0000000000..2fe94e0833 --- /dev/null +++ b/studio/backend/core/data_recipe/oxc-validator/validate.mjs @@ -0,0 +1,149 @@ +import { parseSync } from "oxc-parser"; + +const LANG_TO_EXT = { + js: "js", + jsx: "jsx", + ts: "ts", + tsx: "tsx", +}; + +function mapLang(value) { + const normalized = String(value || "").trim().toLowerCase(); + if (normalized === "javascript" || normalized === "js") { + return "js"; + } + if (normalized === "typescript" || normalized === "ts") { + return "ts"; + } + if (normalized === "jsx") { + return "jsx"; + } + if (normalized === "tsx") { + return "tsx"; + } + return "js"; +} + +function normalizeError(error) { + if (typeof error === "string") { + return { + message: error.trim() || "Unknown OXC error", + severity: null, + labels: [], + codeframe: null, + }; + } + if (!error || typeof error !== "object") { + return { + message: "Unknown OXC error", + severity: null, + labels: [], + codeframe: null, + }; + } + const message = String(error.message || error.reason || "").trim() || "Unknown OXC error"; + const severity = typeof error.severity === "string" ? error.severity : null; + const labels = Array.isArray(error.labels) + ? error.labels.map((label) => ({ + message: + label && typeof label === "object" && typeof label.message === "string" + ? label.message + : null, + start: + label && typeof label === "object" && Number.isInteger(label.start) + ? label.start + : null, + end: + label && typeof label === "object" && Number.isInteger(label.end) + ? label.end + : null, + })) + : []; + const codeframe = typeof error.codeframe === "string" ? error.codeframe : null; + return { + message, + severity, + labels, + codeframe, + }; +} + +function validateOne({ code, lang, index }) { + const ext = LANG_TO_EXT[lang] ?? "js"; + const filename = `snippet_${index}.${ext}`; + const source = typeof code === "string" ? code : String(code ?? ""); + + try { + const parsed = parseSync(filename, source, { + lang, + sourceType: "module", + showSemanticErrors: true, + }); + const errors = Array.isArray(parsed?.errors) + ? parsed.errors.map(normalizeError).filter(Boolean) + : []; + const first = errors[0] ?? null; + return { + is_valid: errors.length === 0, + error_count: errors.length, + error_message: errors.slice(0, 3).map((error) => error.message).join(" | "), + severity: first ? first.severity : null, + labels: first ? first.labels : [], + codeframe: first ? first.codeframe : null, + }; + } catch (error) { + const normalized = normalizeError(error); + return { + is_valid: false, + error_count: 1, + error_message: normalized.message, + severity: normalized.severity, + labels: normalized.labels, + codeframe: normalized.codeframe, + }; + } +} + +function readStdin() { + return new Promise((resolve, reject) => { + let data = ""; + process.stdin.setEncoding("utf8"); + process.stdin.on("data", (chunk) => { + data += chunk; + }); + process.stdin.on("end", () => resolve(data)); + process.stdin.on("error", (error) => reject(error)); + }); +} + +async function main() { + const raw = await readStdin(); + let payload; + try { + payload = JSON.parse(raw || "{}"); + } catch { + process.stdout.write( + JSON.stringify([ + { + is_valid: false, + error_count: 1, + error_message: "Invalid JSON payload", + severity: null, + labels: [], + codeframe: null, + }, + ]), + ); + return; + } + + const lang = mapLang(payload?.lang); + const codes = Array.isArray(payload?.codes) ? payload.codes : []; + const out = codes.map((code, index) => validateOne({ code, lang, index })); + process.stdout.write(JSON.stringify(out)); +} + +main().catch((error) => { + process.stderr.write(String(error?.stack || error)); + process.exit(1); +}); diff --git a/studio/backend/core/data_recipe/service.py b/studio/backend/core/data_recipe/service.py index 6c2af4438e..f2c25e62f8 100644 --- a/studio/backend/core/data_recipe/service.py +++ b/studio/backend/core/data_recipe/service.py @@ -7,6 +7,10 @@ from pathlib import Path from typing import Any from .jsonable import to_jsonable +from .local_callable_validators import ( + register_oxc_local_callable_validators, + split_oxc_local_callable_validators, +) _IMAGE_CONTEXT_PATCHED = False @@ -196,7 +200,14 @@ def build_config_builder(recipe: dict[str, Any]): for key, value in recipe.items() if key not in {"model_providers", "mcp_providers"} } + recipe_core, oxc_local_callable_specs = split_oxc_local_callable_validators( + recipe_core + ) builder = DataDesignerConfigBuilder.from_config({"data_designer": recipe_core}) + register_oxc_local_callable_validators( + builder=builder, + specs=oxc_local_callable_specs, + ) # DataDesignerConfigBuilder.from_config currently skips processors. # Re-attach explicitly so drop_columns/schema_transform survive API payload. diff --git a/studio/frontend/src/features/recipe-studio/blocks/definitions.ts b/studio/frontend/src/features/recipe-studio/blocks/definitions.ts index a37a24ad3b..2dcd3c4c43 100644 --- a/studio/frontend/src/features/recipe-studio/blocks/definitions.ts +++ b/studio/frontend/src/features/recipe-studio/blocks/definitions.ts @@ -46,6 +46,7 @@ export type BlockType = | LlmType | "validator_python" | "validator_sql" + | "validator_oxc" | "expression" | "markdown_note" | "seed" @@ -303,7 +304,8 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [ description: "Validate Python code columns.", icon: Shield02Icon, dialogKey: "validator", - createConfig: (id, existing) => makeValidatorConfig(id, "python", existing), + createConfig: (id, existing) => + makeValidatorConfig(id, "code", "python", existing), }, { kind: "validator", @@ -313,7 +315,17 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [ icon: Shield02Icon, dialogKey: "validator", createConfig: (id, existing) => - makeValidatorConfig(id, "sql:sqlite", existing), + makeValidatorConfig(id, "code", "sql:sqlite", existing), + }, + { + kind: "validator", + type: "validator_oxc", + title: "OXC Validator", + description: "Validate JavaScript or TypeScript code columns.", + icon: Shield02Icon, + dialogKey: "validator", + createConfig: (id, existing) => + makeValidatorConfig(id, "oxc", "javascript", existing), }, { kind: "expression", @@ -372,6 +384,9 @@ export function getBlockDefinitionForConfig( return getBlockDefinition("llm", config.llm_type); } if (config.kind === "validator") { + if (config.validator_type === "oxc") { + return getBlockDefinition("validator", "validator_oxc"); + } const isSql = config.code_lang.startsWith("sql:"); return getBlockDefinition( "validator", diff --git a/studio/frontend/src/features/recipe-studio/components/block-sheet.tsx b/studio/frontend/src/features/recipe-studio/components/block-sheet.tsx index adc6512aab..bf4785943e 100644 --- a/studio/frontend/src/features/recipe-studio/components/block-sheet.tsx +++ b/studio/frontend/src/features/recipe-studio/components/block-sheet.tsx @@ -71,7 +71,9 @@ type BlockSheetProps = { onAddModelProvider: () => void; onAddModelConfig: () => void; onAddExpression: () => void; - onAddValidator: (type: "validator_python" | "validator_sql") => void; + onAddValidator: ( + type: "validator_python" | "validator_sql" | "validator_oxc", + ) => void; onAddMarkdownNote: () => void; onOpenProcessors: () => void; copied: boolean; @@ -318,7 +320,9 @@ export function BlockSheet({ return; } if (kind === "validator") { - onAddValidator(type as "validator_python" | "validator_sql"); + onAddValidator( + type as "validator_python" | "validator_sql" | "validator_oxc", + ); return; } if (kind === "expression") { diff --git a/studio/frontend/src/features/recipe-studio/dialogs/validators/validator-dialog.tsx b/studio/frontend/src/features/recipe-studio/dialogs/validators/validator-dialog.tsx index fa6e3341fb..536d2445da 100644 --- a/studio/frontend/src/features/recipe-studio/dialogs/validators/validator-dialog.tsx +++ b/studio/frontend/src/features/recipe-studio/dialogs/validators/validator-dialog.tsx @@ -14,7 +14,11 @@ import { import { type ReactElement, useMemo } from "react"; import { useRecipeStudioStore } from "../../stores/recipe-studio"; import type { ValidatorConfig } from "../../types"; -import { isValidatorCodeLang } from "../../utils/validators/code-lang"; +import { + isValidatorCodeLang, + VALIDATOR_OXC_CODE_LANGS, + VALIDATOR_SQL_CODE_LANGS, +} from "../../utils/validators/code-lang"; import { FieldLabel } from "../shared/field-label"; import { NameField } from "../shared/name-field"; @@ -40,6 +44,22 @@ export function ValidatorDialog({ if (!(item.kind === "llm" && item.llm_type === "code")) { return []; } + if (config.validator_type === "oxc") { + const lang = item.code_lang?.trim() ?? ""; + if (!VALIDATOR_OXC_CODE_LANGS.includes(lang as typeof config.code_lang)) { + return []; + } + } else { + const lang = item.code_lang?.trim() ?? ""; + if ( + !( + lang === "python" || + VALIDATOR_SQL_CODE_LANGS.includes(lang as typeof config.code_lang) + ) + ) { + return []; + } + } return [ { name: item.name, @@ -102,7 +122,9 @@ export function ValidatorDialog({ {codeOptions.length === 0 && (

- Add an LLM Code block first. + {config.validator_type === "oxc" + ? "Add an LLM Code block with javascript/typescript first." + : "Add an LLM Code block first."}

)} diff --git a/studio/frontend/src/features/recipe-studio/hooks/use-recipe-editor-graph.ts b/studio/frontend/src/features/recipe-studio/hooks/use-recipe-editor-graph.ts index d0837c81f5..8e8582b855 100644 --- a/studio/frontend/src/features/recipe-studio/hooks/use-recipe-editor-graph.ts +++ b/studio/frontend/src/features/recipe-studio/hooks/use-recipe-editor-graph.ts @@ -69,7 +69,7 @@ type UseRecipeEditorGraphArgs = { addModelConfigNode: (position?: XYPosition, openDialog?: boolean) => void; addExpressionNode: (position?: XYPosition, openDialog?: boolean) => void; addValidatorNode: ( - type: "validator_python" | "validator_sql", + type: "validator_python" | "validator_sql" | "validator_oxc", position?: XYPosition, openDialog?: boolean, ) => void; @@ -92,7 +92,7 @@ type UseRecipeEditorGraphResult = { handleAddModelConfigFromSheet: () => void; handleAddExpressionFromSheet: () => void; handleAddValidatorFromSheet: ( - type: "validator_python" | "validator_sql", + type: "validator_python" | "validator_sql" | "validator_oxc", ) => void; handleAddMarkdownNoteFromSheet: () => void; }; @@ -213,7 +213,7 @@ export function useRecipeEditorGraph({ } if (payload.kind === "validator") { addValidatorNode( - payload.type as "validator_python" | "validator_sql", + payload.type as "validator_python" | "validator_sql" | "validator_oxc", position, false, ); @@ -291,7 +291,7 @@ export function useRecipeEditorGraph({ }, [addExpressionNode, getViewportCenterPosition]); const handleAddValidatorFromSheet = useCallback( - (type: "validator_python" | "validator_sql") => { + (type: "validator_python" | "validator_sql" | "validator_oxc") => { addValidatorNode(type, getViewportCenterPosition()); }, [addValidatorNode, getViewportCenterPosition], diff --git a/studio/frontend/src/features/recipe-studio/stores/recipe-studio.ts b/studio/frontend/src/features/recipe-studio/stores/recipe-studio.ts index 5183c725ab..38f4dd56d4 100644 --- a/studio/frontend/src/features/recipe-studio/stores/recipe-studio.ts +++ b/studio/frontend/src/features/recipe-studio/stores/recipe-studio.ts @@ -97,7 +97,7 @@ type RecipeStudioState = { addModelConfigNode: (position?: XYPosition, openDialog?: boolean) => void; addExpressionNode: (position?: XYPosition, openDialog?: boolean) => void; addValidatorNode: ( - type: "validator_python" | "validator_sql", + type: "validator_python" | "validator_sql" | "validator_oxc", position?: XYPosition, openDialog?: boolean, ) => void; diff --git a/studio/frontend/src/features/recipe-studio/types/index.ts b/studio/frontend/src/features/recipe-studio/types/index.ts index 1b8a829192..210716a540 100644 --- a/studio/frontend/src/features/recipe-studio/types/index.ts +++ b/studio/frontend/src/features/recipe-studio/types/index.ts @@ -14,6 +14,10 @@ export type SamplerType = export type LlmType = "text" | "structured" | "code" | "judge"; export type ValidatorCodeLang = + | "javascript" + | "typescript" + | "jsx" + | "tsx" | "python" | "sql:sqlite" | "sql:postgres" @@ -21,6 +25,7 @@ export type ValidatorCodeLang = | "sql:tsql" | "sql:bigquery" | "sql:ansi"; +export type ValidatorType = "code" | "oxc"; export type ExpressionDtype = "str" | "int" | "float" | "bool"; @@ -48,6 +53,7 @@ export type RecipeNodeData = { | LlmType | "validator_python" | "validator_sql" + | "validator_oxc" | "expression" | "seed" | "markdown_note" @@ -258,6 +264,8 @@ export type ValidatorConfig = { drop?: boolean; // biome-ignore lint/style/useNamingConvention: api schema target_columns: string[]; + // ui-only + validator_type: ValidatorType; // biome-ignore lint/style/useNamingConvention: api schema code_lang: ValidatorCodeLang; // ui ergonomics (serialized to int in payload) diff --git a/studio/frontend/src/features/recipe-studio/utils/config-factories.ts b/studio/frontend/src/features/recipe-studio/utils/config-factories.ts index 67de6dc7e4..44ca5b0fca 100644 --- a/studio/frontend/src/features/recipe-studio/utils/config-factories.ts +++ b/studio/frontend/src/features/recipe-studio/utils/config-factories.ts @@ -11,6 +11,7 @@ import type { SamplerConfig, SamplerType, ValidatorCodeLang, + ValidatorType, ValidatorConfig, } from "../types"; import { nextName } from "./naming"; @@ -279,17 +280,26 @@ export function makeExpressionConfig( export function makeValidatorConfig( id: string, + validatorType: ValidatorType, codeLang: ValidatorCodeLang, existing: NodeConfig[], ): ValidatorConfig { - const isSql = codeLang.startsWith("sql:"); + const isSql = validatorType === "code" && codeLang.startsWith("sql:"); + const isOxc = validatorType === "oxc"; + let namePrefix = "validator_python"; + if (isSql) { + namePrefix = "validator_sql"; + } else if (isOxc) { + namePrefix = "validator_oxc"; + } return { id, kind: "validator", - name: nextName(existing, isSql ? "validator_sql" : "validator_python"), + name: nextName(existing, namePrefix), drop: false, // biome-ignore lint/style/useNamingConvention: api schema target_columns: [], + validator_type: validatorType, // biome-ignore lint/style/useNamingConvention: api schema code_lang: codeLang, batch_size: "10", diff --git a/studio/frontend/src/features/recipe-studio/utils/graph/recipe-graph-connection.ts b/studio/frontend/src/features/recipe-studio/utils/graph/recipe-graph-connection.ts index 82bf105e4d..fc109804e6 100644 --- a/studio/frontend/src/features/recipe-studio/utils/graph/recipe-graph-connection.ts +++ b/studio/frontend/src/features/recipe-studio/utils/graph/recipe-graph-connection.ts @@ -15,6 +15,10 @@ import { isLlmConfig, isSubcategoryConfig, } from "../index"; +import { + VALIDATOR_OXC_CODE_LANGS, + VALIDATOR_SQL_CODE_LANGS, +} from "../validators/code-lang"; function buildTemplateWithRef(template: string, ref: string): string { if (template.includes(ref)) { @@ -147,6 +151,25 @@ function isModelSemanticRelation(source: NodeConfig, target: NodeConfig): boolea ); } +function canApplyCodeLangToValidator( + validator: Extract, + codeLang: string, +): boolean { + const normalized = codeLang.trim(); + if (!normalized) { + return false; + } + if (validator.validator_type === "oxc") { + return VALIDATOR_OXC_CODE_LANGS.includes( + normalized as typeof validator.code_lang, + ); + } + if (normalized === "python") { + return true; + } + return VALIDATOR_SQL_CODE_LANGS.includes(normalized as typeof validator.code_lang); +} + function countHandleUsage( edges: Edge[], nodeId: string, @@ -366,13 +389,19 @@ export function applyRecipeConnection( target.kind === "validator" ) { const nextCodeLang = (source.code_lang ?? "").trim(); + const canUseCodeLangForTarget = canApplyCodeLangToValidator( + target, + nextCodeLang, + ); const next = { ...target, // biome-ignore lint/style/useNamingConvention: api schema target_columns: [source.name], // biome-ignore lint/style/useNamingConvention: api schema code_lang: - (nextCodeLang || target.code_lang) as typeof target.code_lang, + ( + canUseCodeLangForTarget ? nextCodeLang : target.code_lang + ) as typeof target.code_lang, }; return { edges: nextEdges, configs: { ...configs, [target.id]: next } }; } diff --git a/studio/frontend/src/features/recipe-studio/utils/import/parsers/validator-parser.ts b/studio/frontend/src/features/recipe-studio/utils/import/parsers/validator-parser.ts index 466b20dafd..f982245821 100644 --- a/studio/frontend/src/features/recipe-studio/utils/import/parsers/validator-parser.ts +++ b/studio/frontend/src/features/recipe-studio/utils/import/parsers/validator-parser.ts @@ -2,6 +2,36 @@ import type { ValidatorConfig } from "../../../types"; import { readNumberString } from "../helpers"; import { normalizeValidatorCodeLang } from "../../validators/code-lang"; +const OXC_VALIDATION_FN_MARKER = "unsloth_oxc_validator"; + +function parseOxcCodeLang(validationFunctionRaw: string): string { + if (!validationFunctionRaw.startsWith(OXC_VALIDATION_FN_MARKER)) { + return ""; + } + const suffix = validationFunctionRaw + .slice(OXC_VALIDATION_FN_MARKER.length) + .trim(); + if (!suffix) { + return ""; + } + const normalizedSuffix = suffix.startsWith(":") || suffix.startsWith("_") + ? suffix.slice(1).trim().toLowerCase() + : suffix.toLowerCase(); + if (normalizedSuffix === "js") { + return "javascript"; + } + if (normalizedSuffix === "ts") { + return "typescript"; + } + if (normalizedSuffix === "jsx" || normalizedSuffix === "tsx") { + return normalizedSuffix; + } + if (normalizedSuffix === "javascript" || normalizedSuffix === "typescript") { + return normalizedSuffix; + } + return ""; +} + export function parseValidator( column: Record, name: string, @@ -17,6 +47,14 @@ export function parseValidator( column.validator_params && typeof column.validator_params === "object" ? (column.validator_params as Record) : {}; + const validationFunctionRaw = + typeof params.validation_function === "string" + ? params.validation_function.trim() + : ""; + const isOxc = + String(column.validator_type ?? "").trim() === "local_callable" && + validationFunctionRaw.startsWith(OXC_VALIDATION_FN_MARKER); + const oxcLang = isOxc ? parseOxcCodeLang(validationFunctionRaw) : ""; return { id, kind: "validator", @@ -24,8 +62,11 @@ export function parseValidator( drop: column.drop === true, // biome-ignore lint/style/useNamingConvention: api schema target_columns: targetColumns, + validator_type: isOxc ? "oxc" : "code", // biome-ignore lint/style/useNamingConvention: api schema - code_lang: normalizeValidatorCodeLang(params.code_lang), + code_lang: normalizeValidatorCodeLang( + isOxc ? oxcLang || "javascript" : params.code_lang, + ), batch_size: readNumberString(column.batch_size) || "10", }; } diff --git a/studio/frontend/src/features/recipe-studio/utils/node-data.ts b/studio/frontend/src/features/recipe-studio/utils/node-data.ts index 1f71481d9a..c887a979c3 100644 --- a/studio/frontend/src/features/recipe-studio/utils/node-data.ts +++ b/studio/frontend/src/features/recipe-studio/utils/node-data.ts @@ -30,13 +30,22 @@ export function nodeDataFromConfig( }; } if (config.kind === "validator") { + const isOxc = config.validator_type === "oxc"; + const isSql = config.code_lang.startsWith("sql:"); + let subtype = "Python"; + let blockType: RecipeNodeData["blockType"] = "validator_python"; + if (isOxc) { + subtype = "OXC"; + blockType = "validator_oxc"; + } else if (isSql) { + subtype = "SQL"; + blockType = "validator_sql"; + } return { title: "Validator", kind: "validator", - subtype: config.code_lang.startsWith("sql:") ? "SQL" : "Python", - blockType: config.code_lang.startsWith("sql:") - ? "validator_sql" - : "validator_python", + subtype, + blockType, name: config.name, layoutDirection, }; diff --git a/studio/frontend/src/features/recipe-studio/utils/payload/builders-validator.ts b/studio/frontend/src/features/recipe-studio/utils/payload/builders-validator.ts index 6fd8ccea37..a4c473ab24 100644 --- a/studio/frontend/src/features/recipe-studio/utils/payload/builders-validator.ts +++ b/studio/frontend/src/features/recipe-studio/utils/payload/builders-validator.ts @@ -1,5 +1,7 @@ import type { ValidatorConfig } from "../../types"; +const OXC_VALIDATION_FN_MARKER = "unsloth_oxc_validator"; + function parseBatchSize(value: string): number { const parsed = Number.parseInt(value, 10); if (!Number.isFinite(parsed) || parsed < 1) { @@ -18,6 +20,27 @@ export function buildValidatorColumn( if (targetColumns.length === 0) { errors.push(`Validator ${config.name}: target code column required.`); } + if (config.validator_type === "oxc") { + return { + // biome-ignore lint/style/useNamingConvention: api schema + column_type: "validation", + name: config.name, + drop: config.drop ?? false, + // biome-ignore lint/style/useNamingConvention: api schema + target_columns: targetColumns, + // biome-ignore lint/style/useNamingConvention: api schema + validator_type: "local_callable", + // biome-ignore lint/style/useNamingConvention: api schema + validator_params: { + // backend resolves this marker to a real callable. + // biome-ignore lint/style/useNamingConvention: api schema + validation_function: `${OXC_VALIDATION_FN_MARKER}:${config.code_lang}`, + }, + // biome-ignore lint/style/useNamingConvention: api schema + batch_size: parseBatchSize(config.batch_size), + }; + } + return { // biome-ignore lint/style/useNamingConvention: api schema column_type: "validation", diff --git a/studio/frontend/src/features/recipe-studio/utils/payload/validate.ts b/studio/frontend/src/features/recipe-studio/utils/payload/validate.ts index 446e61c7a3..bc303689fb 100644 --- a/studio/frontend/src/features/recipe-studio/utils/payload/validate.ts +++ b/studio/frontend/src/features/recipe-studio/utils/payload/validate.ts @@ -2,8 +2,10 @@ import type { ModelConfig, ModelProviderConfig, NodeConfig, + ValidatorCodeLang, ValidatorConfig, } from "../../types"; +import { VALIDATOR_OXC_CODE_LANGS } from "../validators/code-lang"; export function validateSubcategoryConfigs( configs: Record, @@ -134,6 +136,17 @@ export function validateValidatorConfigs( errors.push(`Validator ${config.name}: target '${target}' must be LLM Code.`); continue; } + if ( + config.validator_type === "oxc" && + !VALIDATOR_OXC_CODE_LANGS.includes( + (targetConfig.code_lang ?? "").trim() as ValidatorCodeLang, + ) + ) { + errors.push( + `Validator ${config.name}: target '${target}' must use javascript/typescript/jsx/tsx.`, + ); + continue; + } if ((targetConfig.code_lang ?? "").trim() !== config.code_lang.trim()) { errors.push( `Validator ${config.name}: code_lang '${config.code_lang}' must match target '${target}' (${targetConfig.code_lang ?? "unknown"}).`, diff --git a/studio/frontend/src/features/recipe-studio/utils/validation.ts b/studio/frontend/src/features/recipe-studio/utils/validation.ts index 8d62c45eb7..a130fabd15 100644 --- a/studio/frontend/src/features/recipe-studio/utils/validation.ts +++ b/studio/frontend/src/features/recipe-studio/utils/validation.ts @@ -1,5 +1,6 @@ import type { NodeConfig } from "../types"; import { isValidSex, parseAgeRange, parseIntNumber, parseNumber } from "./parse"; +import { VALIDATOR_OXC_CODE_LANGS, VALIDATOR_SQL_CODE_LANGS } from "./validators/code-lang"; const TRACE_MODES = new Set(["none", "last_message", "all_messages"]); @@ -205,6 +206,15 @@ export function getConfigErrors(config: NodeConfig | null): string[] { } if (!config.code_lang.trim()) { errors.push("Validator code language is required."); + } else if (config.validator_type === "oxc") { + if (!VALIDATOR_OXC_CODE_LANGS.includes(config.code_lang)) { + errors.push("OXC validator code language must be javascript/typescript/jsx/tsx."); + } + } else if ( + config.code_lang !== "python" && + !VALIDATOR_SQL_CODE_LANGS.includes(config.code_lang) + ) { + errors.push("Code validator code language must be python or sql dialect."); } } if (config.kind === "seed") { diff --git a/studio/frontend/src/features/recipe-studio/utils/validators/code-lang.ts b/studio/frontend/src/features/recipe-studio/utils/validators/code-lang.ts index 1b2778f8b7..35aeb32388 100644 --- a/studio/frontend/src/features/recipe-studio/utils/validators/code-lang.ts +++ b/studio/frontend/src/features/recipe-studio/utils/validators/code-lang.ts @@ -1,5 +1,12 @@ import type { ValidatorCodeLang } from "../../types"; +export const VALIDATOR_OXC_CODE_LANGS: ValidatorCodeLang[] = [ + "javascript", + "typescript", + "jsx", + "tsx", +]; + export const VALIDATOR_SQL_CODE_LANGS: ValidatorCodeLang[] = [ "sql:sqlite", "sql:postgres", @@ -10,6 +17,7 @@ export const VALIDATOR_SQL_CODE_LANGS: ValidatorCodeLang[] = [ ]; const VALIDATOR_CODE_LANG_SET = new Set([ + ...VALIDATOR_OXC_CODE_LANGS, "python", ...VALIDATOR_SQL_CODE_LANGS, ]); @@ -25,6 +33,9 @@ export function normalizeValidatorCodeLang( if (!raw) { return "python"; } + if (VALIDATOR_OXC_CODE_LANGS.includes(raw as ValidatorCodeLang)) { + return raw as ValidatorCodeLang; + } if (raw === "python") { return "python"; }