feat(data-recipes, validators): add OXC validator runtime and integration with recipe studio
This commit is contained in:
parent
b277308b7e
commit
552eb06bed
21 changed files with 1132 additions and 19 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -53,6 +53,7 @@ firebase-debug.log
|
|||
# Other
|
||||
resources/
|
||||
tmp/
|
||||
**/node_modules/
|
||||
auth.db
|
||||
studio/frontend/package-lock.json
|
||||
|
||||
|
|
|
|||
2
setup.sh
2
setup.sh
|
|
@ -99,6 +99,8 @@ echo "Building frontend..."
|
|||
cd "$SCRIPT_DIR/studio/frontend"
|
||||
run_quiet "npm install" npm install
|
||||
run_quiet "npm run build" npm run build
|
||||
cd "$SCRIPT_DIR/studio/backend/core/data_recipe/oxc-validator"
|
||||
run_quiet "npm install (oxc validator runtime)" npm install
|
||||
cd "$SCRIPT_DIR"
|
||||
echo "✅ Frontend built to studio/frontend/dist"
|
||||
|
||||
|
|
|
|||
301
studio/backend/core/data_recipe/local_callable_validators.py
Normal file
301
studio/backend/core/data_recipe/local_callable_validators.py
Normal file
|
|
@ -0,0 +1,301 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OXC_VALIDATION_FN_MARKER = "unsloth_oxc_validator"
|
||||
|
||||
_OXC_LANG_TO_NODE_LANG = {
|
||||
"javascript": "js",
|
||||
"typescript": "ts",
|
||||
"jsx": "jsx",
|
||||
"tsx": "tsx",
|
||||
}
|
||||
|
||||
_OXC_TOOL_DIR = Path(__file__).resolve().parent / "oxc-validator"
|
||||
_OXC_RUNNER_PATH = _OXC_TOOL_DIR / "validate.mjs"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OxcLocalCallableValidatorSpec:
|
||||
name: str
|
||||
drop: bool
|
||||
target_columns: list[str]
|
||||
batch_size: int
|
||||
code_lang: str
|
||||
|
||||
|
||||
def split_oxc_local_callable_validators(
|
||||
recipe_core: dict[str, Any],
|
||||
) -> tuple[dict[str, Any], list[OxcLocalCallableValidatorSpec]]:
|
||||
columns = recipe_core.get("columns")
|
||||
if not isinstance(columns, list):
|
||||
return recipe_core, []
|
||||
|
||||
sanitized = deepcopy(recipe_core)
|
||||
sanitized_columns = sanitized.get("columns")
|
||||
if not isinstance(sanitized_columns, list):
|
||||
return sanitized, []
|
||||
|
||||
llm_code_lang_by_name = _extract_llm_code_lang_by_name(sanitized_columns)
|
||||
kept_columns: list[Any] = []
|
||||
oxc_specs: list[OxcLocalCallableValidatorSpec] = []
|
||||
|
||||
for column in sanitized_columns:
|
||||
if not isinstance(column, dict):
|
||||
kept_columns.append(column)
|
||||
continue
|
||||
|
||||
maybe_spec = _parse_oxc_spec(
|
||||
column=column,
|
||||
llm_code_lang_by_name=llm_code_lang_by_name,
|
||||
)
|
||||
if maybe_spec is None:
|
||||
kept_columns.append(column)
|
||||
continue
|
||||
oxc_specs.append(maybe_spec)
|
||||
|
||||
sanitized["columns"] = kept_columns
|
||||
return sanitized, oxc_specs
|
||||
|
||||
|
||||
def register_oxc_local_callable_validators(
|
||||
*,
|
||||
builder,
|
||||
specs: list[OxcLocalCallableValidatorSpec],
|
||||
) -> None:
|
||||
if not specs:
|
||||
return
|
||||
|
||||
from data_designer.config.column_configs import ValidationColumnConfig
|
||||
from data_designer.config.validator_params import (
|
||||
LocalCallableValidatorParams,
|
||||
ValidatorType,
|
||||
)
|
||||
|
||||
for spec in specs:
|
||||
validation_function = _build_oxc_validation_function(spec.code_lang)
|
||||
builder.add_column(
|
||||
ValidationColumnConfig(
|
||||
name=spec.name,
|
||||
drop=spec.drop,
|
||||
target_columns=spec.target_columns,
|
||||
validator_type=ValidatorType.LOCAL_CALLABLE,
|
||||
validator_params=LocalCallableValidatorParams(
|
||||
validation_function=validation_function,
|
||||
),
|
||||
batch_size=spec.batch_size,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _parse_oxc_spec(
|
||||
*,
|
||||
column: dict[str, Any],
|
||||
llm_code_lang_by_name: dict[str, str],
|
||||
) -> OxcLocalCallableValidatorSpec | None:
|
||||
if str(column.get("column_type") or "").strip() != "validation":
|
||||
return None
|
||||
if str(column.get("validator_type") or "").strip() != "local_callable":
|
||||
return None
|
||||
|
||||
params = column.get("validator_params")
|
||||
if not isinstance(params, dict):
|
||||
return None
|
||||
|
||||
fn_raw = params.get("validation_function")
|
||||
fn_name = fn_raw.strip() if isinstance(fn_raw, str) else ""
|
||||
if not fn_name.startswith(OXC_VALIDATION_FN_MARKER):
|
||||
return None
|
||||
|
||||
name = str(column.get("name") or "").strip()
|
||||
if not name:
|
||||
return None
|
||||
|
||||
target_columns_raw = column.get("target_columns")
|
||||
target_columns = (
|
||||
[value.strip() for value in target_columns_raw if isinstance(value, str) and value.strip()]
|
||||
if isinstance(target_columns_raw, list)
|
||||
else []
|
||||
)
|
||||
if not target_columns:
|
||||
return None
|
||||
|
||||
code_lang = _resolve_oxc_lang(
|
||||
fn_name=fn_name,
|
||||
target_columns=target_columns,
|
||||
llm_code_lang_by_name=llm_code_lang_by_name,
|
||||
)
|
||||
batch_size = _parse_batch_size(column.get("batch_size"))
|
||||
drop = bool(column.get("drop") is True)
|
||||
|
||||
return OxcLocalCallableValidatorSpec(
|
||||
name=name,
|
||||
drop=drop,
|
||||
target_columns=target_columns,
|
||||
batch_size=batch_size,
|
||||
code_lang=code_lang,
|
||||
)
|
||||
|
||||
|
||||
def _parse_batch_size(value: Any) -> int:
|
||||
try:
|
||||
parsed = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return 10
|
||||
return parsed if parsed >= 1 else 10
|
||||
|
||||
|
||||
def _extract_llm_code_lang_by_name(columns: list[Any]) -> dict[str, str]:
|
||||
out: dict[str, str] = {}
|
||||
for column in columns:
|
||||
if not isinstance(column, dict):
|
||||
continue
|
||||
if str(column.get("column_type") or "").strip() != "llm-code":
|
||||
continue
|
||||
name = str(column.get("name") or "").strip()
|
||||
code_lang = str(column.get("code_lang") or "").strip()
|
||||
if name and code_lang:
|
||||
out[name] = code_lang
|
||||
return out
|
||||
|
||||
|
||||
def _resolve_oxc_lang(
|
||||
*,
|
||||
fn_name: str,
|
||||
target_columns: list[str],
|
||||
llm_code_lang_by_name: dict[str, str],
|
||||
) -> str:
|
||||
_, _, marker_lang = fn_name.partition(":")
|
||||
marker_lang = marker_lang.strip()
|
||||
if marker_lang in _OXC_LANG_TO_NODE_LANG:
|
||||
return marker_lang
|
||||
|
||||
first_target = target_columns[0]
|
||||
target_lang = llm_code_lang_by_name.get(first_target, "").strip()
|
||||
if target_lang in _OXC_LANG_TO_NODE_LANG:
|
||||
return target_lang
|
||||
return "javascript"
|
||||
|
||||
|
||||
@lru_cache(maxsize=8)
|
||||
def _build_oxc_validation_function(lang: str):
|
||||
node_lang = _OXC_LANG_TO_NODE_LANG.get(lang, "js")
|
||||
|
||||
def _validator(df):
|
||||
import pandas as pd # imported lazily for local callable runtime
|
||||
|
||||
row_count = int(len(df.index))
|
||||
if row_count == 0:
|
||||
return pd.DataFrame({"is_valid": []})
|
||||
|
||||
code_column = str(df.columns[0]) if len(df.columns) > 0 else ""
|
||||
code_values = (
|
||||
["" for _ in range(row_count)]
|
||||
if not code_column
|
||||
else ["" if value is None else str(value) for value in df[code_column].tolist()]
|
||||
)
|
||||
|
||||
results = _run_oxc_batch(node_lang=node_lang, code_values=code_values)
|
||||
if len(results) != row_count:
|
||||
results = _fallback_results(
|
||||
row_count,
|
||||
"OXC validator returned mismatched result size.",
|
||||
)
|
||||
return pd.DataFrame(results)
|
||||
|
||||
_validator.__name__ = f"{OXC_VALIDATION_FN_MARKER}_{node_lang}"
|
||||
return _validator
|
||||
|
||||
|
||||
def _run_oxc_batch(*, node_lang: str, code_values: list[str]) -> list[dict[str, Any]]:
|
||||
if not _OXC_RUNNER_PATH.exists():
|
||||
return _fallback_results(
|
||||
len(code_values),
|
||||
f"OXC runner missing at {_OXC_RUNNER_PATH}",
|
||||
)
|
||||
|
||||
payload = {
|
||||
"lang": node_lang,
|
||||
"codes": code_values,
|
||||
}
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["node", str(_OXC_RUNNER_PATH)],
|
||||
cwd=str(_OXC_TOOL_DIR),
|
||||
input=json.dumps(payload),
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
except (OSError, ValueError) as exc:
|
||||
logger.warning("OXC subprocess launch failed: %s", exc)
|
||||
return _fallback_results(len(code_values), f"OXC launch failed: {exc}")
|
||||
|
||||
if proc.returncode != 0:
|
||||
message = (proc.stderr or proc.stdout or "unknown error").strip()
|
||||
if len(message) > 300:
|
||||
message = f"{message[:300]}..."
|
||||
return _fallback_results(len(code_values), f"OXC failed: {message}")
|
||||
|
||||
try:
|
||||
raw = json.loads(proc.stdout)
|
||||
except json.JSONDecodeError:
|
||||
return _fallback_results(len(code_values), "OXC output parse failed.")
|
||||
|
||||
if not isinstance(raw, list):
|
||||
return _fallback_results(len(code_values), "OXC output must be an array.")
|
||||
|
||||
out: list[dict[str, Any]] = []
|
||||
for item in raw:
|
||||
if not isinstance(item, dict):
|
||||
out.append(
|
||||
{
|
||||
"is_valid": False,
|
||||
"error_count": 1,
|
||||
"error_message": "Invalid OXC result entry.",
|
||||
"severity": None,
|
||||
"labels": [],
|
||||
"codeframe": None,
|
||||
}
|
||||
)
|
||||
continue
|
||||
is_valid_raw = item.get("is_valid")
|
||||
error_count_raw = item.get("error_count")
|
||||
message_raw = item.get("error_message")
|
||||
severity_raw = item.get("severity")
|
||||
labels_raw = item.get("labels")
|
||||
codeframe_raw = item.get("codeframe")
|
||||
out.append(
|
||||
{
|
||||
"is_valid": bool(is_valid_raw) if isinstance(is_valid_raw, bool) else False,
|
||||
"error_count": int(error_count_raw) if isinstance(error_count_raw, int) else 0,
|
||||
"error_message": str(message_raw or ""),
|
||||
"severity": str(severity_raw) if isinstance(severity_raw, str) else None,
|
||||
"labels": labels_raw if isinstance(labels_raw, list) else [],
|
||||
"codeframe": str(codeframe_raw) if isinstance(codeframe_raw, str) else None,
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _fallback_results(row_count: int, message: str) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"is_valid": False,
|
||||
"error_count": 1,
|
||||
"error_message": message,
|
||||
"severity": None,
|
||||
"labels": [],
|
||||
"codeframe": None,
|
||||
}
|
||||
for _ in range(row_count)
|
||||
]
|
||||
445
studio/backend/core/data_recipe/oxc-validator/package-lock.json
generated
Normal file
445
studio/backend/core/data_recipe/oxc-validator/package-lock.json
generated
Normal file
|
|
@ -0,0 +1,445 @@
|
|||
{
|
||||
"name": "unsloth-oxc-validator-runtime",
|
||||
"version": "0.0.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "unsloth-oxc-validator-runtime",
|
||||
"version": "0.0.1",
|
||||
"dependencies": {
|
||||
"oxc-parser": "^0.116.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@emnapi/core": {
|
||||
"version": "1.8.1",
|
||||
"resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.8.1.tgz",
|
||||
"integrity": "sha512-AvT9QFpxK0Zd8J0jopedNm+w/2fIzvtPKPjqyw9jwvBaReTTqPBk9Hixaz7KbjimP+QNz605/XnjFcDAL2pqBg==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"@emnapi/wasi-threads": "1.1.0",
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@emnapi/runtime": {
|
||||
"version": "1.8.1",
|
||||
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.1.tgz",
|
||||
"integrity": "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@emnapi/wasi-threads": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz",
|
||||
"integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/wasm-runtime": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.1.tgz",
|
||||
"integrity": "sha512-p64ah1M1ld8xjWv3qbvFwHiFVWrq1yFvV4f7w+mzaqiR4IlSgkqhcRdHwsGgomwzBH51sRY4NEowLxnaBjcW/A==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"@emnapi/core": "^1.7.1",
|
||||
"@emnapi/runtime": "^1.7.1",
|
||||
"@tybys/wasm-util": "^0.10.1"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-android-arm-eabi": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm-eabi/-/binding-android-arm-eabi-0.116.0.tgz",
|
||||
"integrity": "sha512-AOET7YIOU3+ANO/3xQeRVGN5Xx6+JGXaIwlqkcHSfxJ/zzw2B6jb0YaLhX45SeRluKVTU8rka4N/tHtNoJjoCg==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"android"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-android-arm64": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm64/-/binding-android-arm64-0.116.0.tgz",
|
||||
"integrity": "sha512-yh0Zvth5cQ6XZkP3QF9MDrXf695zr5XxXq/wBQqpZb0uAgI9wpr98/Hx2RZITMfnNjkIq2VcyU44o3A0bdEmlQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"android"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-darwin-arm64": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-arm64/-/binding-darwin-arm64-0.116.0.tgz",
|
||||
"integrity": "sha512-plcTd/Jska55dToZz6XdRBPRVsj+asjD8QCpQFvt3Wj8pY+10D1pE53Mei3POAS/wSRSy7HiQ2twrm7H2A0CjA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-darwin-x64": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-x64/-/binding-darwin-x64-0.116.0.tgz",
|
||||
"integrity": "sha512-ahqcF3e3x5Z2ZepzXpZ8ugREdmxvBL+g1nQ0SxO11pIZfck6UtbOtwtdAAxnQXBHHtidu7lPcrBq1SEx26t1PQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-freebsd-x64": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-freebsd-x64/-/binding-freebsd-x64-0.116.0.tgz",
|
||||
"integrity": "sha512-yo2/LaSXtlzKBurvNbwun/sN/RJwW3XhbMr069FwNVtft7GBnaLLdPIz/sf47icxw/BPViEX6wFvzeD12mtrAg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"freebsd"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-arm-gnueabihf": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-0.116.0.tgz",
|
||||
"integrity": "sha512-EiZeliIPPdFsuaPx8PzDMVijD/4YaUxO46/eYPk5raRocJqjjxOG6GAacQ8UrG3fbrgYjaEChfYL1e8DyE445A==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-arm-musleabihf": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-0.116.0.tgz",
|
||||
"integrity": "sha512-Nf7hnKRYRSIgglQcLAqE2St4b/Yr6dh+Z7in8mxol065Knevw71XZAiV1fmPSojq6uKPLV9eoH/wFrgr4TnZXw==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-arm64-gnu": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-0.116.0.tgz",
|
||||
"integrity": "sha512-9SJI0S4Qggn3QHpT8Y1jtZceA0m4BlpvO3ne2Wxd33UdTHMmelAnrXryjWutHWQtjCzOwSnFBEoQAdNNyt1u3A==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-arm64-musl": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-musl/-/binding-linux-arm64-musl-0.116.0.tgz",
|
||||
"integrity": "sha512-wMZ6//GI+q1JwO7G2OR51+eA5P8Gr3BobU8RAzCGJptvyGMkWb7KQ1E8s8naVZRr6bSGWAL2p3mCzKOxmEPmrA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-ppc64-gnu": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-0.116.0.tgz",
|
||||
"integrity": "sha512-5BO0KCzTG2HZTnp3r6SCAOeCs/GwFBQJ1WAOG/ROfDf1fVVEy6hrtLKTLCuUMaamH38v+1+RVEmzRkzBj+rMDQ==",
|
||||
"cpu": [
|
||||
"ppc64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-riscv64-gnu": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-0.116.0.tgz",
|
||||
"integrity": "sha512-M24gYb/ocVMnLwnH2wY5sLt4sRBkAUHDmfiYtyUYdKTkfPOKtpopd5otsL/BPLnIhpMD8zby4uXVvw7BU0UIlw==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-riscv64-musl": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-0.116.0.tgz",
|
||||
"integrity": "sha512-LHLXTHCH0bdvGjlitwr1ngeh32GAgq9HYzQ5VAgt0k0UT84AS8AkXj9Spoa9l20fXkVgSvAKcCEkydi4Ol23Dw==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-s390x-gnu": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-0.116.0.tgz",
|
||||
"integrity": "sha512-VE+XsztuE5jdHvLIDIQMuyDpz5NJGq1Vx/8EXYF0sS/gehlv9GhDpGVWU0SCZ/LjzIy4io/Z0W84UudqufvP3g==",
|
||||
"cpu": [
|
||||
"s390x"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-x64-gnu": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-gnu/-/binding-linux-x64-gnu-0.116.0.tgz",
|
||||
"integrity": "sha512-rxUkauyjjCmgA7BoR63ogRGEtgubROnCm8AXE9ydg+p42jCGLLqG05mFcS2eC+FYyAU58ZFJNXXeqFW1iCyTGQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-linux-x64-musl": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-musl/-/binding-linux-x64-musl-0.116.0.tgz",
|
||||
"integrity": "sha512-0zoZlk9MmXe6oTgSh5lT1D51SDC1bfwC96JmE1amMFAPdEbJk5MFRisfTN9TFBpBigQua65842tjaxqMiorAYw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-openharmony-arm64": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-openharmony-arm64/-/binding-openharmony-arm64-0.116.0.tgz",
|
||||
"integrity": "sha512-PGS7Xqik77U9WMyW626gAD5A2rSN629UvyYJKAl/tgpT+KqZI4+56pJfExhv8IW/PpSHjYHwjmakwobLikz8ww==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"openharmony"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-wasm32-wasi": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-wasm32-wasi/-/binding-wasm32-wasi-0.116.0.tgz",
|
||||
"integrity": "sha512-lGNf/9PU8XxB4Gt1Gr1AKwSrjxGYa6os0PlrT4bpoQsfE3gaZonQTKwJyKhiQdgy7pBCI+ed1LB1NNib1FYULw==",
|
||||
"cpu": [
|
||||
"wasm32"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"@napi-rs/wasm-runtime": "^1.1.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-win32-arm64-msvc": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-0.116.0.tgz",
|
||||
"integrity": "sha512-tcsOHE31duBSRQXZ7NfdtjmMKZwQYlS00PwAMJ4w5oXs3iPCvisUuIXP7Ko4FzeOBTRvkd64btxtQ6cRM0Kwlw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-win32-ia32-msvc": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-0.116.0.tgz",
|
||||
"integrity": "sha512-higCz/x+dOQ264YEk22hnu4RDqvjhfehjFORpxoh42QyUxsP6eIembYesBUu5ilALWo0HvRD+m89az2BSTwqpQ==",
|
||||
"cpu": [
|
||||
"ia32"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-parser/binding-win32-x64-msvc": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.116.0.tgz",
|
||||
"integrity": "sha512-Lg2SRmVHpGG85knDVLbv44r1bYn0OpIV0vg9jVmoEIpDj3Q4kwXuQ6MWVtuslwHR8o2CSiqdBeEn1n1URrs6Eg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@oxc-project/types": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.116.0.tgz",
|
||||
"integrity": "sha512-uOT8S1tlPmDckNxMNtIudN/yXpLdnhlJMX2oLS7cxCd7L0sUF09A/EbSVMWT3Y/iT44IwXCJSJfgfSxXAqWf9Q==",
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/Boshen"
|
||||
}
|
||||
},
|
||||
"node_modules/@tybys/wasm-util": {
|
||||
"version": "0.10.1",
|
||||
"resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz",
|
||||
"integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/oxc-parser": {
|
||||
"version": "0.116.0",
|
||||
"resolved": "https://registry.npmjs.org/oxc-parser/-/oxc-parser-0.116.0.tgz",
|
||||
"integrity": "sha512-ugEo6wwqaqCGcpi7GsLCwSkoD7gIXzvtdaTxE+mbrXFYazU5Q9YdpZdAj9z2b79i/xlv+uW2aAvyzGAlpUzhKQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@oxc-project/types": "^0.116.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^20.19.0 || >=22.12.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/Boshen"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@oxc-parser/binding-android-arm-eabi": "0.116.0",
|
||||
"@oxc-parser/binding-android-arm64": "0.116.0",
|
||||
"@oxc-parser/binding-darwin-arm64": "0.116.0",
|
||||
"@oxc-parser/binding-darwin-x64": "0.116.0",
|
||||
"@oxc-parser/binding-freebsd-x64": "0.116.0",
|
||||
"@oxc-parser/binding-linux-arm-gnueabihf": "0.116.0",
|
||||
"@oxc-parser/binding-linux-arm-musleabihf": "0.116.0",
|
||||
"@oxc-parser/binding-linux-arm64-gnu": "0.116.0",
|
||||
"@oxc-parser/binding-linux-arm64-musl": "0.116.0",
|
||||
"@oxc-parser/binding-linux-ppc64-gnu": "0.116.0",
|
||||
"@oxc-parser/binding-linux-riscv64-gnu": "0.116.0",
|
||||
"@oxc-parser/binding-linux-riscv64-musl": "0.116.0",
|
||||
"@oxc-parser/binding-linux-s390x-gnu": "0.116.0",
|
||||
"@oxc-parser/binding-linux-x64-gnu": "0.116.0",
|
||||
"@oxc-parser/binding-linux-x64-musl": "0.116.0",
|
||||
"@oxc-parser/binding-openharmony-arm64": "0.116.0",
|
||||
"@oxc-parser/binding-wasm32-wasi": "0.116.0",
|
||||
"@oxc-parser/binding-win32-arm64-msvc": "0.116.0",
|
||||
"@oxc-parser/binding-win32-ia32-msvc": "0.116.0",
|
||||
"@oxc-parser/binding-win32-x64-msvc": "0.116.0"
|
||||
}
|
||||
},
|
||||
"node_modules/tslib": {
|
||||
"version": "2.8.1",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
|
||||
"integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
|
||||
"license": "0BSD",
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"name": "unsloth-oxc-validator-runtime",
|
||||
"private": true,
|
||||
"version": "0.0.1",
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"oxc-parser": "^0.116.0"
|
||||
}
|
||||
}
|
||||
149
studio/backend/core/data_recipe/oxc-validator/validate.mjs
Normal file
149
studio/backend/core/data_recipe/oxc-validator/validate.mjs
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
import { parseSync } from "oxc-parser";
|
||||
|
||||
const LANG_TO_EXT = {
|
||||
js: "js",
|
||||
jsx: "jsx",
|
||||
ts: "ts",
|
||||
tsx: "tsx",
|
||||
};
|
||||
|
||||
function mapLang(value) {
|
||||
const normalized = String(value || "").trim().toLowerCase();
|
||||
if (normalized === "javascript" || normalized === "js") {
|
||||
return "js";
|
||||
}
|
||||
if (normalized === "typescript" || normalized === "ts") {
|
||||
return "ts";
|
||||
}
|
||||
if (normalized === "jsx") {
|
||||
return "jsx";
|
||||
}
|
||||
if (normalized === "tsx") {
|
||||
return "tsx";
|
||||
}
|
||||
return "js";
|
||||
}
|
||||
|
||||
function normalizeError(error) {
|
||||
if (typeof error === "string") {
|
||||
return {
|
||||
message: error.trim() || "Unknown OXC error",
|
||||
severity: null,
|
||||
labels: [],
|
||||
codeframe: null,
|
||||
};
|
||||
}
|
||||
if (!error || typeof error !== "object") {
|
||||
return {
|
||||
message: "Unknown OXC error",
|
||||
severity: null,
|
||||
labels: [],
|
||||
codeframe: null,
|
||||
};
|
||||
}
|
||||
const message = String(error.message || error.reason || "").trim() || "Unknown OXC error";
|
||||
const severity = typeof error.severity === "string" ? error.severity : null;
|
||||
const labels = Array.isArray(error.labels)
|
||||
? error.labels.map((label) => ({
|
||||
message:
|
||||
label && typeof label === "object" && typeof label.message === "string"
|
||||
? label.message
|
||||
: null,
|
||||
start:
|
||||
label && typeof label === "object" && Number.isInteger(label.start)
|
||||
? label.start
|
||||
: null,
|
||||
end:
|
||||
label && typeof label === "object" && Number.isInteger(label.end)
|
||||
? label.end
|
||||
: null,
|
||||
}))
|
||||
: [];
|
||||
const codeframe = typeof error.codeframe === "string" ? error.codeframe : null;
|
||||
return {
|
||||
message,
|
||||
severity,
|
||||
labels,
|
||||
codeframe,
|
||||
};
|
||||
}
|
||||
|
||||
function validateOne({ code, lang, index }) {
|
||||
const ext = LANG_TO_EXT[lang] ?? "js";
|
||||
const filename = `snippet_${index}.${ext}`;
|
||||
const source = typeof code === "string" ? code : String(code ?? "");
|
||||
|
||||
try {
|
||||
const parsed = parseSync(filename, source, {
|
||||
lang,
|
||||
sourceType: "module",
|
||||
showSemanticErrors: true,
|
||||
});
|
||||
const errors = Array.isArray(parsed?.errors)
|
||||
? parsed.errors.map(normalizeError).filter(Boolean)
|
||||
: [];
|
||||
const first = errors[0] ?? null;
|
||||
return {
|
||||
is_valid: errors.length === 0,
|
||||
error_count: errors.length,
|
||||
error_message: errors.slice(0, 3).map((error) => error.message).join(" | "),
|
||||
severity: first ? first.severity : null,
|
||||
labels: first ? first.labels : [],
|
||||
codeframe: first ? first.codeframe : null,
|
||||
};
|
||||
} catch (error) {
|
||||
const normalized = normalizeError(error);
|
||||
return {
|
||||
is_valid: false,
|
||||
error_count: 1,
|
||||
error_message: normalized.message,
|
||||
severity: normalized.severity,
|
||||
labels: normalized.labels,
|
||||
codeframe: normalized.codeframe,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function readStdin() {
|
||||
return new Promise((resolve, reject) => {
|
||||
let data = "";
|
||||
process.stdin.setEncoding("utf8");
|
||||
process.stdin.on("data", (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
process.stdin.on("end", () => resolve(data));
|
||||
process.stdin.on("error", (error) => reject(error));
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const raw = await readStdin();
|
||||
let payload;
|
||||
try {
|
||||
payload = JSON.parse(raw || "{}");
|
||||
} catch {
|
||||
process.stdout.write(
|
||||
JSON.stringify([
|
||||
{
|
||||
is_valid: false,
|
||||
error_count: 1,
|
||||
error_message: "Invalid JSON payload",
|
||||
severity: null,
|
||||
labels: [],
|
||||
codeframe: null,
|
||||
},
|
||||
]),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const lang = mapLang(payload?.lang);
|
||||
const codes = Array.isArray(payload?.codes) ? payload.codes : [];
|
||||
const out = codes.map((code, index) => validateOne({ code, lang, index }));
|
||||
process.stdout.write(JSON.stringify(out));
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
process.stderr.write(String(error?.stack || error));
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -7,6 +7,10 @@ from pathlib import Path
|
|||
from typing import Any
|
||||
|
||||
from .jsonable import to_jsonable
|
||||
from .local_callable_validators import (
|
||||
register_oxc_local_callable_validators,
|
||||
split_oxc_local_callable_validators,
|
||||
)
|
||||
_IMAGE_CONTEXT_PATCHED = False
|
||||
|
||||
|
||||
|
|
@ -196,7 +200,14 @@ def build_config_builder(recipe: dict[str, Any]):
|
|||
for key, value in recipe.items()
|
||||
if key not in {"model_providers", "mcp_providers"}
|
||||
}
|
||||
recipe_core, oxc_local_callable_specs = split_oxc_local_callable_validators(
|
||||
recipe_core
|
||||
)
|
||||
builder = DataDesignerConfigBuilder.from_config({"data_designer": recipe_core})
|
||||
register_oxc_local_callable_validators(
|
||||
builder=builder,
|
||||
specs=oxc_local_callable_specs,
|
||||
)
|
||||
|
||||
# DataDesignerConfigBuilder.from_config currently skips processors.
|
||||
# Re-attach explicitly so drop_columns/schema_transform survive API payload.
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ export type BlockType =
|
|||
| LlmType
|
||||
| "validator_python"
|
||||
| "validator_sql"
|
||||
| "validator_oxc"
|
||||
| "expression"
|
||||
| "markdown_note"
|
||||
| "seed"
|
||||
|
|
@ -303,7 +304,8 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [
|
|||
description: "Validate Python code columns.",
|
||||
icon: Shield02Icon,
|
||||
dialogKey: "validator",
|
||||
createConfig: (id, existing) => makeValidatorConfig(id, "python", existing),
|
||||
createConfig: (id, existing) =>
|
||||
makeValidatorConfig(id, "code", "python", existing),
|
||||
},
|
||||
{
|
||||
kind: "validator",
|
||||
|
|
@ -313,7 +315,17 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [
|
|||
icon: Shield02Icon,
|
||||
dialogKey: "validator",
|
||||
createConfig: (id, existing) =>
|
||||
makeValidatorConfig(id, "sql:sqlite", existing),
|
||||
makeValidatorConfig(id, "code", "sql:sqlite", existing),
|
||||
},
|
||||
{
|
||||
kind: "validator",
|
||||
type: "validator_oxc",
|
||||
title: "OXC Validator",
|
||||
description: "Validate JavaScript or TypeScript code columns.",
|
||||
icon: Shield02Icon,
|
||||
dialogKey: "validator",
|
||||
createConfig: (id, existing) =>
|
||||
makeValidatorConfig(id, "oxc", "javascript", existing),
|
||||
},
|
||||
{
|
||||
kind: "expression",
|
||||
|
|
@ -372,6 +384,9 @@ export function getBlockDefinitionForConfig(
|
|||
return getBlockDefinition("llm", config.llm_type);
|
||||
}
|
||||
if (config.kind === "validator") {
|
||||
if (config.validator_type === "oxc") {
|
||||
return getBlockDefinition("validator", "validator_oxc");
|
||||
}
|
||||
const isSql = config.code_lang.startsWith("sql:");
|
||||
return getBlockDefinition(
|
||||
"validator",
|
||||
|
|
|
|||
|
|
@ -71,7 +71,9 @@ type BlockSheetProps = {
|
|||
onAddModelProvider: () => void;
|
||||
onAddModelConfig: () => void;
|
||||
onAddExpression: () => void;
|
||||
onAddValidator: (type: "validator_python" | "validator_sql") => void;
|
||||
onAddValidator: (
|
||||
type: "validator_python" | "validator_sql" | "validator_oxc",
|
||||
) => void;
|
||||
onAddMarkdownNote: () => void;
|
||||
onOpenProcessors: () => void;
|
||||
copied: boolean;
|
||||
|
|
@ -318,7 +320,9 @@ export function BlockSheet({
|
|||
return;
|
||||
}
|
||||
if (kind === "validator") {
|
||||
onAddValidator(type as "validator_python" | "validator_sql");
|
||||
onAddValidator(
|
||||
type as "validator_python" | "validator_sql" | "validator_oxc",
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (kind === "expression") {
|
||||
|
|
|
|||
|
|
@ -14,7 +14,11 @@ import {
|
|||
import { type ReactElement, useMemo } from "react";
|
||||
import { useRecipeStudioStore } from "../../stores/recipe-studio";
|
||||
import type { ValidatorConfig } from "../../types";
|
||||
import { isValidatorCodeLang } from "../../utils/validators/code-lang";
|
||||
import {
|
||||
isValidatorCodeLang,
|
||||
VALIDATOR_OXC_CODE_LANGS,
|
||||
VALIDATOR_SQL_CODE_LANGS,
|
||||
} from "../../utils/validators/code-lang";
|
||||
import { FieldLabel } from "../shared/field-label";
|
||||
import { NameField } from "../shared/name-field";
|
||||
|
||||
|
|
@ -40,6 +44,22 @@ export function ValidatorDialog({
|
|||
if (!(item.kind === "llm" && item.llm_type === "code")) {
|
||||
return [];
|
||||
}
|
||||
if (config.validator_type === "oxc") {
|
||||
const lang = item.code_lang?.trim() ?? "";
|
||||
if (!VALIDATOR_OXC_CODE_LANGS.includes(lang as typeof config.code_lang)) {
|
||||
return [];
|
||||
}
|
||||
} else {
|
||||
const lang = item.code_lang?.trim() ?? "";
|
||||
if (
|
||||
!(
|
||||
lang === "python" ||
|
||||
VALIDATOR_SQL_CODE_LANGS.includes(lang as typeof config.code_lang)
|
||||
)
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
return [
|
||||
{
|
||||
name: item.name,
|
||||
|
|
@ -102,7 +122,9 @@ export function ValidatorDialog({
|
|||
</Select>
|
||||
{codeOptions.length === 0 && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Add an LLM Code block first.
|
||||
{config.validator_type === "oxc"
|
||||
? "Add an LLM Code block with javascript/typescript first."
|
||||
: "Add an LLM Code block first."}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ type UseRecipeEditorGraphArgs = {
|
|||
addModelConfigNode: (position?: XYPosition, openDialog?: boolean) => void;
|
||||
addExpressionNode: (position?: XYPosition, openDialog?: boolean) => void;
|
||||
addValidatorNode: (
|
||||
type: "validator_python" | "validator_sql",
|
||||
type: "validator_python" | "validator_sql" | "validator_oxc",
|
||||
position?: XYPosition,
|
||||
openDialog?: boolean,
|
||||
) => void;
|
||||
|
|
@ -92,7 +92,7 @@ type UseRecipeEditorGraphResult = {
|
|||
handleAddModelConfigFromSheet: () => void;
|
||||
handleAddExpressionFromSheet: () => void;
|
||||
handleAddValidatorFromSheet: (
|
||||
type: "validator_python" | "validator_sql",
|
||||
type: "validator_python" | "validator_sql" | "validator_oxc",
|
||||
) => void;
|
||||
handleAddMarkdownNoteFromSheet: () => void;
|
||||
};
|
||||
|
|
@ -213,7 +213,7 @@ export function useRecipeEditorGraph({
|
|||
}
|
||||
if (payload.kind === "validator") {
|
||||
addValidatorNode(
|
||||
payload.type as "validator_python" | "validator_sql",
|
||||
payload.type as "validator_python" | "validator_sql" | "validator_oxc",
|
||||
position,
|
||||
false,
|
||||
);
|
||||
|
|
@ -291,7 +291,7 @@ export function useRecipeEditorGraph({
|
|||
}, [addExpressionNode, getViewportCenterPosition]);
|
||||
|
||||
const handleAddValidatorFromSheet = useCallback(
|
||||
(type: "validator_python" | "validator_sql") => {
|
||||
(type: "validator_python" | "validator_sql" | "validator_oxc") => {
|
||||
addValidatorNode(type, getViewportCenterPosition());
|
||||
},
|
||||
[addValidatorNode, getViewportCenterPosition],
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ type RecipeStudioState = {
|
|||
addModelConfigNode: (position?: XYPosition, openDialog?: boolean) => void;
|
||||
addExpressionNode: (position?: XYPosition, openDialog?: boolean) => void;
|
||||
addValidatorNode: (
|
||||
type: "validator_python" | "validator_sql",
|
||||
type: "validator_python" | "validator_sql" | "validator_oxc",
|
||||
position?: XYPosition,
|
||||
openDialog?: boolean,
|
||||
) => void;
|
||||
|
|
|
|||
|
|
@ -14,6 +14,10 @@ export type SamplerType =
|
|||
|
||||
export type LlmType = "text" | "structured" | "code" | "judge";
|
||||
export type ValidatorCodeLang =
|
||||
| "javascript"
|
||||
| "typescript"
|
||||
| "jsx"
|
||||
| "tsx"
|
||||
| "python"
|
||||
| "sql:sqlite"
|
||||
| "sql:postgres"
|
||||
|
|
@ -21,6 +25,7 @@ export type ValidatorCodeLang =
|
|||
| "sql:tsql"
|
||||
| "sql:bigquery"
|
||||
| "sql:ansi";
|
||||
export type ValidatorType = "code" | "oxc";
|
||||
|
||||
export type ExpressionDtype = "str" | "int" | "float" | "bool";
|
||||
|
||||
|
|
@ -48,6 +53,7 @@ export type RecipeNodeData = {
|
|||
| LlmType
|
||||
| "validator_python"
|
||||
| "validator_sql"
|
||||
| "validator_oxc"
|
||||
| "expression"
|
||||
| "seed"
|
||||
| "markdown_note"
|
||||
|
|
@ -258,6 +264,8 @@ export type ValidatorConfig = {
|
|||
drop?: boolean;
|
||||
// biome-ignore lint/style/useNamingConvention: api schema
|
||||
target_columns: string[];
|
||||
// ui-only
|
||||
validator_type: ValidatorType;
|
||||
// biome-ignore lint/style/useNamingConvention: api schema
|
||||
code_lang: ValidatorCodeLang;
|
||||
// ui ergonomics (serialized to int in payload)
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import type {
|
|||
SamplerConfig,
|
||||
SamplerType,
|
||||
ValidatorCodeLang,
|
||||
ValidatorType,
|
||||
ValidatorConfig,
|
||||
} from "../types";
|
||||
import { nextName } from "./naming";
|
||||
|
|
@ -279,17 +280,26 @@ export function makeExpressionConfig(
|
|||
|
||||
export function makeValidatorConfig(
|
||||
id: string,
|
||||
validatorType: ValidatorType,
|
||||
codeLang: ValidatorCodeLang,
|
||||
existing: NodeConfig[],
|
||||
): ValidatorConfig {
|
||||
const isSql = codeLang.startsWith("sql:");
|
||||
const isSql = validatorType === "code" && codeLang.startsWith("sql:");
|
||||
const isOxc = validatorType === "oxc";
|
||||
let namePrefix = "validator_python";
|
||||
if (isSql) {
|
||||
namePrefix = "validator_sql";
|
||||
} else if (isOxc) {
|
||||
namePrefix = "validator_oxc";
|
||||
}
|
||||
return {
|
||||
id,
|
||||
kind: "validator",
|
||||
name: nextName(existing, isSql ? "validator_sql" : "validator_python"),
|
||||
name: nextName(existing, namePrefix),
|
||||
drop: false,
|
||||
// biome-ignore lint/style/useNamingConvention: api schema
|
||||
target_columns: [],
|
||||
validator_type: validatorType,
|
||||
// biome-ignore lint/style/useNamingConvention: api schema
|
||||
code_lang: codeLang,
|
||||
batch_size: "10",
|
||||
|
|
|
|||
|
|
@ -15,6 +15,10 @@ import {
|
|||
isLlmConfig,
|
||||
isSubcategoryConfig,
|
||||
} from "../index";
|
||||
import {
|
||||
VALIDATOR_OXC_CODE_LANGS,
|
||||
VALIDATOR_SQL_CODE_LANGS,
|
||||
} from "../validators/code-lang";
|
||||
|
||||
function buildTemplateWithRef(template: string, ref: string): string {
|
||||
if (template.includes(ref)) {
|
||||
|
|
@ -147,6 +151,25 @@ function isModelSemanticRelation(source: NodeConfig, target: NodeConfig): boolea
|
|||
);
|
||||
}
|
||||
|
||||
function canApplyCodeLangToValidator(
|
||||
validator: Extract<NodeConfig, { kind: "validator" }>,
|
||||
codeLang: string,
|
||||
): boolean {
|
||||
const normalized = codeLang.trim();
|
||||
if (!normalized) {
|
||||
return false;
|
||||
}
|
||||
if (validator.validator_type === "oxc") {
|
||||
return VALIDATOR_OXC_CODE_LANGS.includes(
|
||||
normalized as typeof validator.code_lang,
|
||||
);
|
||||
}
|
||||
if (normalized === "python") {
|
||||
return true;
|
||||
}
|
||||
return VALIDATOR_SQL_CODE_LANGS.includes(normalized as typeof validator.code_lang);
|
||||
}
|
||||
|
||||
function countHandleUsage(
|
||||
edges: Edge[],
|
||||
nodeId: string,
|
||||
|
|
@ -366,13 +389,19 @@ export function applyRecipeConnection(
|
|||
target.kind === "validator"
|
||||
) {
|
||||
const nextCodeLang = (source.code_lang ?? "").trim();
|
||||
const canUseCodeLangForTarget = canApplyCodeLangToValidator(
|
||||
target,
|
||||
nextCodeLang,
|
||||
);
|
||||
const next = {
|
||||
...target,
|
||||
// biome-ignore lint/style/useNamingConvention: api schema
|
||||
target_columns: [source.name],
|
||||
// biome-ignore lint/style/useNamingConvention: api schema
|
||||
code_lang:
|
||||
(nextCodeLang || target.code_lang) as typeof target.code_lang,
|
||||
(
|
||||
canUseCodeLangForTarget ? nextCodeLang : target.code_lang
|
||||
) as typeof target.code_lang,
|
||||
};
|
||||
return { edges: nextEdges, configs: { ...configs, [target.id]: next } };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,36 @@ import type { ValidatorConfig } from "../../../types";
|
|||
import { readNumberString } from "../helpers";
|
||||
import { normalizeValidatorCodeLang } from "../../validators/code-lang";
|
||||
|
||||
const OXC_VALIDATION_FN_MARKER = "unsloth_oxc_validator";
|
||||
|
||||
function parseOxcCodeLang(validationFunctionRaw: string): string {
|
||||
if (!validationFunctionRaw.startsWith(OXC_VALIDATION_FN_MARKER)) {
|
||||
return "";
|
||||
}
|
||||
const suffix = validationFunctionRaw
|
||||
.slice(OXC_VALIDATION_FN_MARKER.length)
|
||||
.trim();
|
||||
if (!suffix) {
|
||||
return "";
|
||||
}
|
||||
const normalizedSuffix = suffix.startsWith(":") || suffix.startsWith("_")
|
||||
? suffix.slice(1).trim().toLowerCase()
|
||||
: suffix.toLowerCase();
|
||||
if (normalizedSuffix === "js") {
|
||||
return "javascript";
|
||||
}
|
||||
if (normalizedSuffix === "ts") {
|
||||
return "typescript";
|
||||
}
|
||||
if (normalizedSuffix === "jsx" || normalizedSuffix === "tsx") {
|
||||
return normalizedSuffix;
|
||||
}
|
||||
if (normalizedSuffix === "javascript" || normalizedSuffix === "typescript") {
|
||||
return normalizedSuffix;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
export function parseValidator(
|
||||
column: Record<string, unknown>,
|
||||
name: string,
|
||||
|
|
@ -17,6 +47,14 @@ export function parseValidator(
|
|||
column.validator_params && typeof column.validator_params === "object"
|
||||
? (column.validator_params as Record<string, unknown>)
|
||||
: {};
|
||||
const validationFunctionRaw =
|
||||
typeof params.validation_function === "string"
|
||||
? params.validation_function.trim()
|
||||
: "";
|
||||
const isOxc =
|
||||
String(column.validator_type ?? "").trim() === "local_callable" &&
|
||||
validationFunctionRaw.startsWith(OXC_VALIDATION_FN_MARKER);
|
||||
const oxcLang = isOxc ? parseOxcCodeLang(validationFunctionRaw) : "";
|
||||
return {
|
||||
id,
|
||||
kind: "validator",
|
||||
|
|
@ -24,8 +62,11 @@ export function parseValidator(
|
|||
drop: column.drop === true,
|
||||
// biome-ignore lint/style/useNamingConvention: api schema
|
||||
target_columns: targetColumns,
|
||||
validator_type: isOxc ? "oxc" : "code",
|
||||
// biome-ignore lint/style/useNamingConvention: api schema
|
||||
code_lang: normalizeValidatorCodeLang(params.code_lang),
|
||||
code_lang: normalizeValidatorCodeLang(
|
||||
isOxc ? oxcLang || "javascript" : params.code_lang,
|
||||
),
|
||||
batch_size: readNumberString(column.batch_size) || "10",
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,13 +30,22 @@ export function nodeDataFromConfig(
|
|||
};
|
||||
}
|
||||
if (config.kind === "validator") {
|
||||
const isOxc = config.validator_type === "oxc";
|
||||
const isSql = config.code_lang.startsWith("sql:");
|
||||
let subtype = "Python";
|
||||
let blockType: RecipeNodeData["blockType"] = "validator_python";
|
||||
if (isOxc) {
|
||||
subtype = "OXC";
|
||||
blockType = "validator_oxc";
|
||||
} else if (isSql) {
|
||||
subtype = "SQL";
|
||||
blockType = "validator_sql";
|
||||
}
|
||||
return {
|
||||
title: "Validator",
|
||||
kind: "validator",
|
||||
subtype: config.code_lang.startsWith("sql:") ? "SQL" : "Python",
|
||||
blockType: config.code_lang.startsWith("sql:")
|
||||
? "validator_sql"
|
||||
: "validator_python",
|
||||
subtype,
|
||||
blockType,
|
||||
name: config.name,
|
||||
layoutDirection,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import type { ValidatorConfig } from "../../types";
|
||||
|
||||
const OXC_VALIDATION_FN_MARKER = "unsloth_oxc_validator";
|
||||
|
||||
function parseBatchSize(value: string): number {
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
if (!Number.isFinite(parsed) || parsed < 1) {
|
||||
|
|
@ -18,6 +20,27 @@ export function buildValidatorColumn(
|
|||
if (targetColumns.length === 0) {
|
||||
errors.push(`Validator ${config.name}: target code column required.`);
|
||||
}
|
||||
if (config.validator_type === "oxc") {
|
||||
return {
|
||||
// biome-ignore lint/style/useNamingConvention: api schema
|
||||
column_type: "validation",
|
||||
name: config.name,
|
||||
drop: config.drop ?? false,
|
||||
// biome-ignore lint/style/useNamingConvention: api schema
|
||||
target_columns: targetColumns,
|
||||
// biome-ignore lint/style/useNamingConvention: api schema
|
||||
validator_type: "local_callable",
|
||||
// biome-ignore lint/style/useNamingConvention: api schema
|
||||
validator_params: {
|
||||
// backend resolves this marker to a real callable.
|
||||
// biome-ignore lint/style/useNamingConvention: api schema
|
||||
validation_function: `${OXC_VALIDATION_FN_MARKER}:${config.code_lang}`,
|
||||
},
|
||||
// biome-ignore lint/style/useNamingConvention: api schema
|
||||
batch_size: parseBatchSize(config.batch_size),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
// biome-ignore lint/style/useNamingConvention: api schema
|
||||
column_type: "validation",
|
||||
|
|
|
|||
|
|
@ -2,8 +2,10 @@ import type {
|
|||
ModelConfig,
|
||||
ModelProviderConfig,
|
||||
NodeConfig,
|
||||
ValidatorCodeLang,
|
||||
ValidatorConfig,
|
||||
} from "../../types";
|
||||
import { VALIDATOR_OXC_CODE_LANGS } from "../validators/code-lang";
|
||||
|
||||
export function validateSubcategoryConfigs(
|
||||
configs: Record<string, NodeConfig>,
|
||||
|
|
@ -134,6 +136,17 @@ export function validateValidatorConfigs(
|
|||
errors.push(`Validator ${config.name}: target '${target}' must be LLM Code.`);
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
config.validator_type === "oxc" &&
|
||||
!VALIDATOR_OXC_CODE_LANGS.includes(
|
||||
(targetConfig.code_lang ?? "").trim() as ValidatorCodeLang,
|
||||
)
|
||||
) {
|
||||
errors.push(
|
||||
`Validator ${config.name}: target '${target}' must use javascript/typescript/jsx/tsx.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if ((targetConfig.code_lang ?? "").trim() !== config.code_lang.trim()) {
|
||||
errors.push(
|
||||
`Validator ${config.name}: code_lang '${config.code_lang}' must match target '${target}' (${targetConfig.code_lang ?? "unknown"}).`,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import type { NodeConfig } from "../types";
|
||||
import { isValidSex, parseAgeRange, parseIntNumber, parseNumber } from "./parse";
|
||||
import { VALIDATOR_OXC_CODE_LANGS, VALIDATOR_SQL_CODE_LANGS } from "./validators/code-lang";
|
||||
|
||||
const TRACE_MODES = new Set(["none", "last_message", "all_messages"]);
|
||||
|
||||
|
|
@ -205,6 +206,15 @@ export function getConfigErrors(config: NodeConfig | null): string[] {
|
|||
}
|
||||
if (!config.code_lang.trim()) {
|
||||
errors.push("Validator code language is required.");
|
||||
} else if (config.validator_type === "oxc") {
|
||||
if (!VALIDATOR_OXC_CODE_LANGS.includes(config.code_lang)) {
|
||||
errors.push("OXC validator code language must be javascript/typescript/jsx/tsx.");
|
||||
}
|
||||
} else if (
|
||||
config.code_lang !== "python" &&
|
||||
!VALIDATOR_SQL_CODE_LANGS.includes(config.code_lang)
|
||||
) {
|
||||
errors.push("Code validator code language must be python or sql dialect.");
|
||||
}
|
||||
}
|
||||
if (config.kind === "seed") {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,12 @@
|
|||
import type { ValidatorCodeLang } from "../../types";
|
||||
|
||||
export const VALIDATOR_OXC_CODE_LANGS: ValidatorCodeLang[] = [
|
||||
"javascript",
|
||||
"typescript",
|
||||
"jsx",
|
||||
"tsx",
|
||||
];
|
||||
|
||||
export const VALIDATOR_SQL_CODE_LANGS: ValidatorCodeLang[] = [
|
||||
"sql:sqlite",
|
||||
"sql:postgres",
|
||||
|
|
@ -10,6 +17,7 @@ export const VALIDATOR_SQL_CODE_LANGS: ValidatorCodeLang[] = [
|
|||
];
|
||||
|
||||
const VALIDATOR_CODE_LANG_SET = new Set<ValidatorCodeLang>([
|
||||
...VALIDATOR_OXC_CODE_LANGS,
|
||||
"python",
|
||||
...VALIDATOR_SQL_CODE_LANGS,
|
||||
]);
|
||||
|
|
@ -25,6 +33,9 @@ export function normalizeValidatorCodeLang(
|
|||
if (!raw) {
|
||||
return "python";
|
||||
}
|
||||
if (VALIDATOR_OXC_CODE_LANGS.includes(raw as ValidatorCodeLang)) {
|
||||
return raw as ValidatorCodeLang;
|
||||
}
|
||||
if (raw === "python") {
|
||||
return "python";
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue