refactor of payload files

This commit is contained in:
shine1i 2026-02-05 23:41:27 +01:00
commit 271ddcfb4a
7 changed files with 390 additions and 309 deletions

View file

@ -17,7 +17,7 @@ This doc explains current architecture, how nodes map to payload/import, and how
4. Graph connection logic updates references + semantic edges:
`/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/utils/graph.ts`
5. Export (preview/copy) converts in-memory graph/config to API payload:
`/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/utils/payload.ts`
`/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/utils/payload/build-payload.ts`
6. Import reconstructs configs, nodes, edges from JSON:
`/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/utils/import/importer.ts`
@ -192,7 +192,7 @@ This keeps graph fields stable when upstream nodes renamed/deleted.
## 9) Payload building (node graph -> API)
File:
`/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/utils/payload.ts`
`/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/utils/payload/build-payload.ts`
`buildCanvasPayload(configs, nodes, edges)` outputs:
@ -283,7 +283,7 @@ Minimal path for a new block type:
5. Add store add-action if block should be special-cased from sheet:
`/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/stores/canvas-lab.ts`
6. Add payload serialization/validation in:
`/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/utils/payload.ts`
`/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/utils/payload/`
7. Add import parsing + inferred edges in:
`/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/utils/import/parsers.ts`
`/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/utils/import/importer.ts`

View file

@ -0,0 +1,154 @@
import type { Edge } from "@xyflow/react";
import type {
CanvasProcessorConfig,
CanvasNode,
ModelConfig,
ModelProviderConfig,
NodeConfig,
} from "../../types";
import { getConfigErrors } from "../index";
import {
buildExpressionColumn,
buildLlmColumn,
buildModelConfig,
buildModelProvider,
buildProcessors,
buildSamplerColumn,
} from "./builders";
import type { CanvasPayloadResult } from "./types";
import {
isSemanticRelation,
validateModelAliasLinks,
validateModelConfigProviders,
validateSubcategoryConfigs,
validateTimedeltaConfigs,
validateUsedProviders,
} from "./validate";
// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: payload build
export function buildCanvasPayload(
configs: Record<string, NodeConfig>,
nodes: CanvasNode[],
edges: Edge[],
processors: CanvasProcessorConfig[] = [],
): CanvasPayloadResult {
const errors: string[] = [];
const columns: Record<string, unknown>[] = [];
const modelAliases = new Set<string>();
const modelProviderNames = new Set<string>();
const modelProviders: Record<string, unknown>[] = [];
const modelConfigs: Record<string, unknown>[] = [];
const modelProviderConfigs: ModelProviderConfig[] = [];
const modelConfigConfigs: ModelConfig[] = [];
const nameSet = new Set<string>();
const nameToConfig = new Map<string, NodeConfig>();
for (const node of nodes) {
const config = configs[node.id];
if (!config) {
continue;
}
for (const error of getConfigErrors(config)) {
errors.push(`${config.name}: ${error}`);
}
if (nameSet.has(config.name)) {
errors.push(`Duplicate node name: ${config.name}.`);
}
nameSet.add(config.name);
if (config.kind === "sampler") {
nameToConfig.set(config.name, config);
columns.push(buildSamplerColumn(config, errors));
continue;
}
if (config.kind === "llm") {
columns.push(buildLlmColumn(config, errors));
if (config.model_alias) {
modelAliases.add(config.model_alias);
}
nameToConfig.set(config.name, config);
continue;
}
if (config.kind === "expression") {
columns.push(buildExpressionColumn(config, errors));
nameToConfig.set(config.name, config);
continue;
}
if (config.kind === "model_provider") {
modelProviderNames.add(config.name);
modelProviders.push(buildModelProvider(config, errors));
modelProviderConfigs.push(config);
continue;
}
modelConfigs.push(buildModelConfig(config));
modelConfigConfigs.push(config);
}
validateSubcategoryConfigs(configs, nameToConfig, errors);
validateTimedeltaConfigs(configs, nameToConfig, errors);
validateModelAliasLinks(modelAliases, modelConfigConfigs, errors);
validateModelConfigProviders(
modelConfigConfigs,
modelAliases,
modelProviderNames,
errors,
);
validateUsedProviders(modelProviderConfigs, modelConfigConfigs, errors);
const uiNodes = nodes.flatMap((node) => {
const config = configs[node.id];
if (!config) {
return [];
}
return [
{
id: config.name,
x: node.position.x,
y: node.position.y,
},
];
});
const uiEdges = edges.flatMap((edge) => {
const source = edge.source ? configs[edge.source] : null;
const target = edge.target ? configs[edge.target] : null;
if (!(source && target)) {
return [];
}
return [
{
from: source.name,
to: target.name,
type:
edge.type === "semantic" || isSemanticRelation(source, target)
? "semantic"
: "canvas",
},
];
});
const recipeProcessors = buildProcessors(processors, errors);
return {
errors,
payload: {
recipe: {
// biome-ignore lint/style/useNamingConvention: api schema
model_providers: modelProviders,
// biome-ignore lint/style/useNamingConvention: api schema
model_configs: modelConfigs,
columns,
processors: recipeProcessors,
},
run: {
rows: 5,
preview: true,
// biome-ignore lint/style/useNamingConvention: api schema
output_formats: ["jsonl"],
},
ui: {
nodes: uiNodes,
edges: uiEdges,
},
},
};
}

View file

@ -1,97 +1,18 @@
import type { Edge } from "@xyflow/react";
import type {
CanvasProcessorConfig,
CategoryConditionalParams,
CanvasNode,
ExpressionConfig,
LlmConfig,
ModelConfig,
ModelProviderConfig,
NodeConfig,
SamplerConfig,
} from "../types";
import { getConfigErrors } from "./index";
type CanvasPayload = {
recipe: {
// biome-ignore lint/style/useNamingConvention: api schema
model_providers: Record<string, unknown>[];
// biome-ignore lint/style/useNamingConvention: api schema
model_configs: Record<string, unknown>[];
columns: Record<string, unknown>[];
processors: Record<string, unknown>[];
};
run: {
rows: number;
preview: boolean;
// biome-ignore lint/style/useNamingConvention: api schema
output_formats: string[];
};
ui: {
nodes: { id: string; x: number; y: number }[];
edges: { from: string; to: string; type?: string }[];
};
};
export type CanvasPayloadResult = {
errors: string[];
payload: CanvasPayload;
};
function isSemanticRelation(
source: NodeConfig,
target: NodeConfig,
): boolean {
if (source.kind === "model_provider" && target.kind === "model_config") {
return true;
}
return source.kind === "model_config" && target.kind === "llm";
}
function parseNumber(value?: string): number | null {
if (!value) {
return null;
}
const num = Number(value);
return Number.isFinite(num) ? num : null;
}
function parseAgeRange(value?: string): [number, number] | null {
if (!value) {
return null;
}
const parts = value.split(/[^0-9.]+/).filter(Boolean);
if (parts.length !== 2) {
return null;
}
const min = Number(parts[0]);
const max = Number(parts[1]);
if (!Number.isFinite(min) || !Number.isFinite(max)) {
return null;
}
return [min, max];
}
function parseJsonObject(
value: string | undefined,
label: string,
errors: string[],
): Record<string, unknown> | undefined {
if (!value || !value.trim()) {
return undefined;
}
try {
const parsed = JSON.parse(value);
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
return parsed as Record<string, unknown>;
}
} catch {
errors.push(`${label}: invalid JSON.`);
return undefined;
}
errors.push(`${label}: must be a JSON object.`);
return undefined;
}
} from "../../types";
import {
isValidSex,
parseAgeRange,
parseJsonObject,
parseNumber,
} from "./parse";
function buildCategoryConditionalParams(
config: SamplerConfig,
@ -133,7 +54,7 @@ function buildCategoryConditionalParams(
return Object.keys(output).length > 0 ? output : undefined;
}
function buildModelProvider(
export function buildModelProvider(
config: ModelProviderConfig,
errors: string[],
): Record<string, unknown> {
@ -163,7 +84,7 @@ function buildModelProvider(
};
}
function buildModelConfig(config: ModelConfig): Record<string, unknown> {
export function buildModelConfig(config: ModelConfig): Record<string, unknown> {
const inference: Record<string, unknown> = {};
const temp = config.inference_temperature?.trim();
const topP = config.inference_top_p?.trim();
@ -200,13 +121,6 @@ function buildModelConfig(config: ModelConfig): Record<string, unknown> {
};
}
function isValidSex(value?: string): value is "Male" | "Female" {
if (!value) {
return false;
}
return value === "Male" || value === "Female";
}
// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: per type logic
function buildSamplerParams(
config: SamplerConfig,
@ -311,7 +225,32 @@ function buildSamplerParams(
return params;
}
function buildLlmColumn(
export function buildSamplerColumn(
config: SamplerConfig,
errors: string[],
): Record<string, unknown> {
const samplerColumn: Record<string, unknown> = {
// biome-ignore lint/style/useNamingConvention: api schema
column_type: "sampler",
name: config.name,
drop: config.drop ?? false,
// biome-ignore lint/style/useNamingConvention: api schema
sampler_type: config.sampler_type,
params: buildSamplerParams(config, errors),
// biome-ignore lint/style/useNamingConvention: api schema
convert_to: config.convert_to ?? undefined,
};
if (config.sampler_type === "category") {
const conditionalParams = buildCategoryConditionalParams(config, errors);
if (conditionalParams) {
// biome-ignore lint/style/useNamingConvention: api schema
samplerColumn.conditional_params = conditionalParams;
}
}
return samplerColumn;
}
export function buildLlmColumn(
config: LlmConfig,
errors: string[],
): Record<string, unknown> {
@ -392,7 +331,7 @@ function buildLlmColumn(
};
}
function buildExpressionColumn(
export function buildExpressionColumn(
config: ExpressionConfig,
errors: string[],
): Record<string, unknown> {
@ -409,7 +348,7 @@ function buildExpressionColumn(
};
}
function buildProcessors(
export function buildProcessors(
processors: CanvasProcessorConfig[],
errors: string[],
): Record<string, unknown>[] {
@ -442,211 +381,3 @@ function buildProcessors(
}
return output;
}
// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: payload build
export function buildCanvasPayload(
configs: Record<string, NodeConfig>,
nodes: CanvasNode[],
edges: Edge[],
processors: CanvasProcessorConfig[] = [],
): CanvasPayloadResult {
const errors: string[] = [];
const columns: Record<string, unknown>[] = [];
const modelAliases = new Set<string>();
const modelProviderNames = new Set<string>();
const modelProviders: Record<string, unknown>[] = [];
const modelConfigs: Record<string, unknown>[] = [];
const modelProviderConfigs: ModelProviderConfig[] = [];
const modelConfigConfigs: ModelConfig[] = [];
const nameSet = new Set<string>();
const nameToConfig = new Map<string, NodeConfig>();
for (const node of nodes) {
const config = configs[node.id];
if (!config) {
continue;
}
for (const error of getConfigErrors(config)) {
errors.push(`${config.name}: ${error}`);
}
if (nameSet.has(config.name)) {
errors.push(`Duplicate node name: ${config.name}.`);
}
nameSet.add(config.name);
if (config.kind === "sampler") {
nameToConfig.set(config.name, config);
const samplerColumn: Record<string, unknown> = {
// biome-ignore lint/style/useNamingConvention: api schema
column_type: "sampler",
name: config.name,
drop: config.drop ?? false,
// biome-ignore lint/style/useNamingConvention: api schema
sampler_type: config.sampler_type,
params: buildSamplerParams(config, errors),
// biome-ignore lint/style/useNamingConvention: api schema
convert_to: config.convert_to ?? undefined,
};
if (config.sampler_type === "category") {
const conditionalParams = buildCategoryConditionalParams(config, errors);
if (conditionalParams) {
// biome-ignore lint/style/useNamingConvention: api schema
samplerColumn.conditional_params = conditionalParams;
}
}
columns.push(samplerColumn);
} else if (config.kind === "llm") {
columns.push(buildLlmColumn(config, errors));
if (config.model_alias) {
modelAliases.add(config.model_alias);
}
nameToConfig.set(config.name, config);
} else if (config.kind === "expression") {
columns.push(buildExpressionColumn(config, errors));
nameToConfig.set(config.name, config);
} else if (config.kind === "model_provider") {
modelProviderNames.add(config.name);
modelProviders.push(buildModelProvider(config, errors));
modelProviderConfigs.push(config);
} else if (config.kind === "model_config") {
modelConfigs.push(buildModelConfig(config));
modelConfigConfigs.push(config);
}
}
for (const config of Object.values(configs)) {
if (config.kind !== "sampler" || config.sampler_type !== "subcategory") {
continue;
}
const parentName = config.subcategory_parent;
if (!parentName) {
errors.push(`Subcategory ${config.name}: parent category required.`);
continue;
}
const parent = nameToConfig.get(parentName);
const parentValues =
parent && parent.kind === "sampler" && parent.sampler_type === "category"
? (parent.values ?? [])
: [];
const mapping = config.subcategory_mapping ?? {};
for (const value of parentValues) {
const list = mapping[value];
if (!list || list.length === 0) {
errors.push(
`Subcategory ${config.name}: '${value}' needs at least 1 subcategory.`,
);
}
}
}
for (const config of Object.values(configs)) {
if (config.kind !== "sampler" || config.sampler_type !== "timedelta") {
continue;
}
const reference = config.reference_column_name?.trim() ?? "";
if (!reference) {
errors.push(`Timedelta ${config.name}: reference datetime column required.`);
continue;
}
const parent = nameToConfig.get(reference);
if (
!parent ||
parent.kind !== "sampler" ||
parent.sampler_type !== "datetime"
) {
errors.push(`Timedelta ${config.name}: reference '${reference}' must be datetime.`);
}
}
for (const alias of modelAliases) {
if (
!modelConfigs.some(
(config) => (config.alias as string | undefined) === alias,
)
) {
errors.push(`LLM model_alias ${alias}: missing model config.`);
}
}
for (const config of modelConfigConfigs) {
const provider = config.provider.trim();
const alias = config.name;
if (modelAliases.has(alias) && !config.model.trim()) {
errors.push(`Model config ${alias}: model is required.`);
}
if (provider && !modelProviderNames.has(provider)) {
errors.push(`Model config ${alias}: provider ${provider} not found.`);
}
}
const usedProviders = new Set(
modelConfigConfigs.map((config) => config.provider.trim()).filter(Boolean),
);
for (const provider of modelProviderConfigs) {
if (!usedProviders.has(provider.name)) {
continue;
}
if (!provider.endpoint.trim()) {
errors.push(`Model provider ${provider.name}: endpoint is required.`);
}
if (!provider.provider_type.trim()) {
errors.push(`Model provider ${provider.name}: provider_type is required.`);
}
}
const uiNodes = nodes.flatMap((node) => {
const config = configs[node.id];
if (!config) {
return [];
}
return [
{
id: config.name,
x: node.position.x,
y: node.position.y,
},
];
});
const uiEdges = edges.flatMap((edge) => {
const source = edge.source ? configs[edge.source] : null;
const target = edge.target ? configs[edge.target] : null;
if (!(source && target)) {
return [];
}
return [
{
from: source.name,
to: target.name,
type:
edge.type === "semantic" || isSemanticRelation(source, target)
? "semantic"
: "canvas",
},
];
});
const recipeProcessors = buildProcessors(processors, errors);
return {
errors,
payload: {
recipe: {
// biome-ignore lint/style/useNamingConvention: api schema
model_providers: modelProviders,
// biome-ignore lint/style/useNamingConvention: api schema
model_configs: modelConfigs,
columns,
processors: recipeProcessors,
},
run: {
rows: 5,
preview: true,
// biome-ignore lint/style/useNamingConvention: api schema
output_formats: ["jsonl"],
},
ui: {
nodes: uiNodes,
edges: uiEdges,
},
},
};
}

View file

@ -0,0 +1,2 @@
export { buildCanvasPayload } from "./build-payload";
export type { CanvasPayload, CanvasPayloadResult } from "./types";

View file

@ -0,0 +1,51 @@
export function parseNumber(value?: string): number | null {
if (!value) {
return null;
}
const num = Number(value);
return Number.isFinite(num) ? num : null;
}
export function parseAgeRange(value?: string): [number, number] | null {
if (!value) {
return null;
}
const parts = value.split(/[^0-9.]+/).filter(Boolean);
if (parts.length !== 2) {
return null;
}
const min = Number(parts[0]);
const max = Number(parts[1]);
if (!Number.isFinite(min) || !Number.isFinite(max)) {
return null;
}
return [min, max];
}
export function parseJsonObject(
value: string | undefined,
label: string,
errors: string[],
): Record<string, unknown> | undefined {
if (!value || !value.trim()) {
return undefined;
}
try {
const parsed = JSON.parse(value);
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
return parsed as Record<string, unknown>;
}
} catch {
errors.push(`${label}: invalid JSON.`);
return undefined;
}
errors.push(`${label}: must be a JSON object.`);
return undefined;
}
export function isValidSex(value?: string): value is "Male" | "Female" {
if (!value) {
return false;
}
return value === "Male" || value === "Female";
}

View file

@ -0,0 +1,25 @@
export type CanvasPayload = {
recipe: {
// biome-ignore lint/style/useNamingConvention: api schema
model_providers: Record<string, unknown>[];
// biome-ignore lint/style/useNamingConvention: api schema
model_configs: Record<string, unknown>[];
columns: Record<string, unknown>[];
processors: Record<string, unknown>[];
};
run: {
rows: number;
preview: boolean;
// biome-ignore lint/style/useNamingConvention: api schema
output_formats: string[];
};
ui: {
nodes: { id: string; x: number; y: number }[];
edges: { from: string; to: string; type?: string }[];
};
};
export type CanvasPayloadResult = {
errors: string[];
payload: CanvasPayload;
};

View file

@ -0,0 +1,118 @@
import type { ModelConfig, ModelProviderConfig, NodeConfig } from "../../types";
export function isSemanticRelation(
source: NodeConfig,
target: NodeConfig,
): boolean {
if (source.kind === "model_provider" && target.kind === "model_config") {
return true;
}
return source.kind === "model_config" && target.kind === "llm";
}
export function validateSubcategoryConfigs(
configs: Record<string, NodeConfig>,
nameToConfig: Map<string, NodeConfig>,
errors: string[],
): void {
for (const config of Object.values(configs)) {
if (config.kind !== "sampler" || config.sampler_type !== "subcategory") {
continue;
}
const parentName = config.subcategory_parent;
if (!parentName) {
errors.push(`Subcategory ${config.name}: parent category required.`);
continue;
}
const parent = nameToConfig.get(parentName);
const parentValues =
parent && parent.kind === "sampler" && parent.sampler_type === "category"
? (parent.values ?? [])
: [];
const mapping = config.subcategory_mapping ?? {};
for (const value of parentValues) {
const list = mapping[value];
if (!list || list.length === 0) {
errors.push(
`Subcategory ${config.name}: '${value}' needs at least 1 subcategory.`,
);
}
}
}
}
export function validateTimedeltaConfigs(
configs: Record<string, NodeConfig>,
nameToConfig: Map<string, NodeConfig>,
errors: string[],
): void {
for (const config of Object.values(configs)) {
if (config.kind !== "sampler" || config.sampler_type !== "timedelta") {
continue;
}
const reference = config.reference_column_name?.trim() ?? "";
if (!reference) {
errors.push(`Timedelta ${config.name}: reference datetime column required.`);
continue;
}
const parent = nameToConfig.get(reference);
if (
!parent ||
parent.kind !== "sampler" ||
parent.sampler_type !== "datetime"
) {
errors.push(`Timedelta ${config.name}: reference '${reference}' must be datetime.`);
}
}
}
export function validateModelAliasLinks(
modelAliases: Set<string>,
modelConfigConfigs: ModelConfig[],
errors: string[],
): void {
for (const alias of modelAliases) {
if (!modelConfigConfigs.some((config) => config.name === alias)) {
errors.push(`LLM model_alias ${alias}: missing model config.`);
}
}
}
export function validateModelConfigProviders(
modelConfigConfigs: ModelConfig[],
modelAliases: Set<string>,
modelProviderNames: Set<string>,
errors: string[],
): void {
for (const config of modelConfigConfigs) {
const provider = config.provider.trim();
const alias = config.name;
if (modelAliases.has(alias) && !config.model.trim()) {
errors.push(`Model config ${alias}: model is required.`);
}
if (provider && !modelProviderNames.has(provider)) {
errors.push(`Model config ${alias}: provider ${provider} not found.`);
}
}
}
export function validateUsedProviders(
modelProviderConfigs: ModelProviderConfig[],
modelConfigConfigs: ModelConfig[],
errors: string[],
): void {
const usedProviders = new Set(
modelConfigConfigs.map((config) => config.provider.trim()).filter(Boolean),
);
for (const provider of modelProviderConfigs) {
if (!usedProviders.has(provider.name)) {
continue;
}
if (!provider.endpoint.trim()) {
errors.push(`Model provider ${provider.name}: endpoint is required.`);
}
if (!provider.provider_type.trim()) {
errors.push(`Model provider ${provider.name}: provider_type is required.`);
}
}
}