diff --git a/docs/canvas-lab-architecture.md b/docs/canvas-lab-architecture.md index 334d96bc7a..dfeef33afc 100644 --- a/docs/canvas-lab-architecture.md +++ b/docs/canvas-lab-architecture.md @@ -17,7 +17,7 @@ This doc explains current architecture, how nodes map to payload/import, and how 4. Graph connection logic updates references + semantic edges: `/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/utils/graph.ts` 5. Export (preview/copy) converts in-memory graph/config to API payload: -`/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/utils/payload.ts` +`/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/utils/payload/build-payload.ts` 6. Import reconstructs configs, nodes, edges from JSON: `/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/utils/import/importer.ts` @@ -192,7 +192,7 @@ This keeps graph fields stable when upstream nodes renamed/deleted. ## 9) Payload building (node graph -> API) File: -`/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/utils/payload.ts` +`/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/utils/payload/build-payload.ts` `buildCanvasPayload(configs, nodes, edges)` outputs: @@ -283,7 +283,7 @@ Minimal path for a new block type: 5. Add store add-action if block should be special-cased from sheet: `/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/stores/canvas-lab.ts` 6. Add payload serialization/validation in: -`/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/utils/payload.ts` +`/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/utils/payload/` 7. Add import parsing + inferred edges in: `/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/utils/import/parsers.ts` `/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/utils/import/importer.ts` diff --git a/studio/frontend/src/features/canvas-lab/utils/payload/build-payload.ts b/studio/frontend/src/features/canvas-lab/utils/payload/build-payload.ts new file mode 100644 index 0000000000..9c58e91c60 --- /dev/null +++ b/studio/frontend/src/features/canvas-lab/utils/payload/build-payload.ts @@ -0,0 +1,154 @@ +import type { Edge } from "@xyflow/react"; +import type { + CanvasProcessorConfig, + CanvasNode, + ModelConfig, + ModelProviderConfig, + NodeConfig, +} from "../../types"; +import { getConfigErrors } from "../index"; +import { + buildExpressionColumn, + buildLlmColumn, + buildModelConfig, + buildModelProvider, + buildProcessors, + buildSamplerColumn, +} from "./builders"; +import type { CanvasPayloadResult } from "./types"; +import { + isSemanticRelation, + validateModelAliasLinks, + validateModelConfigProviders, + validateSubcategoryConfigs, + validateTimedeltaConfigs, + validateUsedProviders, +} from "./validate"; + +// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: payload build +export function buildCanvasPayload( + configs: Record, + nodes: CanvasNode[], + edges: Edge[], + processors: CanvasProcessorConfig[] = [], +): CanvasPayloadResult { + const errors: string[] = []; + const columns: Record[] = []; + const modelAliases = new Set(); + const modelProviderNames = new Set(); + const modelProviders: Record[] = []; + const modelConfigs: Record[] = []; + const modelProviderConfigs: ModelProviderConfig[] = []; + const modelConfigConfigs: ModelConfig[] = []; + const nameSet = new Set(); + const nameToConfig = new Map(); + + for (const node of nodes) { + const config = configs[node.id]; + if (!config) { + continue; + } + for (const error of getConfigErrors(config)) { + errors.push(`${config.name}: ${error}`); + } + if (nameSet.has(config.name)) { + errors.push(`Duplicate node name: ${config.name}.`); + } + nameSet.add(config.name); + + if (config.kind === "sampler") { + nameToConfig.set(config.name, config); + columns.push(buildSamplerColumn(config, errors)); + continue; + } + if (config.kind === "llm") { + columns.push(buildLlmColumn(config, errors)); + if (config.model_alias) { + modelAliases.add(config.model_alias); + } + nameToConfig.set(config.name, config); + continue; + } + if (config.kind === "expression") { + columns.push(buildExpressionColumn(config, errors)); + nameToConfig.set(config.name, config); + continue; + } + if (config.kind === "model_provider") { + modelProviderNames.add(config.name); + modelProviders.push(buildModelProvider(config, errors)); + modelProviderConfigs.push(config); + continue; + } + modelConfigs.push(buildModelConfig(config)); + modelConfigConfigs.push(config); + } + + validateSubcategoryConfigs(configs, nameToConfig, errors); + validateTimedeltaConfigs(configs, nameToConfig, errors); + validateModelAliasLinks(modelAliases, modelConfigConfigs, errors); + validateModelConfigProviders( + modelConfigConfigs, + modelAliases, + modelProviderNames, + errors, + ); + validateUsedProviders(modelProviderConfigs, modelConfigConfigs, errors); + + const uiNodes = nodes.flatMap((node) => { + const config = configs[node.id]; + if (!config) { + return []; + } + return [ + { + id: config.name, + x: node.position.x, + y: node.position.y, + }, + ]; + }); + + const uiEdges = edges.flatMap((edge) => { + const source = edge.source ? configs[edge.source] : null; + const target = edge.target ? configs[edge.target] : null; + if (!(source && target)) { + return []; + } + return [ + { + from: source.name, + to: target.name, + type: + edge.type === "semantic" || isSemanticRelation(source, target) + ? "semantic" + : "canvas", + }, + ]; + }); + const recipeProcessors = buildProcessors(processors, errors); + + return { + errors, + payload: { + recipe: { + // biome-ignore lint/style/useNamingConvention: api schema + model_providers: modelProviders, + // biome-ignore lint/style/useNamingConvention: api schema + model_configs: modelConfigs, + columns, + processors: recipeProcessors, + }, + run: { + rows: 5, + preview: true, + // biome-ignore lint/style/useNamingConvention: api schema + output_formats: ["jsonl"], + }, + ui: { + nodes: uiNodes, + edges: uiEdges, + }, + }, + }; +} diff --git a/studio/frontend/src/features/canvas-lab/utils/payload.ts b/studio/frontend/src/features/canvas-lab/utils/payload/builders.ts similarity index 55% rename from studio/frontend/src/features/canvas-lab/utils/payload.ts rename to studio/frontend/src/features/canvas-lab/utils/payload/builders.ts index 834432b7e2..98e3b3aec6 100644 --- a/studio/frontend/src/features/canvas-lab/utils/payload.ts +++ b/studio/frontend/src/features/canvas-lab/utils/payload/builders.ts @@ -1,97 +1,18 @@ -import type { Edge } from "@xyflow/react"; import type { CanvasProcessorConfig, CategoryConditionalParams, - CanvasNode, ExpressionConfig, LlmConfig, ModelConfig, ModelProviderConfig, - NodeConfig, SamplerConfig, -} from "../types"; -import { getConfigErrors } from "./index"; - -type CanvasPayload = { - recipe: { - // biome-ignore lint/style/useNamingConvention: api schema - model_providers: Record[]; - // biome-ignore lint/style/useNamingConvention: api schema - model_configs: Record[]; - columns: Record[]; - processors: Record[]; - }; - run: { - rows: number; - preview: boolean; - // biome-ignore lint/style/useNamingConvention: api schema - output_formats: string[]; - }; - ui: { - nodes: { id: string; x: number; y: number }[]; - edges: { from: string; to: string; type?: string }[]; - }; -}; - -export type CanvasPayloadResult = { - errors: string[]; - payload: CanvasPayload; -}; - -function isSemanticRelation( - source: NodeConfig, - target: NodeConfig, -): boolean { - if (source.kind === "model_provider" && target.kind === "model_config") { - return true; - } - return source.kind === "model_config" && target.kind === "llm"; -} - -function parseNumber(value?: string): number | null { - if (!value) { - return null; - } - const num = Number(value); - return Number.isFinite(num) ? num : null; -} - -function parseAgeRange(value?: string): [number, number] | null { - if (!value) { - return null; - } - const parts = value.split(/[^0-9.]+/).filter(Boolean); - if (parts.length !== 2) { - return null; - } - const min = Number(parts[0]); - const max = Number(parts[1]); - if (!Number.isFinite(min) || !Number.isFinite(max)) { - return null; - } - return [min, max]; -} - -function parseJsonObject( - value: string | undefined, - label: string, - errors: string[], -): Record | undefined { - if (!value || !value.trim()) { - return undefined; - } - try { - const parsed = JSON.parse(value); - if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { - return parsed as Record; - } - } catch { - errors.push(`${label}: invalid JSON.`); - return undefined; - } - errors.push(`${label}: must be a JSON object.`); - return undefined; -} +} from "../../types"; +import { + isValidSex, + parseAgeRange, + parseJsonObject, + parseNumber, +} from "./parse"; function buildCategoryConditionalParams( config: SamplerConfig, @@ -133,7 +54,7 @@ function buildCategoryConditionalParams( return Object.keys(output).length > 0 ? output : undefined; } -function buildModelProvider( +export function buildModelProvider( config: ModelProviderConfig, errors: string[], ): Record { @@ -163,7 +84,7 @@ function buildModelProvider( }; } -function buildModelConfig(config: ModelConfig): Record { +export function buildModelConfig(config: ModelConfig): Record { const inference: Record = {}; const temp = config.inference_temperature?.trim(); const topP = config.inference_top_p?.trim(); @@ -200,13 +121,6 @@ function buildModelConfig(config: ModelConfig): Record { }; } -function isValidSex(value?: string): value is "Male" | "Female" { - if (!value) { - return false; - } - return value === "Male" || value === "Female"; -} - // biome-ignore lint/complexity/noExcessiveCognitiveComplexity: per type logic function buildSamplerParams( config: SamplerConfig, @@ -311,7 +225,32 @@ function buildSamplerParams( return params; } -function buildLlmColumn( +export function buildSamplerColumn( + config: SamplerConfig, + errors: string[], +): Record { + const samplerColumn: Record = { + // biome-ignore lint/style/useNamingConvention: api schema + column_type: "sampler", + name: config.name, + drop: config.drop ?? false, + // biome-ignore lint/style/useNamingConvention: api schema + sampler_type: config.sampler_type, + params: buildSamplerParams(config, errors), + // biome-ignore lint/style/useNamingConvention: api schema + convert_to: config.convert_to ?? undefined, + }; + if (config.sampler_type === "category") { + const conditionalParams = buildCategoryConditionalParams(config, errors); + if (conditionalParams) { + // biome-ignore lint/style/useNamingConvention: api schema + samplerColumn.conditional_params = conditionalParams; + } + } + return samplerColumn; +} + +export function buildLlmColumn( config: LlmConfig, errors: string[], ): Record { @@ -392,7 +331,7 @@ function buildLlmColumn( }; } -function buildExpressionColumn( +export function buildExpressionColumn( config: ExpressionConfig, errors: string[], ): Record { @@ -409,7 +348,7 @@ function buildExpressionColumn( }; } -function buildProcessors( +export function buildProcessors( processors: CanvasProcessorConfig[], errors: string[], ): Record[] { @@ -442,211 +381,3 @@ function buildProcessors( } return output; } - -// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: payload build -export function buildCanvasPayload( - configs: Record, - nodes: CanvasNode[], - edges: Edge[], - processors: CanvasProcessorConfig[] = [], -): CanvasPayloadResult { - const errors: string[] = []; - const columns: Record[] = []; - const modelAliases = new Set(); - const modelProviderNames = new Set(); - const modelProviders: Record[] = []; - const modelConfigs: Record[] = []; - const modelProviderConfigs: ModelProviderConfig[] = []; - const modelConfigConfigs: ModelConfig[] = []; - const nameSet = new Set(); - const nameToConfig = new Map(); - - for (const node of nodes) { - const config = configs[node.id]; - if (!config) { - continue; - } - for (const error of getConfigErrors(config)) { - errors.push(`${config.name}: ${error}`); - } - if (nameSet.has(config.name)) { - errors.push(`Duplicate node name: ${config.name}.`); - } - nameSet.add(config.name); - - if (config.kind === "sampler") { - nameToConfig.set(config.name, config); - const samplerColumn: Record = { - // biome-ignore lint/style/useNamingConvention: api schema - column_type: "sampler", - name: config.name, - drop: config.drop ?? false, - // biome-ignore lint/style/useNamingConvention: api schema - sampler_type: config.sampler_type, - params: buildSamplerParams(config, errors), - // biome-ignore lint/style/useNamingConvention: api schema - convert_to: config.convert_to ?? undefined, - }; - if (config.sampler_type === "category") { - const conditionalParams = buildCategoryConditionalParams(config, errors); - if (conditionalParams) { - // biome-ignore lint/style/useNamingConvention: api schema - samplerColumn.conditional_params = conditionalParams; - } - } - columns.push(samplerColumn); - } else if (config.kind === "llm") { - columns.push(buildLlmColumn(config, errors)); - if (config.model_alias) { - modelAliases.add(config.model_alias); - } - nameToConfig.set(config.name, config); - } else if (config.kind === "expression") { - columns.push(buildExpressionColumn(config, errors)); - nameToConfig.set(config.name, config); - } else if (config.kind === "model_provider") { - modelProviderNames.add(config.name); - modelProviders.push(buildModelProvider(config, errors)); - modelProviderConfigs.push(config); - } else if (config.kind === "model_config") { - modelConfigs.push(buildModelConfig(config)); - modelConfigConfigs.push(config); - } - } - - for (const config of Object.values(configs)) { - if (config.kind !== "sampler" || config.sampler_type !== "subcategory") { - continue; - } - const parentName = config.subcategory_parent; - if (!parentName) { - errors.push(`Subcategory ${config.name}: parent category required.`); - continue; - } - const parent = nameToConfig.get(parentName); - const parentValues = - parent && parent.kind === "sampler" && parent.sampler_type === "category" - ? (parent.values ?? []) - : []; - const mapping = config.subcategory_mapping ?? {}; - for (const value of parentValues) { - const list = mapping[value]; - if (!list || list.length === 0) { - errors.push( - `Subcategory ${config.name}: '${value}' needs at least 1 subcategory.`, - ); - } - } - } - for (const config of Object.values(configs)) { - if (config.kind !== "sampler" || config.sampler_type !== "timedelta") { - continue; - } - const reference = config.reference_column_name?.trim() ?? ""; - if (!reference) { - errors.push(`Timedelta ${config.name}: reference datetime column required.`); - continue; - } - const parent = nameToConfig.get(reference); - if ( - !parent || - parent.kind !== "sampler" || - parent.sampler_type !== "datetime" - ) { - errors.push(`Timedelta ${config.name}: reference '${reference}' must be datetime.`); - } - } - - for (const alias of modelAliases) { - if ( - !modelConfigs.some( - (config) => (config.alias as string | undefined) === alias, - ) - ) { - errors.push(`LLM model_alias ${alias}: missing model config.`); - } - } - - for (const config of modelConfigConfigs) { - const provider = config.provider.trim(); - const alias = config.name; - if (modelAliases.has(alias) && !config.model.trim()) { - errors.push(`Model config ${alias}: model is required.`); - } - if (provider && !modelProviderNames.has(provider)) { - errors.push(`Model config ${alias}: provider ${provider} not found.`); - } - } - - const usedProviders = new Set( - modelConfigConfigs.map((config) => config.provider.trim()).filter(Boolean), - ); - for (const provider of modelProviderConfigs) { - if (!usedProviders.has(provider.name)) { - continue; - } - if (!provider.endpoint.trim()) { - errors.push(`Model provider ${provider.name}: endpoint is required.`); - } - if (!provider.provider_type.trim()) { - errors.push(`Model provider ${provider.name}: provider_type is required.`); - } - } - - const uiNodes = nodes.flatMap((node) => { - const config = configs[node.id]; - if (!config) { - return []; - } - return [ - { - id: config.name, - x: node.position.x, - y: node.position.y, - }, - ]; - }); - - const uiEdges = edges.flatMap((edge) => { - const source = edge.source ? configs[edge.source] : null; - const target = edge.target ? configs[edge.target] : null; - if (!(source && target)) { - return []; - } - return [ - { - from: source.name, - to: target.name, - type: - edge.type === "semantic" || isSemanticRelation(source, target) - ? "semantic" - : "canvas", - }, - ]; - }); - const recipeProcessors = buildProcessors(processors, errors); - - return { - errors, - payload: { - recipe: { - // biome-ignore lint/style/useNamingConvention: api schema - model_providers: modelProviders, - // biome-ignore lint/style/useNamingConvention: api schema - model_configs: modelConfigs, - columns, - processors: recipeProcessors, - }, - run: { - rows: 5, - preview: true, - // biome-ignore lint/style/useNamingConvention: api schema - output_formats: ["jsonl"], - }, - ui: { - nodes: uiNodes, - edges: uiEdges, - }, - }, - }; -} diff --git a/studio/frontend/src/features/canvas-lab/utils/payload/index.ts b/studio/frontend/src/features/canvas-lab/utils/payload/index.ts new file mode 100644 index 0000000000..c584f4dc10 --- /dev/null +++ b/studio/frontend/src/features/canvas-lab/utils/payload/index.ts @@ -0,0 +1,2 @@ +export { buildCanvasPayload } from "./build-payload"; +export type { CanvasPayload, CanvasPayloadResult } from "./types"; diff --git a/studio/frontend/src/features/canvas-lab/utils/payload/parse.ts b/studio/frontend/src/features/canvas-lab/utils/payload/parse.ts new file mode 100644 index 0000000000..f6ebdceaa2 --- /dev/null +++ b/studio/frontend/src/features/canvas-lab/utils/payload/parse.ts @@ -0,0 +1,51 @@ +export function parseNumber(value?: string): number | null { + if (!value) { + return null; + } + const num = Number(value); + return Number.isFinite(num) ? num : null; +} + +export function parseAgeRange(value?: string): [number, number] | null { + if (!value) { + return null; + } + const parts = value.split(/[^0-9.]+/).filter(Boolean); + if (parts.length !== 2) { + return null; + } + const min = Number(parts[0]); + const max = Number(parts[1]); + if (!Number.isFinite(min) || !Number.isFinite(max)) { + return null; + } + return [min, max]; +} + +export function parseJsonObject( + value: string | undefined, + label: string, + errors: string[], +): Record | undefined { + if (!value || !value.trim()) { + return undefined; + } + try { + const parsed = JSON.parse(value); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + return parsed as Record; + } + } catch { + errors.push(`${label}: invalid JSON.`); + return undefined; + } + errors.push(`${label}: must be a JSON object.`); + return undefined; +} + +export function isValidSex(value?: string): value is "Male" | "Female" { + if (!value) { + return false; + } + return value === "Male" || value === "Female"; +} diff --git a/studio/frontend/src/features/canvas-lab/utils/payload/types.ts b/studio/frontend/src/features/canvas-lab/utils/payload/types.ts new file mode 100644 index 0000000000..ab37d1c943 --- /dev/null +++ b/studio/frontend/src/features/canvas-lab/utils/payload/types.ts @@ -0,0 +1,25 @@ +export type CanvasPayload = { + recipe: { + // biome-ignore lint/style/useNamingConvention: api schema + model_providers: Record[]; + // biome-ignore lint/style/useNamingConvention: api schema + model_configs: Record[]; + columns: Record[]; + processors: Record[]; + }; + run: { + rows: number; + preview: boolean; + // biome-ignore lint/style/useNamingConvention: api schema + output_formats: string[]; + }; + ui: { + nodes: { id: string; x: number; y: number }[]; + edges: { from: string; to: string; type?: string }[]; + }; +}; + +export type CanvasPayloadResult = { + errors: string[]; + payload: CanvasPayload; +}; diff --git a/studio/frontend/src/features/canvas-lab/utils/payload/validate.ts b/studio/frontend/src/features/canvas-lab/utils/payload/validate.ts new file mode 100644 index 0000000000..f7be3caf22 --- /dev/null +++ b/studio/frontend/src/features/canvas-lab/utils/payload/validate.ts @@ -0,0 +1,118 @@ +import type { ModelConfig, ModelProviderConfig, NodeConfig } from "../../types"; + +export function isSemanticRelation( + source: NodeConfig, + target: NodeConfig, +): boolean { + if (source.kind === "model_provider" && target.kind === "model_config") { + return true; + } + return source.kind === "model_config" && target.kind === "llm"; +} + +export function validateSubcategoryConfigs( + configs: Record, + nameToConfig: Map, + errors: string[], +): void { + for (const config of Object.values(configs)) { + if (config.kind !== "sampler" || config.sampler_type !== "subcategory") { + continue; + } + const parentName = config.subcategory_parent; + if (!parentName) { + errors.push(`Subcategory ${config.name}: parent category required.`); + continue; + } + const parent = nameToConfig.get(parentName); + const parentValues = + parent && parent.kind === "sampler" && parent.sampler_type === "category" + ? (parent.values ?? []) + : []; + const mapping = config.subcategory_mapping ?? {}; + for (const value of parentValues) { + const list = mapping[value]; + if (!list || list.length === 0) { + errors.push( + `Subcategory ${config.name}: '${value}' needs at least 1 subcategory.`, + ); + } + } + } +} + +export function validateTimedeltaConfigs( + configs: Record, + nameToConfig: Map, + errors: string[], +): void { + for (const config of Object.values(configs)) { + if (config.kind !== "sampler" || config.sampler_type !== "timedelta") { + continue; + } + const reference = config.reference_column_name?.trim() ?? ""; + if (!reference) { + errors.push(`Timedelta ${config.name}: reference datetime column required.`); + continue; + } + const parent = nameToConfig.get(reference); + if ( + !parent || + parent.kind !== "sampler" || + parent.sampler_type !== "datetime" + ) { + errors.push(`Timedelta ${config.name}: reference '${reference}' must be datetime.`); + } + } +} + +export function validateModelAliasLinks( + modelAliases: Set, + modelConfigConfigs: ModelConfig[], + errors: string[], +): void { + for (const alias of modelAliases) { + if (!modelConfigConfigs.some((config) => config.name === alias)) { + errors.push(`LLM model_alias ${alias}: missing model config.`); + } + } +} + +export function validateModelConfigProviders( + modelConfigConfigs: ModelConfig[], + modelAliases: Set, + modelProviderNames: Set, + errors: string[], +): void { + for (const config of modelConfigConfigs) { + const provider = config.provider.trim(); + const alias = config.name; + if (modelAliases.has(alias) && !config.model.trim()) { + errors.push(`Model config ${alias}: model is required.`); + } + if (provider && !modelProviderNames.has(provider)) { + errors.push(`Model config ${alias}: provider ${provider} not found.`); + } + } +} + +export function validateUsedProviders( + modelProviderConfigs: ModelProviderConfig[], + modelConfigConfigs: ModelConfig[], + errors: string[], +): void { + const usedProviders = new Set( + modelConfigConfigs.map((config) => config.provider.trim()).filter(Boolean), + ); + for (const provider of modelProviderConfigs) { + if (!usedProviders.has(provider.name)) { + continue; + } + if (!provider.endpoint.trim()) { + errors.push(`Model provider ${provider.name}: endpoint is required.`); + } + if (!provider.provider_type.trim()) { + errors.push(`Model provider ${provider.name}: provider_type is required.`); + } + } +}