From 1ca01e5d21dd9abd88bb2db70030f2dfb932b4a3 Mon Sep 17 00:00:00 2001 From: shine1i Date: Thu, 5 Feb 2026 22:26:48 +0100 Subject: [PATCH] processors and drop column --- docs/canvas-lab-architecture.md | 45 +++++- .../features/canvas-lab/canvas-lab-page.tsx | 40 +++++- .../canvas-lab/components/block-sheet.tsx | 41 +++++- .../canvas-lab/dialogs/config-dialog.tsx | 17 +++ .../canvas-lab/dialogs/processors-dialog.tsx | 134 ++++++++++++++++++ .../features/canvas-lab/stores/canvas-lab.ts | 8 +- .../src/features/canvas-lab/types/index.ts | 13 ++ .../canvas-lab/utils/import/importer.ts | 38 ++++- .../canvas-lab/utils/import/parsers.ts | 12 ++ .../features/canvas-lab/utils/import/types.ts | 7 +- .../src/features/canvas-lab/utils/index.ts | 12 ++ .../src/features/canvas-lab/utils/payload.ts | 42 +++++- .../features/canvas-lab/utils/processors.ts | 11 ++ 13 files changed, 410 insertions(+), 10 deletions(-) create mode 100644 studio/frontend/src/features/canvas-lab/dialogs/processors-dialog.tsx create mode 100644 studio/frontend/src/features/canvas-lab/utils/processors.ts diff --git a/docs/canvas-lab-architecture.md b/docs/canvas-lab-architecture.md index 9935570089..334d96bc7a 100644 --- a/docs/canvas-lab-architecture.md +++ b/docs/canvas-lab-architecture.md @@ -202,13 +202,21 @@ File: model_providers: [...], model_configs: [...], columns: [...], - processors: [], + processors: [...], }, run: { rows: 5, preview: true, output_formats: ["jsonl"] }, ui: { nodes: [...], edges: [...] } } ``` +Current processor UI surface: +- `schema_transform` (sheet -> `Processors` -> `Schema Transform`) +- mapped to recipe processor with `build_stage: "post_batch"` and JSON `template`. + +Current drop policy: +- column dialogs (`sampler` / `llm` / `expression`) expose `drop` toggle. +- payload writes column `drop` directly (preferred over drop-columns processor in v1). + How relation is enforced: - collect `model_alias` values used by LLM columns - ensure each alias exists in `recipe.model_configs` @@ -253,6 +261,8 @@ Model dialogs: `/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/dialogs/models/model-provider-dialog.tsx` - model config: `/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/dialogs/models/model-config-dialog.tsx` +- processors: +`/Volumes/Expansion/projects/new-ui-prototype/studio/frontend/src/features/canvas-lab/dialogs/processors-dialog.tsx` `ModelConfigDialog` and `LlmDialog` use shadcn `Combobox` fed from store configs: - model config `provider` suggests model-provider node names @@ -291,3 +301,36 @@ Minimal path for a new block type: - payload/import utils = external contract boundary. If one piece changes, keep all six in sync. + +## 14) Processors roadmap (decision) + +Current decision: **Option 3 (hybrid)**. + +v1 scope: +- add `drop` toggle on column blocks (sampler/llm/expression). +- add processor config surface for `schema_transform`. +- keep payload builder as single mapper to `recipe.processors`. +- keep processor state separate from node graph for now. + +Reason: +- fastest ship path. +- matches Data Designer column-level `drop`. +- avoids duplicate/complex processor edge logic in v1. + +Future option noted: **Option 2 (processor chain in graph)**. + +Option 2 structure: +- add virtual node `Dataset Output`. +- processors become graph nodes: `Schema Transform`, `Drop Columns`, future processors. +- processor order derived from chain edges: + `Dataset Output -> P1 -> P2 -> ...` +- enforce chain rules: + - no cycles + - one incoming max per processor + - one outgoing max per processor + - chain must start at `Dataset Output` + +Migration from option 3 -> 2: +- keep same processor schema/payload contracts. +- move order source from list/order field to edge traversal. +- UI changes mostly in canvas rendering + validation; payload adapter stays mostly same. diff --git a/studio/frontend/src/features/canvas-lab/canvas-lab-page.tsx b/studio/frontend/src/features/canvas-lab/canvas-lab-page.tsx index a501b3e1f7..8f54f594f1 100644 --- a/studio/frontend/src/features/canvas-lab/canvas-lab-page.tsx +++ b/studio/frontend/src/features/canvas-lab/canvas-lab-page.tsx @@ -22,11 +22,13 @@ import { CanvasEdge } from "./components/canvas-edge"; import { CanvasNode } from "./components/canvas-node"; import { ConfigDialog } from "./dialogs/config-dialog"; import { ImportDialog } from "./dialogs/import-dialog"; +import { ProcessorsDialog } from "./dialogs/processors-dialog"; import { useCanvasLabStore } from "./stores/canvas-lab"; import type { CanvasNodeData, SamplerConfig } from "./types"; import { isCategoryConfig } from "./utils"; import { importCanvasPayload } from "./utils/import"; import { buildCanvasPayload } from "./utils/payload"; +import { buildDefaultSchemaTransform } from "./utils/processors"; const NODE_TYPES: NodeTypes = { builder: CanvasNode }; const EDGE_TYPES: EdgeTypes = { canvas: CanvasEdge, semantic: CanvasEdge }; @@ -68,6 +70,7 @@ export function CanvasLabPage(): ReactElement { nodes, edges, configs, + processors, sheetView, activeConfigId, dialogOpen, @@ -84,6 +87,7 @@ export function CanvasLabPage(): ReactElement { updateConfig, isValidConnection, setSheetView, + setProcessors, setDialogOpen, loadCanvas, setLayoutDirection, @@ -93,6 +97,7 @@ export function CanvasLabPage(): ReactElement { nodes: state.nodes, edges: state.edges, configs: state.configs, + processors: state.processors, sheetView: state.sheetView, activeConfigId: state.activeConfigId, dialogOpen: state.dialogOpen, @@ -109,6 +114,7 @@ export function CanvasLabPage(): ReactElement { updateConfig: state.updateConfig, isValidConnection: state.isValidConnection, setSheetView: state.setSheetView, + setProcessors: state.setProcessors, setDialogOpen: state.setDialogOpen, loadCanvas: state.loadCanvas, setLayoutDirection: state.setLayoutDirection, @@ -120,6 +126,7 @@ export function CanvasLabPage(): ReactElement { ); const [previewLoading, setPreviewLoading] = useState(false); const [importOpen, setImportOpen] = useState(false); + const [processorsOpen, setProcessorsOpen] = useState(false); const [statusMessage, setStatusMessage] = useState<{ tone: "success" | "error"; text: string; @@ -146,7 +153,12 @@ export function CanvasLabPage(): ReactElement { setPreviewLoading(true); setStatusMessage(null); try { - const { payload, errors } = buildCanvasPayload(configs, nodes, edges); + const { payload, errors } = buildCanvasPayload( + configs, + nodes, + edges, + processors, + ); if (errors.length > 0) { setStatusMessage({ tone: "error", @@ -172,7 +184,12 @@ export function CanvasLabPage(): ReactElement { const handleCopyRecipe = async (): Promise => { setStatusMessage(null); - const { payload, errors } = buildCanvasPayload(configs, nodes, edges); + const { payload, errors } = buildCanvasPayload( + configs, + nodes, + edges, + processors, + ); if (errors.length > 0) { setStatusMessage({ tone: "error", @@ -211,6 +228,17 @@ export function CanvasLabPage(): ReactElement { return null; }; + const handleOpenProcessorsFromSheet = useCallback(() => { + if ( + !processors.some( + (processor) => processor.processor_type === "schema_transform", + ) + ) { + setProcessors([...processors, buildDefaultSchemaTransform()]); + } + setProcessorsOpen(true); + }, [processors, setProcessors]); + return (
@@ -313,6 +341,7 @@ export function CanvasLabPage(): ReactElement { onAddModelProvider={addModelProviderNode} onAddModelConfig={addModelConfigNode} onAddExpression={addExpressionNode} + onOpenProcessors={handleOpenProcessorsFromSheet} /> @@ -333,6 +362,13 @@ export function CanvasLabPage(): ReactElement { onImport={handleImport} container={sheetContainer} /> +
); } diff --git a/studio/frontend/src/features/canvas-lab/components/block-sheet.tsx b/studio/frontend/src/features/canvas-lab/components/block-sheet.tsx index fb3b2cceba..c234169e98 100644 --- a/studio/frontend/src/features/canvas-lab/components/block-sheet.tsx +++ b/studio/frontend/src/features/canvas-lab/components/block-sheet.tsx @@ -8,7 +8,9 @@ import { } from "@/components/ui/sheet"; import { ArrowLeft02Icon, - ArrowRight01Icon, type Database02Icon, + ArrowRight01Icon, + CodeIcon, + type Database02Icon, PlusSignIcon, } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; @@ -16,8 +18,15 @@ import type { ReactElement } from "react"; import type { LlmType, SamplerType } from "../types"; import { BLOCK_GROUPS, getBlocksForKind } from "../blocks/registry"; -type SheetView = "root" | "sampler" | "llm" | "expression"; +type SheetView = "root" | "sampler" | "llm" | "expression" | "processor"; type SheetKind = "sampler" | "llm" | "expression"; +type RootSheetView = Exclude; +type RootGroup = { + kind: RootSheetView; + title: string; + description: string; + icon: typeof Database02Icon; +}; type BlockSheetProps = { container: HTMLDivElement | null; @@ -28,6 +37,7 @@ type BlockSheetProps = { onAddModelProvider: () => void; onAddModelConfig: () => void; onAddExpression: () => void; + onOpenProcessors: () => void; }; function getSheetTitle(view: SheetView): string { @@ -40,6 +50,9 @@ function getSheetTitle(view: SheetView): string { if (view === "expression") { return "Expression blocks"; } + if (view === "processor") { + return "Processor blocks"; + } return "LLM blocks"; } @@ -48,8 +61,19 @@ const VIEW_KIND: Record = { sampler: "sampler", llm: "llm", expression: "expression", + processor: null, }; +const ROOT_GROUPS: RootGroup[] = [ + ...BLOCK_GROUPS, + { + kind: "processor", + title: "Processors", + description: "Output schema + post batch.", + icon: CodeIcon, + }, +]; + function BlockSheetButton({ icon, title, @@ -97,6 +121,7 @@ export function BlockSheet({ onAddModelProvider, onAddModelConfig, onAddExpression, + onOpenProcessors, }: BlockSheetProps): ReactElement { const title = getSheetTitle(view); return ( @@ -138,7 +163,7 @@ export function BlockSheet({
{view === "root" && - BLOCK_GROUPS.map((item, index) => ( + ROOT_GROUPS.map((item, index) => ( onViewChange(item.kind)} /> ))} + {view === "processor" && ( + + )} {view !== "root" && + view !== "processor" && getBlocksForKind(VIEW_KIND[view] ?? "sampler").map( (item, index) => ( + {(config.kind === "sampler" || + config.kind === "llm" || + config.kind === "expression") && ( +
+
+

Drop from final dataset

+

+ Keep for generation but omit from exported rows. +

+
+ onUpdate(config.id, { drop: value })} + /> +
+ )} {renderBlockDialog(config, categoryOptions, onUpdate)}
)} diff --git a/studio/frontend/src/features/canvas-lab/dialogs/processors-dialog.tsx b/studio/frontend/src/features/canvas-lab/dialogs/processors-dialog.tsx new file mode 100644 index 0000000000..9dd0a771af --- /dev/null +++ b/studio/frontend/src/features/canvas-lab/dialogs/processors-dialog.tsx @@ -0,0 +1,134 @@ +import { Button } from "@/components/ui/button"; +import { Dialog, DialogContent, DialogFooter } from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; +import { Switch } from "@/components/ui/switch"; +import { Textarea } from "@/components/ui/textarea"; +import { type ReactElement, useMemo } from "react"; +import type { CanvasProcessorConfig } from "../types"; +import { buildDefaultSchemaTransform } from "../utils/processors"; +type ProcessorsDialogProps = { + open: boolean; + onOpenChange: (open: boolean) => void; + processors: CanvasProcessorConfig[]; + onProcessorsChange: (processors: CanvasProcessorConfig[]) => void; + container?: HTMLDivElement | null; +}; + +export function ProcessorsDialog({ + open, + onOpenChange, + processors, + onProcessorsChange, + container, +}: ProcessorsDialogProps): ReactElement { + const schemaIndex = useMemo( + () => + processors.findIndex( + (processor) => processor.processor_type === "schema_transform", + ), + [processors], + ); + const schemaProcessor = schemaIndex >= 0 ? processors[schemaIndex] : null; + const nameId = schemaProcessor ? `${schemaProcessor.id}-name` : "schema-transform-name"; + const templateId = schemaProcessor + ? `${schemaProcessor.id}-template` + : "schema-transform-template"; + + const setSchemaEnabled = (enabled: boolean) => { + if (enabled) { + if (schemaProcessor) { + return; + } + onProcessorsChange([...processors, buildDefaultSchemaTransform()]); + return; + } + onProcessorsChange( + processors.filter( + (processor) => processor.processor_type !== "schema_transform", + ), + ); + }; + + const updateSchema = (patch: Partial) => { + if (!schemaProcessor) { + return; + } + const next = [...processors]; + next[schemaIndex] = { ...schemaProcessor, ...patch }; + onProcessorsChange(next); + }; + + return ( + + +
+
+
+

Schema transform

+

+ Transform final rows to target schema (post-batch). +

+
+ +
+ + {schemaProcessor && ( +
+
+ + updateSchema({ name: event.target.value })} + /> +
+
+ +