diff --git a/studio/frontend/src/features/recipe-studio/blocks/definitions.ts b/studio/frontend/src/features/recipe-studio/blocks/definitions.ts index db78810734..f19b4a84f6 100644 --- a/studio/frontend/src/features/recipe-studio/blocks/definitions.ts +++ b/studio/frontend/src/features/recipe-studio/blocks/definitions.ts @@ -4,6 +4,8 @@ import { CodeIcon, CodeSimpleIcon, DiceFaces03Icon, + DocumentAttachmentIcon, + DocumentCodeIcon, EqualSignIcon, FingerPrintIcon, FunctionIcon, @@ -116,7 +118,7 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [ type: "seed_local", title: "Seed (Local File)", description: "Upload CSV/JSON/JSONL and use rows as seed context.", - icon: Plant01Icon, + icon: DocumentCodeIcon, dialogKey: "seed", createConfig: (id, existing) => makeSeedConfig(id, existing, "local"), }, @@ -125,7 +127,7 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [ type: "seed_unstructured", title: "Seed (Unstructured)", description: "Upload PDF/DOCX/TXT, chunk to text rows, then seed.", - icon: Plant01Icon, + icon: DocumentAttachmentIcon, dialogKey: "seed", createConfig: (id, existing) => makeSeedConfig(id, existing, "unstructured"), }, diff --git a/studio/frontend/src/features/recipe-studio/dialogs/config-dialog.tsx b/studio/frontend/src/features/recipe-studio/dialogs/config-dialog.tsx index d0680641fa..aedb60a5c7 100644 --- a/studio/frontend/src/features/recipe-studio/dialogs/config-dialog.tsx +++ b/studio/frontend/src/features/recipe-studio/dialogs/config-dialog.tsx @@ -77,6 +77,7 @@ export function ConfigDialog({ )} {renderBlockDialog( config, + open, categoryOptions, modelConfigAliases, modelProviderOptions, diff --git a/studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx b/studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx index 535b08b6e9..2b54e539c5 100644 --- a/studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx +++ b/studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx @@ -35,7 +35,7 @@ import { } from "@/components/ui/tabs"; import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters"; import mammoth from "mammoth"; -import { type ReactElement, useEffect, useMemo, useState } from "react"; +import { type ReactElement, useCallback, useEffect, useMemo, useRef, useState } from "react"; import { extractText, getDocumentProxy } from "unpdf"; import { inspectSeedDataset, inspectSeedUpload } from "../../api"; import type { @@ -59,16 +59,14 @@ const SELECTION_OPTIONS: Array<{ value: SeedSelectionType; label: string }> = [ const LOCAL_ACCEPT = ".csv,.json,.jsonl"; const UNSTRUCTURED_ACCEPT = ".txt,.pdf,.docx"; const MAX_UPLOAD_BYTES = 50 * 1024 * 1024; -const CHUNK_SIZE = 1200; -const CHUNK_OVERLAP = 200; -const UNSTRUCTURED_SPLITTER = new RecursiveCharacterTextSplitter({ - chunkSize: CHUNK_SIZE, - chunkOverlap: CHUNK_OVERLAP, -}); +const DEFAULT_CHUNK_SIZE = 1200; +const DEFAULT_CHUNK_OVERLAP = 200; +const MAX_CHUNK_SIZE = 20000; type SeedDialogProps = { config: SeedConfig; onUpdate: (patch: Partial) => void; + open: boolean; }; function getErrorMessage(error: unknown, fallback: string): string { @@ -89,11 +87,54 @@ function stringifyCell(value: unknown): string { } } -async function chunkText(input: string): Promise { +function parseChunkNumber( + value: string | undefined, + fallback: number, + min: number, + max: number, +): number { + const raw = value?.trim(); + if (!raw) return fallback; + const parsed = Number(raw); + if (!Number.isFinite(parsed)) return fallback; + const int = Math.floor(parsed); + if (int < min) return min; + if (int > max) return max; + return int; +} + +function resolveChunking(config: SeedConfig): { + chunkSize: number; + chunkOverlap: number; +} { + const chunkSize = parseChunkNumber( + config.unstructured_chunk_size, + DEFAULT_CHUNK_SIZE, + 1, + MAX_CHUNK_SIZE, + ); + const chunkOverlap = parseChunkNumber( + config.unstructured_chunk_overlap, + DEFAULT_CHUNK_OVERLAP, + 0, + Math.max(0, chunkSize - 1), + ); + return { chunkSize, chunkOverlap }; +} + +async function chunkText( + input: string, + chunkSize: number, + chunkOverlap: number, +): Promise { const text = input.replace(/\r\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim(); if (!text) return []; - const chunks = await UNSTRUCTURED_SPLITTER.splitText(text); + const splitter = new RecursiveCharacterTextSplitter({ + chunkSize, + chunkOverlap, + }); + const chunks = await splitter.splitText(text); return chunks.map((chunk) => chunk.trim()).filter(Boolean); } @@ -129,7 +170,7 @@ async function extractUnstructuredText(file: File): Promise { throw new Error("Unsupported unstructured file type"); } -export function SeedDialog({ config, onUpdate }: SeedDialogProps): ReactElement { +export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactElement { const [inspectError, setInspectError] = useState(null); const [isInspecting, setIsInspecting] = useState(false); const [advancedOpen, setAdvancedOpen] = useState(false); @@ -141,20 +182,53 @@ export function SeedDialog({ config, onUpdate }: SeedDialogProps): ReactElement useEffect(() => { setInspectError(null); - setPreviewRows([]); setLocalFile(null); setUnstructuredFile(null); }, [mode]); + useEffect(() => { + setPreviewRows(config.seed_preview_rows ?? []); + }, [config.seed_preview_rows]); + const samplingId = `${config.id}-sampling`; const selectionId = `${config.id}-selection`; const tokenId = `${config.id}-hf-token`; const subsetId = `${config.id}-hf-subset`; const splitId = `${config.id}-hf-split`; const datasetId = `${config.id}-hf-dataset`; + const chunkSizeId = `${config.id}-chunk-size`; + const chunkOverlapId = `${config.id}-chunk-overlap`; + const [lastLoadedKey, setLastLoadedKey] = useState(null); + const wasOpenRef = useRef(open); - async function loadSeedMetadata(): Promise { - setInspectError(null); + const getCurrentLoadKey = useCallback((): string | null => { + if (mode === "hf") { + const dataset = config.hf_repo_id.trim(); + if (!dataset) return null; + const subset = config.hf_subset?.trim() ?? ""; + const split = config.hf_split?.trim() || "train"; + const token = config.hf_token?.trim() ?? ""; + return `hf:${dataset}|${subset}|${split}|${token}`; + } + if (mode === "local") { + if (!localFile) return null; + return `local:${localFile.name}|${localFile.size}|${localFile.lastModified}`; + } + if (!unstructuredFile) return null; + const { chunkSize, chunkOverlap } = resolveChunking(config); + return `unstructured:${unstructuredFile.name}|${unstructuredFile.size}|${unstructuredFile.lastModified}|${chunkSize}|${chunkOverlap}`; + }, [ + config, + localFile, + mode, + unstructuredFile, + ]); + + const loadSeedMetadata = useCallback(async (opts?: { silent?: boolean }): Promise => { + const loadKey = getCurrentLoadKey(); + if (!opts?.silent) { + setInspectError(null); + } setIsInspecting(true); try { if (mode === "hf") { @@ -172,13 +246,15 @@ export function SeedDialog({ config, onUpdate }: SeedDialogProps): ReactElement onUpdate({ hf_path: response.resolved_path, seed_columns: response.columns, + seed_preview_rows: response.preview_rows ?? [], hf_split: response.split ?? config.hf_split ?? "", hf_subset: response.subset ?? config.hf_subset ?? "", local_file_name: "", unstructured_file_name: "", }); setPreviewRows(response.preview_rows ?? []); - return; + setLastLoadedKey(loadKey); + return true; } if (mode === "local") { @@ -197,6 +273,7 @@ export function SeedDialog({ config, onUpdate }: SeedDialogProps): ReactElement onUpdate({ hf_path: response.resolved_path, seed_columns: response.columns, + seed_preview_rows: response.preview_rows ?? [], hf_repo_id: "", hf_subset: "", hf_split: "", @@ -204,7 +281,8 @@ export function SeedDialog({ config, onUpdate }: SeedDialogProps): ReactElement unstructured_file_name: "", }); setPreviewRows(response.preview_rows ?? []); - return; + setLastLoadedKey(loadKey); + return true; } if (!unstructuredFile) { @@ -215,7 +293,8 @@ export function SeedDialog({ config, onUpdate }: SeedDialogProps): ReactElement } const text = await extractUnstructuredText(unstructuredFile); - const chunks = await chunkText(text); + const { chunkSize, chunkOverlap } = resolveChunking(config); + const chunks = await chunkText(text, chunkSize, chunkOverlap); if (chunks.length === 0) { throw new Error("No text found in file."); } @@ -238,6 +317,7 @@ export function SeedDialog({ config, onUpdate }: SeedDialogProps): ReactElement onUpdate({ hf_path: response.resolved_path, seed_columns: response.columns, + seed_preview_rows: response.preview_rows ?? [], hf_repo_id: "", hf_subset: "", hf_split: "", @@ -245,13 +325,38 @@ export function SeedDialog({ config, onUpdate }: SeedDialogProps): ReactElement unstructured_file_name: unstructuredFile.name, }); setPreviewRows(response.preview_rows ?? []); + setLastLoadedKey(loadKey); + return true; } catch (error) { - setInspectError(getErrorMessage(error, "Failed to load seed metadata.")); + if (!opts?.silent) { + setInspectError(getErrorMessage(error, "Failed to load seed metadata.")); + } setPreviewRows([]); + return false; } finally { setIsInspecting(false); } - } + }, [ + config, + getCurrentLoadKey, + localFile, + mode, + onUpdate, + unstructuredFile, + ]); + + useEffect(() => { + const wasOpen = wasOpenRef.current; + wasOpenRef.current = open; + if (!wasOpen || open || isInspecting) { + return; + } + const key = getCurrentLoadKey(); + if (!key || key === lastLoadedKey) { + return; + } + void loadSeedMetadata({ silent: true }); + }, [getCurrentLoadKey, isInspecting, lastLoadedKey, loadSeedMetadata, open]); const previewColumns = useMemo(() => { const loadedColumns = config.seed_columns ?? []; @@ -260,13 +365,6 @@ export function SeedDialog({ config, onUpdate }: SeedDialogProps): ReactElement return []; }, [config.seed_columns, previewRows]); - const canLoad = - mode === "hf" - ? Boolean(config.hf_repo_id.trim()) - : mode === "local" - ? Boolean(localFile) - : Boolean(unstructuredFile); - return ( @@ -284,19 +382,31 @@ export function SeedDialog({ config, onUpdate }: SeedDialogProps): ReactElement htmlFor={datasetId} hint="Hugging Face dataset repo id (org/repo)." /> - - onUpdate({ - hf_repo_id: event.target.value, - hf_path: "", - seed_columns: [], - }) - } - /> +
+ + onUpdate({ + hf_repo_id: event.target.value, + hf_path: "", + seed_columns: [], + seed_preview_rows: [], + }) + } + /> + +
@@ -353,21 +463,39 @@ export function SeedDialog({ config, onUpdate }: SeedDialogProps): ReactElement label="Local file" hint="Upload CSV, JSON, or JSONL seed file." /> - { - const file = event.target.files?.[0] ?? null; - setLocalFile(file); - onUpdate({ hf_path: "", seed_columns: [] }); - }} - /> +
+ { + const file = event.target.files?.[0] ?? null; + setLocalFile(file); + onUpdate({ + hf_path: "", + seed_columns: [], + seed_preview_rows: [], + local_file_name: file?.name ?? "", + }); + }} + /> + +

Upload-only. Max 50MB.

- {localFile && ( -

Selected: {localFile.name}

+ {(localFile?.name || config.local_file_name?.trim()) && ( +

+ Selected: {localFile?.name ?? config.local_file_name?.trim()} +

)}
)} @@ -378,37 +506,45 @@ export function SeedDialog({ config, onUpdate }: SeedDialogProps): ReactElement label="Unstructured file" hint="Upload PDF, DOCX, or TXT. We chunk text into seed rows." /> - { - const file = event.target.files?.[0] ?? null; - setUnstructuredFile(file); - onUpdate({ hf_path: "", seed_columns: [] }); - }} - /> +
+ { + const file = event.target.files?.[0] ?? null; + setUnstructuredFile(file); + onUpdate({ + hf_path: "", + seed_columns: [], + seed_preview_rows: [], + unstructured_file_name: file?.name ?? "", + }); + }} + /> + +

Chunking uses chunk_text only. Max 50MB.

- {unstructuredFile && ( + {(unstructuredFile?.name || + config.unstructured_file_name?.trim()) && (

- Selected: {unstructuredFile.name} + Selected:{" "} + {unstructuredFile?.name ?? config.unstructured_file_name?.trim()}

)} )} - - {inspectError &&

{inspectError}

} @@ -472,6 +608,46 @@ export function SeedDialog({ config, onUpdate }: SeedDialogProps): ReactElement + {mode === "unstructured" && ( +
+
+ + + onUpdate({ unstructured_chunk_size: event.target.value }) + } + /> +
+
+ + + onUpdate({ unstructured_chunk_overlap: event.target.value }) + } + /> +
+
+ )} + {config.selection_type === "index_range" && (
@@ -532,20 +708,10 @@ export function SeedDialog({ config, onUpdate }: SeedDialogProps): ReactElement Seed preview - Click load to fetch 10 rows from the selected seed source. + Use the load button next to the source input to fetch 10 rows. - - - +
) : ( @@ -553,8 +719,8 @@ export function SeedDialog({ config, onUpdate }: SeedDialogProps): ReactElement
Loaded columns: {previewColumns.join(", ") || "None"}
-
- +
+
{previewColumns.map((col) => ( diff --git a/studio/frontend/src/features/recipe-studio/stores/recipe-studio.ts b/studio/frontend/src/features/recipe-studio/stores/recipe-studio.ts index b7cbc4726a..6ed63abd4b 100644 --- a/studio/frontend/src/features/recipe-studio/stores/recipe-studio.ts +++ b/studio/frontend/src/features/recipe-studio/stores/recipe-studio.ts @@ -319,6 +319,9 @@ export const useRecipeStudioStore = create((set, get) => ({ local_file_name: "", unstructured_file_name: "", seed_columns: [], + seed_preview_rows: [], + unstructured_chunk_size: "1200", + unstructured_chunk_overlap: "200", }; return { configs: { diff --git a/studio/frontend/src/features/recipe-studio/types/index.ts b/studio/frontend/src/features/recipe-studio/types/index.ts index c019f393a8..e349e7e59e 100644 --- a/studio/frontend/src/features/recipe-studio/types/index.ts +++ b/studio/frontend/src/features/recipe-studio/types/index.ts @@ -230,6 +230,12 @@ export type SeedConfig = { hf_endpoint?: string; local_file_name?: string; unstructured_file_name?: string; + // ui-only + seed_preview_rows?: Record[]; + // ui-only (string for input ergonomics) + unstructured_chunk_size?: string; + // ui-only (string for input ergonomics) + unstructured_chunk_overlap?: string; seed_splits?: string[]; // ui-only // biome-ignore lint/style/useNamingConvention: ui schema diff --git a/studio/frontend/src/features/recipe-studio/utils/config-factories.ts b/studio/frontend/src/features/recipe-studio/utils/config-factories.ts index c987d39644..b39199af86 100644 --- a/studio/frontend/src/features/recipe-studio/utils/config-factories.ts +++ b/studio/frontend/src/features/recipe-studio/utils/config-factories.ts @@ -296,6 +296,9 @@ export function makeSeedConfig( hf_endpoint: "https://huggingface.co", local_file_name: "", unstructured_file_name: "", + seed_preview_rows: [], + unstructured_chunk_size: "1200", + unstructured_chunk_overlap: "200", seed_splits: [], seed_globs_by_split: {}, seed_columns: [], diff --git a/studio/frontend/src/features/recipe-studio/utils/import/importer.ts b/studio/frontend/src/features/recipe-studio/utils/import/importer.ts index 694625fa0c..21166e7a48 100644 --- a/studio/frontend/src/features/recipe-studio/utils/import/importer.ts +++ b/studio/frontend/src/features/recipe-studio/utils/import/importer.ts @@ -31,8 +31,24 @@ type UiInput = { nodes?: unknown; edges?: unknown; seed_source_type?: unknown; + seed_columns?: unknown; + seed_preview_rows?: unknown; + local_file_name?: unknown; + unstructured_file_name?: unknown; + unstructured_chunk_size?: unknown; + unstructured_chunk_overlap?: unknown; }; +function readStringNumber(value: unknown): string | undefined { + if (typeof value === "string") { + return value; + } + if (typeof value === "number" && Number.isFinite(value)) { + return String(value); + } + return undefined; +} + function parseProcessors(input: unknown): RecipeProcessorConfig[] { if (!Array.isArray(input)) { return []; @@ -228,11 +244,36 @@ export function importRecipePayload(input: string): ImportResult { uiSeedSourceTypeRaw === "unstructured" ? uiSeedSourceTypeRaw : undefined; + const uiSeedColumns = Array.isArray(ui?.seed_columns) + ? ui.seed_columns + .map((value) => (typeof value === "string" ? value.trim() : "")) + .filter(Boolean) + : undefined; + const uiSeedPreviewRows = Array.isArray(ui?.seed_preview_rows) + ? ui.seed_preview_rows + .filter((row): row is Record => isRecord(row)) + .map((row) => ({ ...row })) + : undefined; + const uiLocalFileName = readString(ui?.local_file_name) ?? undefined; + const uiUnstructuredFileName = + readString(ui?.unstructured_file_name) ?? undefined; + const uiUnstructuredChunkSize = readStringNumber(ui?.unstructured_chunk_size); + const uiUnstructuredChunkOverlap = readStringNumber( + ui?.unstructured_chunk_overlap, + ); if (recipe.seed_config) { const id = `n${nextId}`; nextId += 1; - const seedConfig = parseSeedConfig(recipe.seed_config, id, uiSeedSourceType); + const seedConfig = parseSeedConfig(recipe.seed_config, id, { + preferredSourceType: uiSeedSourceType, + seed_columns: uiSeedColumns, + seed_preview_rows: uiSeedPreviewRows, + local_file_name: uiLocalFileName, + unstructured_file_name: uiUnstructuredFileName, + unstructured_chunk_size: uiUnstructuredChunkSize, + unstructured_chunk_overlap: uiUnstructuredChunkOverlap, + }); if (seedConfig) { configs.push(seedConfig); } diff --git a/studio/frontend/src/features/recipe-studio/utils/import/parsers/seed-config-parser.ts b/studio/frontend/src/features/recipe-studio/utils/import/parsers/seed-config-parser.ts index 3c59cf1482..351070ba98 100644 --- a/studio/frontend/src/features/recipe-studio/utils/import/parsers/seed-config-parser.ts +++ b/studio/frontend/src/features/recipe-studio/utils/import/parsers/seed-config-parser.ts @@ -27,6 +27,9 @@ function makeDefaultSeedConfig(id: string): SeedConfig { hf_endpoint: "https://huggingface.co", local_file_name: "", unstructured_file_name: "", + seed_preview_rows: [], + unstructured_chunk_size: "1200", + unstructured_chunk_overlap: "200", seed_splits: [], seed_globs_by_split: {}, seed_columns: [], @@ -127,7 +130,15 @@ function parseSeedSettings(seedConfigRaw: unknown): Partial { export function parseSeedConfig( seedConfigRaw: unknown, id: string, - preferredSourceType?: SeedSourceType, + options?: { + preferredSourceType?: SeedSourceType; + seed_columns?: string[]; + seed_preview_rows?: Record[]; + local_file_name?: string; + unstructured_file_name?: string; + unstructured_chunk_size?: string; + unstructured_chunk_overlap?: string; + }, ): SeedConfig | null { if (!seedConfigRaw) { return null; @@ -136,8 +147,8 @@ export function parseSeedConfig( let sourceType: SeedSourceType = "hf"; if (parsed.seed_source_type === "hf") { sourceType = "hf"; - } else if (preferredSourceType) { - sourceType = preferredSourceType; + } else if (options?.preferredSourceType) { + sourceType = options.preferredSourceType; } else if (parsed.seed_source_type) { sourceType = parsed.seed_source_type; } @@ -145,5 +156,21 @@ export function parseSeedConfig( ...makeDefaultSeedConfig(id), ...parsed, // payload-only fields override ui defaults seed_source_type: sourceType, + ...(options?.seed_columns ? { seed_columns: options.seed_columns } : {}), + ...(options?.seed_preview_rows + ? { seed_preview_rows: options.seed_preview_rows } + : {}), + ...(options?.local_file_name !== undefined + ? { local_file_name: options.local_file_name } + : {}), + ...(options?.unstructured_file_name !== undefined + ? { unstructured_file_name: options.unstructured_file_name } + : {}), + ...(options?.unstructured_chunk_size !== undefined + ? { unstructured_chunk_size: options.unstructured_chunk_size } + : {}), + ...(options?.unstructured_chunk_overlap !== undefined + ? { unstructured_chunk_overlap: options.unstructured_chunk_overlap } + : {}), }; } diff --git a/studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts b/studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts index 9516ab31e7..5b9d0ab9b4 100644 --- a/studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts +++ b/studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts @@ -242,6 +242,24 @@ export function buildRecipePayload( nodes: uiNodes, edges: uiEdges, ...(firstSeed && { seed_source_type: firstSeed.seed_source_type }), + ...(firstSeed && { seed_columns: firstSeed.seed_columns ?? [] }), + ...(firstSeed && { seed_preview_rows: firstSeed.seed_preview_rows ?? [] }), + ...(firstSeed && + firstSeed.local_file_name !== undefined && { + local_file_name: firstSeed.local_file_name, + }), + ...(firstSeed && + firstSeed.unstructured_file_name !== undefined && { + unstructured_file_name: firstSeed.unstructured_file_name, + }), + ...(firstSeed && + firstSeed.unstructured_chunk_size !== undefined && { + unstructured_chunk_size: firstSeed.unstructured_chunk_size, + }), + ...(firstSeed && + firstSeed.unstructured_chunk_overlap !== undefined && { + unstructured_chunk_overlap: firstSeed.unstructured_chunk_overlap, + }), }, }, }; diff --git a/studio/frontend/src/features/recipe-studio/utils/payload/types.ts b/studio/frontend/src/features/recipe-studio/utils/payload/types.ts index 170aaa5a96..98857486cb 100644 --- a/studio/frontend/src/features/recipe-studio/utils/payload/types.ts +++ b/studio/frontend/src/features/recipe-studio/utils/payload/types.ts @@ -32,6 +32,13 @@ export type RecipePayload = { edges: { from: string; to: string; type?: string }[]; // ui-only, used to preserve seed block mode across imports/refresh seed_source_type?: "hf" | "local" | "unstructured"; + // ui-only, seed metadata cached for refresh/import UX + seed_columns?: string[]; + seed_preview_rows?: Record[]; + local_file_name?: string; + unstructured_file_name?: string; + unstructured_chunk_size?: string; + unstructured_chunk_overlap?: string; }; };