diff --git a/studio/backend/core/data_recipe/service.py b/studio/backend/core/data_recipe/service.py index dca3e867ce..b04c6bbf00 100644 --- a/studio/backend/core/data_recipe/service.py +++ b/studio/backend/core/data_recipe/service.py @@ -101,13 +101,30 @@ def build_mcp_providers( def build_config_builder(recipe: dict[str, Any]): from data_designer.config import DataDesignerConfigBuilder + from data_designer.config.processors import ProcessorType recipe_core = { key: value for key, value in recipe.items() if key not in {"model_providers", "mcp_providers"} } - return DataDesignerConfigBuilder.from_config({"data_designer": recipe_core}) + builder = DataDesignerConfigBuilder.from_config({"data_designer": recipe_core}) + + # DataDesignerConfigBuilder.from_config currently skips processors. + # Re-attach explicitly so drop_columns/schema_transform survive API payload. + for processor in recipe_core.get("processors") or []: + if not isinstance(processor, dict): + continue + processor_type_raw = processor.get("processor_type") + if not isinstance(processor_type_raw, str): + continue + kwargs = {k: v for k, v in processor.items() if k != "processor_type"} + builder.add_processor( + processor_type=ProcessorType(processor_type_raw), + **kwargs, + ) + + return builder def create_data_designer( diff --git a/studio/frontend/src/features/recipe-studio/dialogs/config-dialog.tsx b/studio/frontend/src/features/recipe-studio/dialogs/config-dialog.tsx index aedb60a5c7..1421ce1ffd 100644 --- a/studio/frontend/src/features/recipe-studio/dialogs/config-dialog.tsx +++ b/studio/frontend/src/features/recipe-studio/dialogs/config-dialog.tsx @@ -32,6 +32,12 @@ export function ConfigDialog({ container, }: ConfigDialogProps): ReactElement { const blockDefinition = getBlockDefinitionForConfig(config); + const showDropToggle = + config?.kind === "sampler" || + config?.kind === "llm" || + config?.kind === "expression" || + (config?.kind === "seed" && + (config.seed_source_type ?? "hf") === "unstructured"); return ( @@ -58,10 +64,7 @@ export function ConfigDialog({ {config && (
- {(config.kind === "sampler" || - config.kind === "llm" || - config.kind === "expression" || - config.kind === "seed") && ( + {showDropToggle && (

Drop from final dataset

diff --git a/studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx b/studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx index 2b54e539c5..c60ee4a5db 100644 --- a/studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx +++ b/studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx @@ -1,4 +1,5 @@ import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; import { Collapsible, CollapsibleContent, @@ -37,6 +38,7 @@ import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters"; import mammoth from "mammoth"; import { type ReactElement, useCallback, useEffect, useMemo, useRef, useState } from "react"; import { extractText, getDocumentProxy } from "unpdf"; +import { cn } from "@/lib/utils"; import { inspectSeedDataset, inspectSeedUpload } from "../../api"; import type { SeedConfig, @@ -62,6 +64,7 @@ const MAX_UPLOAD_BYTES = 50 * 1024 * 1024; const DEFAULT_CHUNK_SIZE = 1200; const DEFAULT_CHUNK_OVERLAP = 200; const MAX_CHUNK_SIZE = 20000; +const PREVIEW_TRUNCATE_AT = 320; type SeedDialogProps = { config: SeedConfig; @@ -87,6 +90,17 @@ function stringifyCell(value: unknown): string { } } +function isExpandablePreviewValue(value: string): boolean { + return value.length > PREVIEW_TRUNCATE_AT; +} + +function truncatePreviewValue(value: string): string { + if (!isExpandablePreviewValue(value)) { + return value; + } + return `${value.slice(0, PREVIEW_TRUNCATE_AT)}…`; +} + function parseChunkNumber( value: string | undefined, fallback: number, @@ -175,6 +189,7 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl const [isInspecting, setIsInspecting] = useState(false); const [advancedOpen, setAdvancedOpen] = useState(false); const [previewRows, setPreviewRows] = useState[]>([]); + const [expandedPreviewRows, setExpandedPreviewRows] = useState>({}); const [localFile, setLocalFile] = useState(null); const [unstructuredFile, setUnstructuredFile] = useState(null); @@ -188,6 +203,7 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl useEffect(() => { setPreviewRows(config.seed_preview_rows ?? []); + setExpandedPreviewRows({}); }, [config.seed_preview_rows]); const samplingId = `${config.id}-sampling`; @@ -246,6 +262,9 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl onUpdate({ hf_path: response.resolved_path, seed_columns: response.columns, + seed_drop_columns: (config.seed_drop_columns ?? []).filter((name) => + response.columns.includes(name), + ), seed_preview_rows: response.preview_rows ?? [], hf_split: response.split ?? config.hf_split ?? "", hf_subset: response.subset ?? config.hf_subset ?? "", @@ -273,6 +292,9 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl onUpdate({ hf_path: response.resolved_path, seed_columns: response.columns, + seed_drop_columns: (config.seed_drop_columns ?? []).filter((name) => + response.columns.includes(name), + ), seed_preview_rows: response.preview_rows ?? [], hf_repo_id: "", hf_subset: "", @@ -317,6 +339,9 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl onUpdate({ hf_path: response.resolved_path, seed_columns: response.columns, + seed_drop_columns: (config.seed_drop_columns ?? []).filter((name) => + response.columns.includes(name), + ), seed_preview_rows: response.preview_rows ?? [], hf_repo_id: "", hf_subset: "", @@ -364,6 +389,14 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl if (previewRows[0]) return Object.keys(previewRows[0]); return []; }, [config.seed_columns, previewRows]); + const selectedSeedDropColumns = useMemo( + () => (config.seed_drop_columns ?? []).filter((name) => name.trim().length > 0), + [config.seed_drop_columns], + ); + const selectedSeedDropSet = useMemo( + () => new Set(selectedSeedDropColumns), + [selectedSeedDropColumns], + ); return ( @@ -393,6 +426,7 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl hf_repo_id: event.target.value, hf_path: "", seed_columns: [], + seed_drop_columns: [], seed_preview_rows: [], }) } @@ -474,6 +508,7 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl onUpdate({ hf_path: "", seed_columns: [], + seed_drop_columns: [], seed_preview_rows: [], local_file_name: file?.name ?? "", }); @@ -517,6 +552,7 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl onUpdate({ hf_path: "", seed_columns: [], + seed_drop_columns: [], seed_preview_rows: [], unstructured_file_name: file?.name ?? "", }); @@ -547,6 +583,44 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl {inspectError &&

{inspectError}

} + {mode !== "unstructured" && ( +
+ + {previewColumns.length === 0 ? ( +

+ Load columns to select which seed fields to drop. +

+ ) : ( +
+ {previewColumns.map((columnName) => { + const checked = selectedSeedDropSet.has(columnName); + return ( + + ); + })} +
+ )} +
+ )} +