ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: add per-column seed drop support with UI integration, validation, and payload enhancements

Browse source
This commit is contained in:
Shine1i 2026-02-23 23:33:59 +01:00
commit ab31aa9ed4
12 changed files with 219 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

19
studio/backend/core/data_recipe/service.py
View file

@ -101,13 +101,30 @@ def build_mcp_providers(
def build_config_builder(recipe: dict[str, Any]):
from data_designer.config import DataDesignerConfigBuilder
from data_designer.config.processors import ProcessorType
recipe_core = {
key: value
for key, value in recipe.items()
if key not in {"model_providers", "mcp_providers"}
}
return DataDesignerConfigBuilder.from_config({"data_designer": recipe_core})
builder = DataDesignerConfigBuilder.from_config({"data_designer": recipe_core})
# DataDesignerConfigBuilder.from_config currently skips processors.
# Re-attach explicitly so drop_columns/schema_transform survive API payload.
for processor in recipe_core.get("processors") or []:
if not isinstance(processor, dict):
continue
processor_type_raw = processor.get("processor_type")
if not isinstance(processor_type_raw, str):
continue
kwargs = {k: v for k, v in processor.items() if k != "processor_type"}
builder.add_processor(
processor_type=ProcessorType(processor_type_raw),
**kwargs,
)
return builder
def create_data_designer(

11
studio/frontend/src/features/recipe-studio/dialogs/config-dialog.tsx
View file

@ -32,6 +32,12 @@ export function ConfigDialog({
container,
}: ConfigDialogProps): ReactElement {
const blockDefinition = getBlockDefinitionForConfig(config);
const showDropToggle =
config?.kind === "sampler" ||
config?.kind === "llm" ||
config?.kind === "expression" ||
(config?.kind === "seed" &&
(config.seed_source_type ?? "hf") === "unstructured");
return (
<Dialog open={open} onOpenChange={onOpenChange}>
@ -58,10 +64,7 @@ export function ConfigDialog({
{config && (
<div className="space-y-4">
<ValidationBanner config={config} />
{(config.kind === "sampler" ||
config.kind === "llm" ||
config.kind === "expression" ||
config.kind === "seed") && (
{showDropToggle && (
<div className="flex items-center corner-squircle justify-between gap-3 rounded-2xl border border-border/60 px-3 py-2">
<div>
<p className="text-sm font-semibold">Drop from final dataset</p>

107
studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx
View file

@ -1,4 +1,5 @@
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
import {
Collapsible,
CollapsibleContent,
@ -37,6 +38,7 @@ import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters";
import mammoth from "mammoth";
import { type ReactElement, useCallback, useEffect, useMemo, useRef, useState } from "react";
import { extractText, getDocumentProxy } from "unpdf";
import { cn } from "@/lib/utils";
import { inspectSeedDataset, inspectSeedUpload } from "../../api";
import type {
SeedConfig,
@ -62,6 +64,7 @@ const MAX_UPLOAD_BYTES = 50 * 1024 * 1024;
const DEFAULT_CHUNK_SIZE = 1200;
const DEFAULT_CHUNK_OVERLAP = 200;
const MAX_CHUNK_SIZE = 20000;
const PREVIEW_TRUNCATE_AT = 320;
type SeedDialogProps = {
config: SeedConfig;
@ -87,6 +90,17 @@ function stringifyCell(value: unknown): string {
}
}
function isExpandablePreviewValue(value: string): boolean {
return value.length > PREVIEW_TRUNCATE_AT;
}
function truncatePreviewValue(value: string): string {
if (!isExpandablePreviewValue(value)) {
return value;
}
return `${value.slice(0, PREVIEW_TRUNCATE_AT)}…`;
}
function parseChunkNumber(
value: string | undefined,
fallback: number,
@ -175,6 +189,7 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
const [isInspecting, setIsInspecting] = useState(false);
const [advancedOpen, setAdvancedOpen] = useState(false);
const [previewRows, setPreviewRows] = useState<Record<string, unknown>[]>([]);
const [expandedPreviewRows, setExpandedPreviewRows] = useState<Record<number, boolean>>({});
const [localFile, setLocalFile] = useState<File | null>(null);
const [unstructuredFile, setUnstructuredFile] = useState<File | null>(null);
@ -188,6 +203,7 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
useEffect(() => {
setPreviewRows(config.seed_preview_rows ?? []);
setExpandedPreviewRows({});
}, [config.seed_preview_rows]);
const samplingId = `${config.id}-sampling`;
@ -246,6 +262,9 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
onUpdate({
hf_path: response.resolved_path,
seed_columns: response.columns,
seed_drop_columns: (config.seed_drop_columns ?? []).filter((name) =>
response.columns.includes(name),
),
seed_preview_rows: response.preview_rows ?? [],
hf_split: response.split ?? config.hf_split ?? "",
hf_subset: response.subset ?? config.hf_subset ?? "",
@ -273,6 +292,9 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
onUpdate({
hf_path: response.resolved_path,
seed_columns: response.columns,
seed_drop_columns: (config.seed_drop_columns ?? []).filter((name) =>
response.columns.includes(name),
),
seed_preview_rows: response.preview_rows ?? [],
hf_repo_id: "",
hf_subset: "",
@ -317,6 +339,9 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
onUpdate({
hf_path: response.resolved_path,
seed_columns: response.columns,
seed_drop_columns: (config.seed_drop_columns ?? []).filter((name) =>
response.columns.includes(name),
),
seed_preview_rows: response.preview_rows ?? [],
hf_repo_id: "",
hf_subset: "",
@ -364,6 +389,14 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
if (previewRows[0]) return Object.keys(previewRows[0]);
return [];
}, [config.seed_columns, previewRows]);
const selectedSeedDropColumns = useMemo(
() => (config.seed_drop_columns ?? []).filter((name) => name.trim().length > 0),
[config.seed_drop_columns],
);
const selectedSeedDropSet = useMemo(
() => new Set(selectedSeedDropColumns),
[selectedSeedDropColumns],
);
return (
<Tabs defaultValue="config" className="w-full">
@ -393,6 +426,7 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
hf_repo_id: event.target.value,
hf_path: "",
seed_columns: [],
seed_drop_columns: [],
seed_preview_rows: [],
})
}
@ -474,6 +508,7 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
onUpdate({
hf_path: "",
seed_columns: [],
seed_drop_columns: [],
seed_preview_rows: [],
local_file_name: file?.name ?? "",
});
@ -517,6 +552,7 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
onUpdate({
hf_path: "",
seed_columns: [],
seed_drop_columns: [],
seed_preview_rows: [],
unstructured_file_name: file?.name ?? "",
});
@ -547,6 +583,44 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
{inspectError && <p className="text-xs text-red-600">{inspectError}</p>}
{mode !== "unstructured" && (
<div className="space-y-2 rounded-xl corner-squircle border border-border/60 p-3">
<FieldLabel
label="Drop specific seed columns"
hint="Dropped columns stay usable in prompts/expressions but are omitted from final dataset."
/>
{previewColumns.length === 0 ? (
<p className="text-xs text-muted-foreground">
Load columns to select which seed fields to drop.
</p>
) : (
<div className="grid gap-2 sm:grid-cols-2">
{previewColumns.map((columnName) => {
const checked = selectedSeedDropSet.has(columnName);
return (
<label
key={columnName}
className="flex cursor-pointer items-center gap-2 rounded-md border border-border/60 px-2 py-1.5 text-xs"
>
<Checkbox
checked={checked}
onCheckedChange={(value) => {
const isChecked = value === true;
const next = isChecked
? Array.from(new Set([...selectedSeedDropColumns, columnName]))
: selectedSeedDropColumns.filter((name) => name !== columnName);
onUpdate({ seed_drop_columns: next });
}}
/>
<span className="truncate">{columnName}</span>
</label>
);
})}
</div>
)}
</div>
)}
<Collapsible open={advancedOpen} onOpenChange={setAdvancedOpen}>
<CollapsibleTrigger asChild={true}>
<button
@ -732,13 +806,42 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
</TableHeader>
<TableBody>
{previewRows.map((row, rowIdx) => (
<TableRow key={`row-${rowIdx}`}>
<TableRow
key={`row-${rowIdx}`}
className={cn(
previewColumns.some((col) =>
isExpandablePreviewValue(stringifyCell(row[col])),
) && "cursor-pointer hover:bg-primary/[0.06]",
expandedPreviewRows[rowIdx] && "bg-primary/[0.05]",
)}
onClick={() => {
const canExpand = previewColumns.some((col) =>
isExpandablePreviewValue(stringifyCell(row[col])),
);
if (!canExpand) {
return;
}
setExpandedPreviewRows((current) => ({
...current,
[rowIdx]: !current[rowIdx],
}));
}}
>
{previewColumns.map((col) => (
<TableCell
key={`${rowIdx}-${col}`}
className="max-w-[260px] whitespace-pre-wrap break-words text-xs"
>
{stringifyCell(row[col])}
{(() => {
const value = stringifyCell(row[col]);
const rowHasExpandableCell = previewColumns.some((columnName) =>
isExpandablePreviewValue(stringifyCell(row[columnName])),
);
const rowExpanded = Boolean(expandedPreviewRows[rowIdx]);
return rowHasExpandableCell && !rowExpanded
? truncatePreviewValue(value)
: value;
})()}
</TableCell>
))}
</TableRow>

1
studio/frontend/src/features/recipe-studio/stores/recipe-studio.ts
View file

@ -319,6 +319,7 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
local_file_name: "",
unstructured_file_name: "",
seed_columns: [],
seed_drop_columns: [],
seed_preview_rows: [],
unstructured_chunk_size: "1200",
unstructured_chunk_overlap: "200",

2
studio/frontend/src/features/recipe-studio/types/index.ts
View file

@ -220,6 +220,8 @@ export type SeedConfig = {
kind: "seed";
name: string;
drop?: boolean;
// ui-only: explicit per-column drop for structured seed sources (hf/local)
seed_drop_columns?: string[];
seed_source_type: SeedSourceType;
// ui-only (serialized in seed_config)
hf_repo_id: string;

1
studio/frontend/src/features/recipe-studio/utils/config-factories.ts
View file

@ -287,6 +287,7 @@ export function makeSeedConfig(
kind: "seed",
name: nextName(existing, "seed"),
drop: false,
seed_drop_columns: [],
seed_source_type: seedSourceType,
hf_repo_id: "",
hf_subset: "",

42
studio/frontend/src/features/recipe-studio/utils/import/importer.ts
View file

@ -32,6 +32,7 @@ type UiInput = {
edges?: unknown;
seed_source_type?: unknown;
seed_columns?: unknown;
seed_drop_columns?: unknown;
seed_preview_rows?: unknown;
local_file_name?: unknown;
unstructured_file_name?: unknown;
@ -83,6 +84,39 @@ function parseProcessors(input: unknown): RecipeProcessorConfig[] {
return processors;
}
function parseSeedDropColumns(input: unknown): string[] {
if (!Array.isArray(input)) {
return [];
}
const values = new Set<string>();
for (const item of input) {
if (!isRecord(item)) {
continue;
}
const type = readString(item.processor_type);
if (type !== "drop_columns") {
continue;
}
const name = readString(item.name);
if (name !== "drop_seed_columns") {
continue;
}
const columnNames = Array.isArray(item.column_names)
? item.column_names
: [];
for (const columnName of columnNames) {
if (typeof columnName !== "string") {
continue;
}
const next = columnName.trim();
if (next) {
values.add(next);
}
}
}
return Array.from(values);
}
function parseMcpProviders(
input: unknown,
): Map<string, LlmMcpProviderConfig> {
@ -249,6 +283,12 @@ export function importRecipePayload(input: string): ImportResult {
.map((value) => (typeof value === "string" ? value.trim() : ""))
.filter(Boolean)
: undefined;
const uiSeedDropColumns = Array.isArray(ui?.seed_drop_columns)
? ui.seed_drop_columns
.map((value) => (typeof value === "string" ? value.trim() : ""))
.filter(Boolean)
: undefined;
const payloadSeedDropColumns = parseSeedDropColumns(recipe.processors);
const uiSeedPreviewRows = Array.isArray(ui?.seed_preview_rows)
? ui.seed_preview_rows
.filter((row): row is Record<string, unknown> => isRecord(row))
@ -268,6 +308,8 @@ export function importRecipePayload(input: string): ImportResult {
const seedConfig = parseSeedConfig(recipe.seed_config, id, {
preferredSourceType: uiSeedSourceType,
seed_columns: uiSeedColumns,
seed_drop_columns:
uiSeedDropColumns ?? payloadSeedDropColumns,
seed_preview_rows: uiSeedPreviewRows,
local_file_name: uiLocalFileName,
unstructured_file_name: uiUnstructuredFileName,

5
studio/frontend/src/features/recipe-studio/utils/import/parsers/seed-config-parser.ts
View file

@ -18,6 +18,7 @@ function makeDefaultSeedConfig(id: string): SeedConfig {
kind: "seed",
name: "seed",
drop: false,
seed_drop_columns: [],
seed_source_type: "hf",
hf_repo_id: "",
hf_subset: "",
@ -133,6 +134,7 @@ export function parseSeedConfig(
options?: {
preferredSourceType?: SeedSourceType;
seed_columns?: string[];
seed_drop_columns?: string[];
seed_preview_rows?: Record<string, unknown>[];
local_file_name?: string;
unstructured_file_name?: string;
@ -157,6 +159,9 @@ export function parseSeedConfig(
...parsed, // payload-only fields override ui defaults
seed_source_type: sourceType,
...(options?.seed_columns ? { seed_columns: options.seed_columns } : {}),
...(options?.seed_drop_columns
? { seed_drop_columns: options.seed_drop_columns }
: {}),
...(options?.seed_preview_rows
? { seed_preview_rows: options.seed_preview_rows }
: {}),

3
studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts
View file

@ -240,6 +240,9 @@ export function buildRecipePayload(
edges: uiEdges,
...(firstSeed && { seed_source_type: firstSeed.seed_source_type }),
...(firstSeed && { seed_columns: firstSeed.seed_columns ?? [] }),
...(firstSeed && {
seed_drop_columns: firstSeed.seed_drop_columns ?? [],
}),
...(firstSeed && {
seed_preview_rows: firstSeed.seed_preview_rows ?? [],
}),

27
studio/frontend/src/features/recipe-studio/utils/payload/builders-seed.ts
View file

@ -80,13 +80,32 @@ export function buildSeedDropProcessor(
config: SeedConfig,
errors: string[],
): Record<string, unknown> | null {
if (!config.drop) {
return null;
const seedSourceType = config.seed_source_type ?? "hf";
const loadedCols = (config.seed_columns ?? []).map((c) => c.trim()).filter(Boolean);
let cols: string[] = [];
if (seedSourceType === "unstructured") {
if (!config.drop) {
return null;
}
cols = loadedCols;
} else {
const selectedDropColumns = (config.seed_drop_columns ?? [])
.map((c) => c.trim())
.filter(Boolean);
if (selectedDropColumns.length === 0) {
return null;
}
const loadedSet = new Set(loadedCols);
cols =
loadedCols.length > 0
? selectedDropColumns.filter((col) => loadedSet.has(col))
: selectedDropColumns;
}
const cols = (config.seed_columns ?? []).map((c) => c.trim()).filter(Boolean);
if (cols.length === 0) {
errors.push(
`Seed ${config.name}: drop enabled but no seed columns loaded.`,
`Seed ${config.name}: selected drop columns are unavailable.`,
);
return null;
}

1
studio/frontend/src/features/recipe-studio/utils/payload/types.ts
View file

@ -36,6 +36,7 @@ export type RecipePayload = {
seed_source_type?: "hf" | "local" | "unstructured";
// ui-only, seed metadata cached for refresh/import UX
seed_columns?: string[];
seed_drop_columns?: string[];
seed_preview_rows?: Record<string, unknown>[];
local_file_name?: string;
unstructured_file_name?: string;

13
studio/frontend/src/features/recipe-studio/utils/validation.ts
View file

@ -194,8 +194,17 @@ export function getConfigErrors(config: NodeConfig | null): string[] {
) {
errors.push("HF endpoint must start with http.");
}
if (config.drop && (config.seed_columns?.length ?? 0) === 0) {
errors.push("Seed drop needs loaded columns (open Seed Preview).");
if (seedSourceType === "unstructured") {
if (config.drop && (config.seed_columns?.length ?? 0) === 0) {
errors.push("Seed drop needs loaded columns.");
}
} else {
const selectedDropColumns = (config.seed_drop_columns ?? [])
.map((value) => value.trim())
.filter(Boolean);
if (selectedDropColumns.length > 0 && (config.seed_columns?.length ?? 0) === 0) {
errors.push("Seed drop columns need loaded columns.");
}
}
if (config.selection_type === "index_range") {