ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix per-block ID collisions and add block cleanup for unstructured uploads (#6944)

Browse source 
* unstructured block removal

* Enhance unstructured block handling

* Restrict block cleanup to upload UIDs

* cleanup for seed block uploads

* upload cleanup queue for unstructured blocks in recipe studio

* Fix unstructured upload cleanup edge cases

* Fix unstructured upload import ownership

* Fix-unstructured-import-path-ownership

* Guard failed-delete restore against stale block in unstructured drop zone

* Drain queued upload cleanups when autosave is skipped

---------

Co-authored-by: Lee Jackson <130007945+Imagineer99@users.noreply.github.com>
Co-authored-by: imagineer99 <samleejackson0@gmail.com>
Co-authored-by: Daniel Han <danielhanchen@gmail.com>
This commit is contained in:
Nilay 2026-07-09 08:33:03 +05:30 • committed by GitHub
commit 3b73cd8829
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 502 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

37
studio/backend/routes/data_recipe/seed.py
View file

@ -10,6 +10,7 @@ import binascii
import json
import os
import re
import shutil
from itertools import islice
from pathlib import Path
from typing import Any
@ -59,6 +60,9 @@ UNSTRUCTURED_ALLOWED_EXTS = {".pdf", ".docx", ".txt", ".md"}
SEED_UPLOAD_DIR = seed_uploads_root()
UNSTRUCTURED_UPLOAD_ROOT = unstructured_uploads_root()
_SAFE_ID_RE = re.compile(r"^[a-zA-Z0-9_-]+$")
# Frontend-generated upload namespace (UUID4 hex). Legacy node ids (n1, ...)
# never match: those directories can be shared by several recipes.
_UPLOAD_UID_RE = re.compile(r"^[0-9a-f]{32}$")
def _validate_safe_id(value: str, label: str) -> str:
@ -580,6 +584,39 @@ async def remove_unstructured_file(block_id: str, file_id: str):
return {"status": "ok"}
@router.delete("/seed/unstructured-block/{block_id}")
async def remove_unstructured_block(block_id: str):
"""Delete a block's upload directory; files on disk still count toward its quota.
Only uid-namespaced directories may be bulk-deleted: they have exactly one
owning block. Legacy node-id directories (n1, ...) can be shared by other
recipes, so they are managed file-by-file instead.
"""
_validate_safe_id(block_id, "block_id")
if not _UPLOAD_UID_RE.match(block_id):
raise HTTPException(400, "Invalid block_id: only uid-namespaced blocks can be deleted")
block_dir = (UNSTRUCTURED_UPLOAD_ROOT / block_id).resolve()
if not block_dir.is_relative_to(UNSTRUCTURED_UPLOAD_ROOT.resolve()):
raise HTTPException(400, "Invalid block_id: outside upload root")
if not block_dir.exists():
return {"status": "ok", "deleted": False}
try:
shutil.rmtree(block_dir)
except OSError as exc:
raise log_and_http_error(
exc,
500,
"failed to delete uploaded files",
event = "data_recipe.seed.unstructured_block_delete_failed",
log = logger,
) from exc
if block_dir.exists():
raise HTTPException(500, "failed to delete uploaded files")
return {"status": "ok", "deleted": True}
@router.post("/seed/inspect-upload", response_model = SeedInspectResponse)
def inspect_seed_upload(payload: SeedInspectUploadRequest) -> SeedInspectResponse:
if payload.file_ids is not None:

97
studio/backend/tests/test_data_recipe_seed.py
View file

@ -124,3 +124,100 @@ def test_unstructured_upload_import_errors_stay_generic(monkeypatch, tmp_path, e
assert result.status == "error"
assert result.error == "Text extraction failed."
assert _block_files(seed_route) == []
_TEST_UPLOAD_UID = "0f" * 16
def test_remove_unstructured_block_deletes_directory(monkeypatch, tmp_path):
seed_route = _load_seed_route(monkeypatch, tmp_path)
_run_upload(seed_route, "notes.txt", b"hello", block_id = _TEST_UPLOAD_UID)
assert _block_files(seed_route, _TEST_UPLOAD_UID) != []
result = asyncio.run(seed_route.remove_unstructured_block(_TEST_UPLOAD_UID))
assert result == {"status": "ok", "deleted": True}
assert not (seed_route.UNSTRUCTURED_UPLOAD_ROOT / _TEST_UPLOAD_UID).exists()
def test_remove_unstructured_block_missing_directory_is_ok(monkeypatch, tmp_path):
seed_route = _load_seed_route(monkeypatch, tmp_path)
result = asyncio.run(seed_route.remove_unstructured_block(_TEST_UPLOAD_UID))
assert result == {"status": "ok", "deleted": False}
def test_remove_unstructured_block_rejects_unsafe_ids(monkeypatch, tmp_path):
seed_route = _load_seed_route(monkeypatch, tmp_path)
with pytest.raises(seed_route.HTTPException) as exc:
asyncio.run(seed_route.remove_unstructured_block("../escape"))
assert exc.value.status_code == 400
def test_remove_unstructured_block_rejects_legacy_node_ids(monkeypatch, tmp_path):
seed_route = _load_seed_route(monkeypatch, tmp_path)
_run_upload(seed_route, "notes.txt", b"hello", block_id = "n1")
assert _block_files(seed_route, "n1") != []
with pytest.raises(seed_route.HTTPException) as exc:
asyncio.run(seed_route.remove_unstructured_block("n1"))
assert exc.value.status_code == 400
assert _block_files(seed_route, "n1") != []
def test_remove_unstructured_block_rejects_symlink_escape(monkeypatch, tmp_path):
seed_route = _load_seed_route(monkeypatch, tmp_path)
outside = tmp_path / "outside"
outside.mkdir()
(outside / "victim.txt").write_text("keep me")
root = seed_route.UNSTRUCTURED_UPLOAD_ROOT
root.mkdir(parents = True)
(root / _TEST_UPLOAD_UID).symlink_to(outside)
with pytest.raises(seed_route.HTTPException) as exc:
asyncio.run(seed_route.remove_unstructured_block(_TEST_UPLOAD_UID))
assert exc.value.status_code == 400
assert (outside / "victim.txt").exists()
def test_remove_unstructured_block_fails_if_directory_remains(monkeypatch, tmp_path):
seed_route = _load_seed_route(monkeypatch, tmp_path)
root = seed_route.UNSTRUCTURED_UPLOAD_ROOT
block_dir = root / _TEST_UPLOAD_UID
block_dir.mkdir(parents = True)
(block_dir / "victim.txt").write_text("keep me")
calls = []
def noop_rmtree(path, *args, **kwargs):
calls.append((path, args, kwargs))
monkeypatch.setattr(seed_route.shutil, "rmtree", noop_rmtree)
with pytest.raises(seed_route.HTTPException) as exc:
asyncio.run(seed_route.remove_unstructured_block(_TEST_UPLOAD_UID))
assert calls
assert exc.value.status_code == 500
assert block_dir.exists()
def test_total_upload_quota_is_scoped_per_block(monkeypatch, tmp_path):
seed_route = _load_seed_route(monkeypatch, tmp_path)
monkeypatch.setattr(seed_route, "UNSTRUCTURED_RECIPE_UPLOAD_TOTAL_MAX_BYTES", 10)
first = _run_upload(seed_route, "a.txt", b"123456789")
assert first.status == "ok"
with pytest.raises(seed_route.HTTPException) as exc:
_run_upload(seed_route, "b.txt", b"123")
assert exc.value.status_code == 413
# Another block starts with its own untouched budget.
other = _run_upload(seed_route, "c.txt", b"123", block_id = "other")
assert other.status == "ok"

10
studio/frontend/src/features/recipe-studio/api/index.ts
View file

@ -494,3 +494,13 @@ export async function removeUnstructuredFile(
throw new Error("Failed to remove file");
}
}
export async function removeUnstructuredBlock(blockId: string): Promise<void> {
const res = await authFetch(
`${DATA_DESIGNER_API_BASE}/seed/unstructured-block/${encodeURIComponent(blockId)}`,
{ method: "DELETE" },
);
if (!res.ok && res.status !== 404) {
throw new Error("Failed to remove uploaded files");
}
}

69
studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx
View file

@ -53,6 +53,11 @@ import {
inspectSeedDataset,
inspectSeedUpload,
} from "../../api";
import { useRecipeStudioStore } from "../../stores/recipe-studio";
import {
makeUnstructuredUploadUid,
resolveUnstructuredUploadBlockId,
} from "../../utils/config-factories";
import { resolveImagePreview } from "../../utils/image-preview";
import type {
GithubItemType,
@ -597,6 +602,41 @@ export function SeedDialog({
const mode = config.seed_source_type ?? "hf";
const previewEmpty = getPreviewEmptyStateCopy(mode);
const queueUploadCleanup = useRecipeStudioStore(
(state) => state.queueUploadCleanup,
);
// config.id collides across recipes (ids reset to n1 on import); use a
// stable per-block uid instead. Generate one synchronously so the first
// rendered drop zone cannot upload under a legacy node id.
const uploadUid = config.unstructured_upload_uid?.trim() ?? "";
const unstructuredFileCount = config.unstructured_file_ids?.length ?? 0;
const generatedUploadUidRef = useRef<string | null>(null);
if (
mode === "unstructured" &&
!uploadUid &&
unstructuredFileCount === 0 &&
generatedUploadUidRef.current === null
) {
generatedUploadUidRef.current = makeUnstructuredUploadUid();
}
const uploadBlockId = resolveUnstructuredUploadBlockId({
configId: config.id,
uploadUid,
generatedUploadUid: generatedUploadUidRef.current,
unstructuredFileCount,
});
useEffect(() => {
if (mode !== "unstructured") return;
if (uploadUid) return;
if (unstructuredFileCount > 0) return;
const nextUid =
generatedUploadUidRef.current ?? makeUnstructuredUploadUid();
generatedUploadUidRef.current = nextUid;
onUpdate({ unstructured_upload_uid: nextUid });
}, [mode, uploadUid, unstructuredFileCount, onUpdate]);
const prevModeRef = useRef(mode);
useEffect(() => {
const prevMode = prevModeRef.current;
@ -720,6 +760,11 @@ export function SeedDialog({
subset: config.hf_subset?.trim() || undefined,
preview_size: 10,
});
// Queue the block's upload directory for deletion after the next
// save; only uid-namespaced directories qualify (single owner).
if (uploadUid && unstructuredFileCount > 0) {
queueUploadCleanup(uploadUid);
}
onUpdate({
hf_path: response.resolved_path,
seed_columns: response.columns,
@ -730,6 +775,7 @@ export function SeedDialog({
hf_split: response.split ?? "",
hf_subset: response.subset ?? "",
local_file_name: "",
unstructured_upload_uid: "",
unstructured_file_ids: [],
unstructured_file_names: [],
unstructured_file_sizes: [],
@ -754,6 +800,11 @@ export function SeedDialog({
content_base64: payload,
preview_size: 10,
});
// Queue the block's upload directory for deletion after the next
// save; only uid-namespaced directories qualify (single owner).
if (uploadUid && unstructuredFileCount > 0) {
queueUploadCleanup(uploadUid);
}
onUpdate({
hf_path: response.resolved_path,
seed_columns: response.columns,
@ -765,6 +816,7 @@ export function SeedDialog({
hf_subset: "",
hf_split: "",
local_file_name: localFile.name,
unstructured_upload_uid: "",
unstructured_file_ids: [],
unstructured_file_names: [],
unstructured_file_sizes: [],
@ -789,7 +841,7 @@ export function SeedDialog({
const { chunkSize, chunkOverlap } = resolveChunking(config);
const response = await inspectSeedUpload({
block_id: config.id,
block_id: uploadBlockId,
file_ids: fileIds,
file_names: fileNames,
preview_size: 10,
@ -827,7 +879,18 @@ export function SeedDialog({
setIsInspecting(false);
}
},
[config, getCurrentLoadKey, localFile, mode, onUpdate, unstructuredFiles],
[
config,
getCurrentLoadKey,
localFile,
mode,
onUpdate,
queueUploadCleanup,
unstructuredFiles,
unstructuredFileCount,
uploadBlockId,
uploadUid,
],
);
useEffect(() => {
@ -997,7 +1060,7 @@ export function SeedDialog({
{mode === "unstructured" && (
<UnstructuredDropZone
blockId={config.id}
blockId={uploadBlockId}
files={unstructuredFiles}
onFilesChange={handleUnstructuredFilesChange}
disabled={isInspecting}

37
studio/frontend/src/features/recipe-studio/dialogs/seed/unstructured-drop-zone.tsx
View file

@ -54,11 +54,17 @@ export function UnstructuredDropZone({
}: UnstructuredDropZoneProps) {
const inputRef = useRef<HTMLInputElement>(null);
const filesRef = useRef(files);
const blockIdRef = useRef(blockId);
const mountedRef = useRef(true);
const [isDragOver, setIsDragOver] = useState(false);
useEffect(() => {
filesRef.current = files;
}, [files]);
blockIdRef.current = blockId;
}, [files, blockId]);
useEffect(() => () => {
mountedRef.current = false;
}, []);
const totalSize = files.reduce((sum, f) => sum + f.size, 0);
@ -134,15 +140,32 @@ export function UnstructuredDropZone({
if (entry.status === "uploading" && entry.abortController) {
entry.abortController.abort();
}
if (
const needsServerRemove =
entry.id &&
entry.status === "ok" &&
!deletedIdsRef.current.has(entry.id)
) {
deletedIdsRef.current.add(entry.id);
void removeUnstructuredFile(blockId, entry.id).catch(() => {});
}
!deletedIdsRef.current.has(entry.id);
onFilesChange((prev) => prev.filter((_, i) => i !== index));
if (!needsServerRemove) return;
deletedIdsRef.current.add(entry.id);
removeUnstructuredFile(blockId, entry.id).catch(() => {
// Skip if the drop zone unmounted or its block changed: the id no
// longer belongs here and restoring would leak it into another block.
if (!mountedRef.current || blockIdRef.current !== blockId) return;
// Still exists server-side (counts toward quota); restore it at its
// original position.
deletedIdsRef.current.delete(entry.id);
onFilesChange((prev) => {
const next = [...prev];
next.splice(Math.min(index, next.length), 0, {
id: entry.id,
name: entry.name,
size: entry.size,
status: "ok",
error: "Remove failed — try again",
});
return next;
});
});
},
[blockId, onFilesChange],
);

86
studio/frontend/src/features/recipe-studio/hooks/use-recipe-persistence.ts
View file

@ -4,11 +4,13 @@
import { useCallback, useEffect, useMemo, useState } from "react";
import { toastError, toastSuccess } from "@/shared/toast";
import { normalizeNonEmptyName } from "@/utils";
import { removeUnstructuredBlock } from "../api";
import {
buildSignature,
copyTextToClipboard,
formatSavedLabel,
} from "../executions/execution-helpers";
import { useRecipeStudioStore } from "../stores/recipe-studio";
import { importRecipePayload, type RecipeSnapshot } from "../utils/import";
import type { RecipePayloadResult } from "../utils/payload/types";
@ -72,7 +74,10 @@ function stripApiKeys(value: unknown): unknown {
!Array.isArray(output.env)
) {
output.env = Object.fromEntries(
Object.keys(output.env as Record<string, unknown>).map((envKey) => [envKey, ""]),
Object.keys(output.env as Record<string, unknown>).map((envKey) => [
envKey,
"",
]),
);
}
return output;
@ -82,10 +87,7 @@ function inferHfRepoIdFromPath(pathValue: unknown): string {
if (typeof pathValue !== "string") {
return "";
}
const parts = pathValue
.trim()
.split("/")
.filter(Boolean);
const parts = pathValue.trim().split("/").filter(Boolean);
if (parts.length >= 3 && parts[0] === "datasets") {
return `${parts[1]}/${parts[2]}`;
}
@ -126,8 +128,7 @@ function sanitizeSeedForShare(payload: unknown): unknown {
typeof ui?.seed_source_type === "string" ? ui.seed_source_type : null;
const sourceType =
typeof source?.seed_type === "string" ? source.seed_type : null;
const shouldResetHfState =
sourceType === "hf" || uiSourceType === "hf";
const shouldResetHfState = sourceType === "hf" || uiSourceType === "hf";
const shouldResetLocalState =
sourceType === "local" ||
sourceType === "unstructured" ||
@ -144,6 +145,7 @@ function sanitizeSeedForShare(payload: unknown): unknown {
ui.seed_drop_columns = [];
ui.seed_preview_rows = [];
ui.local_file_name = "";
ui.unstructured_upload_uid = "";
ui.unstructured_file_ids = [];
ui.unstructured_file_names = [];
ui.unstructured_file_sizes = [];
@ -165,6 +167,7 @@ function sanitizeSeedForShare(payload: unknown): unknown {
ui.seed_drop_columns = [];
ui.seed_preview_rows = [];
ui.local_file_name = "";
ui.unstructured_upload_uid = "";
ui.unstructured_file_ids = [];
ui.unstructured_file_names = [];
ui.unstructured_file_sizes = [];
@ -174,6 +177,43 @@ function sanitizeSeedForShare(payload: unknown): unknown {
return root;
}
// Delete queued upload directories once a save stops referencing them, so a
// reload before autosave can never leave the saved recipe pointing at
// already-deleted files. Skips any uid the just-saved payload still uses.
function drainQueuedUploadCleanups(
savedPayload: RecipePayloadResult["payload"],
): void {
const pending = useRecipeStudioStore.getState().pendingUploadCleanups;
if (pending.length === 0) {
return;
}
const ui =
savedPayload && typeof savedPayload === "object"
? (savedPayload as { ui?: Record<string, unknown> }).ui
: undefined;
const savedUid =
ui && typeof ui.unstructured_upload_uid === "string"
? ui.unstructured_upload_uid
: "";
const ready = pending.filter((uid) => uid !== savedUid);
if (ready.length === 0) {
return;
}
for (const uid of ready) {
void removeUnstructuredBlock(uid)
.then(() => {
useRecipeStudioStore.setState((state) => ({
pendingUploadCleanups: state.pendingUploadCleanups.filter(
(pendingUid) => pendingUid !== uid,
),
}));
})
.catch((error) => {
console.warn("Failed to clean up uploaded documents:", error);
});
}
}
export function useRecipePersistence({
recipeId,
initialRecipeName,
@ -202,8 +242,10 @@ export function useRecipePersistence({
() => buildSignature(normalizedWorkflowName, currentPayload),
[currentPayload, normalizedWorkflowName],
);
const isDirty = savedSignature.length > 0 && currentSignature !== savedSignature;
const saveTone: SaveTone = !isDirty && Boolean(lastSavedAt) ? "success" : "error";
const isDirty =
savedSignature.length > 0 && currentSignature !== savedSignature;
const saveTone: SaveTone =
!isDirty && Boolean(lastSavedAt) ? "success" : "error";
const savedAtLabel = formatSavedLabel(lastSavedAt);
useEffect(() => {
@ -214,7 +256,9 @@ export function useRecipePersistence({
setLastSavedAt(initialSavedAt);
setCopied(false);
const parsed = importRecipePayload(JSON.stringify(initialPayload));
const parsed = importRecipePayload(JSON.stringify(initialPayload), {
preserveUnstructuredUploads: true,
});
if (parsed.snapshot) {
loadRecipe(parsed.snapshot);
} else {
@ -252,6 +296,7 @@ export function useRecipePersistence({
});
setLastSavedAt(result.updatedAt);
setSavedSignature(buildSignature(nextName, currentPayload));
drainQueuedUploadCleanups(currentPayload);
} catch (error) {
console.error("Save recipe failed:", error);
toastError("Save failed", "Could not save recipe.");
@ -270,11 +315,28 @@ export function useRecipePersistence({
return () => window.clearTimeout(timeoutId);
}, [isDirty, persistRecipe, saveLoading]);
// Drain queued cleanups even when autosave is skipped: a net-zero edit (add
// then remove an unstructured seed before the 800ms debounce) keeps isDirty
// false, so the autosave effect never drains and the queued uid leaks its
// upload dir. Not-dirty means currentPayload equals the saved recipe, and
// drain skips the uid it still references, so only dirs no saved recipe
// points at are deleted (keeps the save-first invariant).
useEffect(() => {
if (!initialRecipeReady || isDirty || saveLoading) {
return;
}
drainQueuedUploadCleanups(currentPayload);
}, [currentPayload, initialRecipeReady, isDirty, saveLoading]);
const copyRecipe = useCallback(async (): Promise<void> => {
setCopied(false);
try {
const safePayload = sanitizeSeedForShare(stripApiKeys(payloadResult.payload));
const ok = await copyTextToClipboard(JSON.stringify(safePayload, null, 2));
const safePayload = sanitizeSeedForShare(
stripApiKeys(payloadResult.payload),
);
const ok = await copyTextToClipboard(
JSON.stringify(safePayload, null, 2),
);
if (!ok) {
throw new Error("Clipboard not available.");
}

61
studio/frontend/src/features/recipe-studio/stores/recipe-studio.ts
View file

@ -36,6 +36,7 @@ import {
} from "../utils/handles";
import type { RecipeSnapshot } from "../utils/import";
import { getLayoutedElements } from "../utils/layout";
import { makeUnstructuredUploadUid } from "../utils/config-factories";
import {
centerModelInfraNodes,
optimizeModelInfraEdgeHandles,
@ -76,6 +77,12 @@ type RecipeStudioState = {
nextId: number;
nextY: number;
fitViewTick: number;
// Upload-uid directories whose owning block dropped them; server-side
// deletion is deferred until a save no longer references them, so a
// reload before autosave cannot leave a saved recipe pointing at
// deleted files.
pendingUploadCleanups: string[];
queueUploadCleanup: (uid: string) => void;
setSheetOpen: (open: boolean) => void;
setSheetView: (view: SheetView) => void;
setProcessors: (processors: RecipeProcessorConfig[]) => void;
@ -137,6 +144,7 @@ const INITIAL_STATE = {
nextId: 3,
nextY: 280,
fitViewTick: 0,
pendingUploadCleanups: [],
} satisfies Pick<
RecipeStudioState,
| "nodes"
@ -154,6 +162,7 @@ const INITIAL_STATE = {
| "nextId"
| "nextY"
| "fitViewTick"
| "pendingUploadCleanups"
>;
function buildAddedNodeState(
@ -269,6 +278,20 @@ function isModelSemanticEdge(
);
}
// Upload uid of a seed block whose server-side directory becomes orphaned
// when the block drops it. Only uid directories qualify (single owner);
// legacy node-id directories can be shared by other recipes.
function seedUploadCleanupUid(config: NodeConfig | undefined): string | null {
if (!config || config.kind !== "seed") {
return null;
}
const uid = config.unstructured_upload_uid?.trim();
if (!uid || !config.unstructured_file_ids?.length) {
return null;
}
return uid;
}
export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
...INITIAL_STATE,
setSheetOpen: (open) => set({ sheetOpen: open }),
@ -278,6 +301,12 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
setDialogOpen: (open) => set({ dialogOpen: open }),
setExecutionLocked: (locked) => set({ executionLocked: locked }),
resetRecipe: () => set(INITIAL_STATE),
queueUploadCleanup: (uid) =>
set((state) =>
state.pendingUploadCleanups.includes(uid)
? state
: { pendingUploadCleanups: [...state.pendingUploadCleanups, uid] },
),
selectConfig: (id) => set({ activeConfigId: id, dialogOpen: false }),
openConfig: (id) => set({ activeConfigId: id, dialogOpen: true }),
setLayoutDirection: (direction) =>
@ -383,7 +412,18 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
}
return buildAddedNodeState(state, "sampler", type, position, openDialog);
}),
addSeedNode: (type, position, openDialog = true) =>
addSeedNode: (type, position, openDialog = true) => {
const current = get();
if (!current.executionLocked) {
// The reset below clears the block's upload uid and file list; queue
// its server-side directory for deletion after the next save.
const uid = seedUploadCleanupUid(
Object.values(current.configs).find((config) => config.kind === "seed"),
);
if (uid) {
current.queueUploadCleanup(uid);
}
}
set((state) => {
if (state.executionLocked) {
return state;
@ -413,6 +453,8 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
hf_token: "",
hf_endpoint: "https://huggingface.co",
local_file_name: "",
unstructured_upload_uid:
nextSourceType === "unstructured" ? makeUnstructuredUploadUid() : "",
unstructured_file_ids: [],
unstructured_file_names: [],
unstructured_file_sizes: [],
@ -446,7 +488,8 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
activeConfigId: existing.id,
dialogOpen: openDialog,
};
}),
});
},
addLlmNode: (type, position, openDialog = true) =>
set((state) => {
if (state.executionLocked) {
@ -699,6 +742,9 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
dialogOpen: false,
sheetView: "root",
fitViewTick: state.fitViewTick + 1,
// Queued cleanups belong to the previous recipe; draining them after
// a save of this one could delete files its saved payload still uses.
pendingUploadCleanups: [],
})),
setAuxNodePosition: (id, position) =>
set((state) => {
@ -786,6 +832,17 @@ export const useRecipeStudioStore = create<RecipeStudioState>((set, get) => ({
set(applyUpdate);
},
onNodesChange: (changes) => {
const current = get();
if (!current.executionLocked) {
for (const change of changes) {
if (change.type === "remove") {
const uid = seedUploadCleanupUid(current.configs[change.id]);
if (uid) {
current.queueUploadCleanup(uid);
}
}
}
}
const applyNodesChange = (state: RecipeStudioState) => {
if (state.executionLocked) {
return state;

2
studio/frontend/src/features/recipe-studio/types/index.ts
View file

@ -340,6 +340,8 @@ export type SeedConfig = {
hf_token?: string;
hf_endpoint?: string;
local_file_name?: string;
// ui-only: stable per-block id for uploads, since node ids collide across imports
unstructured_upload_uid?: string;
unstructured_file_ids?: string[];
unstructured_file_names?: string[];
unstructured_file_sizes?: number[];

43
studio/frontend/src/features/recipe-studio/utils/config-factories.ts
View file

@ -20,6 +20,46 @@ import type {
} from "../types";
import { nextName } from "./naming";
export function makeUnstructuredUploadUid(): string {
if (typeof globalThis.crypto?.randomUUID === "function") {
return globalThis.crypto.randomUUID().replace(/-/g, "").toLowerCase();
}
if (typeof globalThis.crypto?.getRandomValues === "function") {
const bytes = new Uint8Array(16);
globalThis.crypto.getRandomValues(bytes);
return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(
"",
);
}
let uid = "";
while (uid.length < 32) {
uid += Math.floor(Math.random() * 0x100000000)
.toString(16)
.padStart(8, "0");
}
return uid.slice(0, 32);
}
export function resolveUnstructuredUploadBlockId({
configId,
uploadUid,
generatedUploadUid,
unstructuredFileCount,
}: {
configId: string;
uploadUid: string;
generatedUploadUid: string | null;
unstructuredFileCount: number;
}): string {
if (uploadUid) {
return uploadUid;
}
if (generatedUploadUid) {
return generatedUploadUid;
}
return unstructuredFileCount > 0 ? configId : "";
}
export function makeSamplerConfig(
id: string,
samplerType: SamplerType,
@ -368,6 +408,9 @@ export function makeSeedConfig(
hf_token: "",
hf_endpoint: "https://huggingface.co",
local_file_name: "",
...(seedSourceType === "unstructured"
? { unstructured_upload_uid: makeUnstructuredUploadUid() }
: {}),
unstructured_file_ids: [],
unstructured_file_names: [],
unstructured_file_sizes: [],

112
studio/frontend/src/features/recipe-studio/utils/import/importer.ts
View file

@ -16,11 +16,7 @@ import type {
} from "../../types";
import { buildEdges } from "./edges";
import { isRecord, parseJson, readString } from "./helpers";
import {
parseColumn,
parseModelConfig,
parseModelProvider,
} from "./parsers";
import { parseColumn, parseModelConfig, parseModelProvider } from "./parsers";
import { parseSeedConfig } from "./parsers/seed-config-parser";
import { buildNodes, parseUi } from "./ui";
import type { ImportResult } from "./types";
@ -43,6 +39,7 @@ type UiInput = {
seed_drop_columns?: unknown;
seed_preview_rows?: unknown;
local_file_name?: unknown;
unstructured_upload_uid?: unknown;
unstructured_file_ids?: unknown;
unstructured_file_names?: unknown;
unstructured_file_sizes?: unknown;
@ -51,6 +48,10 @@ type UiInput = {
advanced_open_by_node?: unknown;
};
type ImportRecipePayloadOptions = {
preserveUnstructuredUploads?: boolean;
};
type UiMarkdownNoteNode = {
name: string;
markdown: string;
@ -90,7 +91,7 @@ function parseProcessors(input: unknown): RecipeProcessorConfig[] {
? templateRaw
: isRecord(templateRaw)
? JSON.stringify(templateRaw, null, 2)
: "{\n \"text\": \"{{ column_name }}\"\n}";
: '{\n "text": "{{ column_name }}"\n}';
processors.push({
id: `p${index + 1}`,
// biome-ignore lint/style/useNamingConvention: api schema
@ -135,9 +136,7 @@ function parseSeedDropColumns(input: unknown): string[] {
return Array.from(values);
}
function parseMcpProviders(
input: unknown,
): Map<string, LlmMcpProviderConfig> {
function parseMcpProviders(input: unknown): Map<string, LlmMcpProviderConfig> {
const providers = new Map<string, LlmMcpProviderConfig>();
if (!Array.isArray(input)) {
return providers;
@ -156,13 +155,12 @@ function parseMcpProviders(
const args = Array.isArray(item.args)
? item.args.map((value) => String(value))
: [];
const envPairs =
isRecord(item.env)
? Object.entries(item.env).map(([key, value]) => ({
key: String(key),
value: String(value),
}))
: [];
const envPairs = isRecord(item.env)
? Object.entries(item.env).map(([key, value]) => ({
key: String(key),
value: String(value),
}))
: [];
providers.set(name, {
id: `mcp-${index + 1}`,
name,
@ -209,7 +207,8 @@ function parseToolConfigs(input: unknown): Map<string, LlmToolConfig> {
allow_tools: allowTools,
// biome-ignore lint/style/useNamingConvention: api schema
max_tool_call_turns:
item.max_tool_call_turns === null || item.max_tool_call_turns === undefined
item.max_tool_call_turns === null ||
item.max_tool_call_turns === undefined
? "5"
: String(item.max_tool_call_turns),
// biome-ignore lint/style/useNamingConvention: api schema
@ -257,7 +256,9 @@ function parseUiMarkdownNoteNodes(input: unknown): UiMarkdownNoteNode[] {
return noteNodes;
}
function parseUiToolProfileNodes(input: unknown): Map<string, Record<string, string[]>> {
function parseUiToolProfileNodes(
input: unknown,
): Map<string, Record<string, string[]>> {
const toolProfiles = new Map<string, Record<string, string[]>>();
if (!Array.isArray(input)) {
return toolProfiles;
@ -312,9 +313,15 @@ function parseAdvancedOpenByNode(input: unknown): Record<string, boolean> {
return out;
}
type AdvancedOpenConfig = LlmConfig | SamplerConfig | SeedConfig | ValidatorConfig;
type AdvancedOpenConfig =
| LlmConfig
| SamplerConfig
| SeedConfig
| ValidatorConfig;
function isAdvancedOpenConfig(config: NodeConfig): config is AdvancedOpenConfig {
function isAdvancedOpenConfig(
config: NodeConfig,
): config is AdvancedOpenConfig {
return (
config.kind === "llm" ||
config.kind === "sampler" ||
@ -350,7 +357,8 @@ function buildToolProfileConfig(
.map((providerName) => mcpProvidersByName.get(providerName))
.flatMap((provider) => (provider ? [cloneMcpProvider(provider)] : [])),
// biome-ignore lint/style/useNamingConvention: ui schema
fetched_tools_by_provider: fetchedToolsByProfileName.get(canonical.tool_alias) ?? {},
fetched_tools_by_provider:
fetchedToolsByProfileName.get(canonical.tool_alias) ?? {},
// biome-ignore lint/style/useNamingConvention: api schema
allow_tools: [...(canonical.allow_tools ?? [])],
// biome-ignore lint/style/useNamingConvention: api schema
@ -360,7 +368,10 @@ function buildToolProfileConfig(
};
}
export function importRecipePayload(input: string): ImportResult {
export function importRecipePayload(
input: string,
options: ImportRecipePayloadOptions = {},
): ImportResult {
const parsed = parseJson(input);
if (!parsed.data || !isRecord(parsed.data)) {
return {
@ -369,9 +380,9 @@ export function importRecipePayload(input: string): ImportResult {
};
}
const recipe = (isRecord(parsed.data.recipe)
? parsed.data.recipe
: parsed.data) as RecipeInput;
const recipe = (
isRecord(parsed.data.recipe) ? parsed.data.recipe : parsed.data
) as RecipeInput;
const ui = isRecord(parsed.data.ui) ? (parsed.data.ui as UiInput) : null;
if (!Array.isArray(recipe.columns)) {
@ -410,21 +421,36 @@ export function importRecipePayload(input: string): ImportResult {
.map((row) => ({ ...row }))
: undefined;
const uiLocalFileName = readString(ui?.local_file_name) ?? undefined;
// Preserve file IDs/names from saved recipes (cleared at share time by sanitizeSeedForShare)
const uiUnstructuredFileIds: string[] = Array.isArray(ui?.unstructured_file_ids)
? (ui.unstructured_file_ids as string[]).filter((v): v is string => typeof v === "string")
: [];
const uiUnstructuredFileNames: string[] = Array.isArray(ui?.unstructured_file_names)
? (ui.unstructured_file_names as string[]).filter((v): v is string => typeof v === "string")
: [];
const uiUnstructuredFileSizes: number[] = Array.isArray(ui?.unstructured_file_sizes)
? (ui.unstructured_file_sizes as number[]).filter((v): v is number => typeof v === "number")
: [];
const preserveUnstructuredUploads =
options.preserveUnstructuredUploads === true;
const uiUnstructuredUploadUid = preserveUnstructuredUploads
? (readString(ui?.unstructured_upload_uid) ?? undefined)
: undefined;
const uiUnstructuredFileIds: string[] =
preserveUnstructuredUploads && Array.isArray(ui?.unstructured_file_ids)
? (ui.unstructured_file_ids as string[]).filter(
(v): v is string => typeof v === "string",
)
: [];
const uiUnstructuredFileNames: string[] =
preserveUnstructuredUploads && Array.isArray(ui?.unstructured_file_names)
? (ui.unstructured_file_names as string[]).filter(
(v): v is string => typeof v === "string",
)
: [];
const uiUnstructuredFileSizes: number[] =
preserveUnstructuredUploads && Array.isArray(ui?.unstructured_file_sizes)
? (ui.unstructured_file_sizes as number[]).filter(
(v): v is number => typeof v === "number",
)
: [];
const uiUnstructuredChunkSize = readStringNumber(ui?.unstructured_chunk_size);
const uiUnstructuredChunkOverlap = readStringNumber(
ui?.unstructured_chunk_overlap,
);
const uiAdvancedOpenByNode = parseAdvancedOpenByNode(ui?.advanced_open_by_node);
const uiAdvancedOpenByNode = parseAdvancedOpenByNode(
ui?.advanced_open_by_node,
);
const uiMarkdownNotes = parseUiMarkdownNoteNodes(ui?.nodes);
const uiToolProfilesByName = parseUiToolProfileNodes(ui?.nodes);
@ -459,11 +485,13 @@ export function importRecipePayload(input: string): ImportResult {
: payloadSeedDropColumns,
seed_preview_rows: uiSeedPreviewRows,
local_file_name: uiLocalFileName,
unstructuredUploadUid: uiUnstructuredUploadUid,
unstructuredFileIds: uiUnstructuredFileIds,
unstructuredFileNames: uiUnstructuredFileNames,
unstructuredFileSizes: uiUnstructuredFileSizes,
unstructured_chunk_size: uiUnstructuredChunkSize,
unstructured_chunk_overlap: uiUnstructuredChunkOverlap,
preserveUnstructuredUploads,
});
if (seedConfig) {
applyAdvancedOpen(seedConfig, uiAdvancedOpenByNode);
@ -567,12 +595,7 @@ export function importRecipePayload(input: string): ImportResult {
const { layouts, auxNodes, edges: uiEdges, layoutDirection } = parseUi(ui);
const resolvedLayoutDirection = layoutDirection ?? "LR";
const nodes = buildNodes(configs, layouts);
const edges = buildEdges(
configs,
nameToId,
uiEdges,
resolvedLayoutDirection,
);
const edges = buildEdges(configs, nameToId, uiEdges, resolvedLayoutDirection);
const auxNodePositions = Object.fromEntries(
auxNodes.flatMap((item) => {
const llmId = nameToId.get(item.llm);
@ -583,10 +606,7 @@ export function importRecipePayload(input: string): ImportResult {
}),
);
const maxY = nodes.reduce(
(acc, node) => Math.max(acc, node.position.y),
0,
);
const maxY = nodes.reduce((acc, node) => Math.max(acc, node.position.y), 0);
return {
errors: [],

14
studio/frontend/src/features/recipe-studio/utils/import/parsers/seed-config-parser.ts
View file

@ -197,17 +197,26 @@ export function parseSeedConfig(
seed_drop_columns?: string[];
seed_preview_rows?: Record<string, unknown>[];
local_file_name?: string;
unstructuredUploadUid?: string;
unstructuredFileIds?: string[];
unstructuredFileNames?: string[];
unstructuredFileSizes?: number[];
unstructured_chunk_size?: string;
unstructured_chunk_overlap?: string;
preserveUnstructuredUploads?: boolean;
},
): SeedConfig | null {
if (!seedConfigRaw) {
return null;
}
const parsed = parseSeedSettings(seedConfigRaw);
const parsed = { ...parseSeedSettings(seedConfigRaw) };
if (
parsed.seed_source_type === "unstructured" &&
options?.preserveUnstructuredUploads !== true
) {
parsed.hf_path = "";
parsed.resolved_paths = [];
}
let sourceType: SeedSourceType = "hf";
if (parsed.seed_source_type === "hf") {
sourceType = "hf";
@ -230,6 +239,9 @@ export function parseSeedConfig(
...(options?.local_file_name !== undefined
? { local_file_name: options.local_file_name }
: {}),
...(options?.unstructuredUploadUid
? { unstructured_upload_uid: options.unstructuredUploadUid }
: {}),
...(options?.unstructuredFileIds !== undefined
? { unstructured_file_ids: options.unstructuredFileIds }
: {}),

3
studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts
View file

@ -440,6 +440,9 @@ export function buildRecipePayload(
unstructured_file_names: firstSeed.unstructured_file_names,
unstructured_file_sizes: firstSeed.unstructured_file_sizes,
}),
...(firstSeed?.unstructured_upload_uid?.trim() && {
unstructured_upload_uid: firstSeed.unstructured_upload_uid,
}),
...(firstSeed &&
firstSeed.unstructured_chunk_size !== undefined && {
unstructured_chunk_size: firstSeed.unstructured_chunk_size,

2
studio/frontend/src/features/recipe-studio/utils/payload/types.ts
View file

@ -71,6 +71,8 @@ export type RecipePayload = {
seed_preview_rows?: Record<string, unknown>[];
local_file_name?: string;
// biome-ignore lint/style/useNamingConvention: api schema
unstructured_upload_uid?: string;
// biome-ignore lint/style/useNamingConvention: api schema
unstructured_file_ids?: string[];
// biome-ignore lint/style/useNamingConvention: api schema
unstructured_file_names?: string[];