From dd283b0605669741d7545bfd2619ec77bc3c166b Mon Sep 17 00:00:00 2001 From: Wasim Yousef Said Date: Fri, 20 Mar 2026 21:22:42 +0100 Subject: [PATCH] feat(studio): multi-file unstructured seed upload with better backend extraction (#4468) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(recipe-studio): prevent fitView from zooming to wrong location on recipe load * feat: add pymupdf/python-docx deps and unstructured uploads storage root * feat: add POST /seed/upload-unstructured-file endpoint * feat: add multi-file chunking with source_file column * feat: update frontend types and API layer for multi-file upload * feat: round-robin preview rows across source files Ensures every uploaded file is represented in the preview table by cycling through sources instead of just taking the first N rows. * fix: disable OCR, fix auto-load timing, fix persistence on reload - Disable pymupdf4llm OCR with write_images=False, show_progress=False - Replace onAllUploaded callback with useEffect that detects uploading→done transition (avoids stale closure reading empty file IDs) - Fix importer to preserve file IDs from saved recipes instead of clearing (clearing only happens at share time via sanitizeSeedForShare) * fix: harden unstructured upload with input validation and state fixes Validate block_id/file_id with alphanumeric regex to prevent path traversal, use exact stem match for file deletion, add error handling for metadata writes and empty files, fix React stale closures and object mutations in upload loop, and correct validation logic for unstructured seed resolved_paths. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix: address PR review - legacy path import, share sanitizer, sync effect Promote legacy source.path into resolved_paths for old unstructured recipes, clear source.paths in share sanitizer to prevent leaking local filesystem paths, and gate file sync effect to dialog open transition so users can actually delete all uploaded files. * fix: CSV column fix (BOM + whitespace + unnamed index re-save) for #4470 * fix: harden unstructured upload flow and polish dialog UX * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- studio/backend/models/data_recipe.py | 43 ++- .../pyproject.toml | 3 + .../chunking.py | 106 ++++++ .../data_designer_unstructured_seed/config.py | 44 ++- .../data_designer_unstructured_seed/impl.py | 22 +- .../single-env/data-designer-deps.txt | 3 + studio/backend/routes/data_recipe/seed.py | 260 ++++++++++++++- studio/backend/utils/paths/__init__.py | 2 + studio/backend/utils/paths/storage_roots.py | 7 +- .../src/features/recipe-studio/api/index.ts | 80 ++++- .../components/controls/layout-controls.tsx | 12 +- .../components/controls/viewport-controls.tsx | 7 +- .../components/inline/inline-seed.tsx | 4 +- .../components/recipe-graph-node.tsx | 5 +- .../dialogs/expression/expression-dialog.tsx | 4 +- .../recipe-studio/dialogs/import-dialog.tsx | 2 +- .../recipe-studio/dialogs/llm/general-tab.tsx | 16 +- .../markdown-note/markdown-note-dialog.tsx | 2 +- .../dialogs/models/model-config-dialog.tsx | 14 +- .../dialogs/models/model-provider-dialog.tsx | 10 +- .../recipe-studio/dialogs/preview-dialog.tsx | 8 +- .../dialogs/processors-dialog.tsx | 4 +- .../dialogs/samplers/bernoulli-dialog.tsx | 2 +- .../dialogs/samplers/category-dialog.tsx | 6 +- .../dialogs/samplers/datetime-dialog.tsx | 6 +- .../dialogs/samplers/gaussian-dialog.tsx | 6 +- .../dialogs/samplers/person-dialog.tsx | 8 +- .../dialogs/samplers/subcategory-dialog.tsx | 2 +- .../dialogs/samplers/timedelta-dialog.tsx | 8 +- .../dialogs/samplers/uniform-dialog.tsx | 6 +- .../dialogs/samplers/uuid-dialog.tsx | 2 +- .../dialogs/seed/seed-dialog.tsx | 308 +++++++++--------- .../dialogs/seed/unstructured-drop-zone.tsx | 240 ++++++++++++++ .../dialogs/shared/field-label.tsx | 4 +- .../dialogs/shared/name-field.tsx | 2 +- .../tool-profile/tool-profile-dialog.tsx | 16 +- .../dialogs/validators/validator-dialog.tsx | 8 +- .../hooks/use-recipe-persistence.ts | 14 +- .../recipe-studio/recipe-studio-page.tsx | 88 +++-- .../recipe-studio/stores/recipe-studio.ts | 5 +- .../src/features/recipe-studio/types/index.ts | 5 +- .../recipe-studio/utils/config-factories.ts | 4 +- .../recipe-studio/utils/graph/fit-view.ts | 51 ++- .../recipe-studio/utils/import/importer.ts | 20 +- .../import/parsers/seed-config-parser.ts | 39 ++- .../utils/payload/build-payload.ts | 6 +- .../utils/payload/builders-seed.ts | 2 +- .../recipe-studio/utils/payload/types.ts | 9 +- .../recipe-studio/utils/validation.ts | 6 +- 49 files changed, 1216 insertions(+), 315 deletions(-) create mode 100644 studio/frontend/src/features/recipe-studio/dialogs/seed/unstructured-drop-zone.tsx diff --git a/studio/backend/models/data_recipe.py b/studio/backend/models/data_recipe.py index d49a50d1e3..b382ddb3d0 100644 --- a/studio/backend/models/data_recipe.py +++ b/studio/backend/models/data_recipe.py @@ -9,7 +9,7 @@ from __future__ import annotations from typing import Any -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator class RecipePayload(BaseModel): @@ -76,13 +76,41 @@ class SeedInspectRequest(BaseModel): class SeedInspectUploadRequest(BaseModel): - filename: str = Field(min_length = 1) - content_base64: str = Field(min_length = 1) + # Legacy single-file flow (mutually exclusive with file_ids) + filename: str | None = None + content_base64: str | None = None + # Multi-file flow (mutually exclusive with content_base64) + block_id: str | None = None + file_ids: list[str] | None = None + file_names: list[str] | None = None + # Shared fields preview_size: int = Field(default = 10, ge = 1, le = 50) seed_source_type: str | None = None unstructured_chunk_size: int | None = Field(default = None, ge = 1, le = 20000) unstructured_chunk_overlap: int | None = Field(default = None, ge = 0, le = 20000) + @model_validator(mode = "after") + def _check_mutual_exclusivity(self) -> "SeedInspectUploadRequest": + has_legacy = self.content_base64 is not None + has_multi = self.file_ids is not None + if has_legacy and has_multi: + raise ValueError("Provide either content_base64 or file_ids, not both") + if not has_legacy and not has_multi: + raise ValueError("Provide either content_base64 or file_ids") + if has_multi: + if len(self.file_ids) == 0: + raise ValueError("file_ids must not be empty") + if not self.block_id: + raise ValueError("block_id is required when using file_ids") + if self.file_names is None or len(self.file_ids) != len(self.file_names): + raise ValueError( + "file_names must be provided and same length as file_ids" + ) + if has_legacy: + if not self.filename: + raise ValueError("filename is required when using content_base64") + return self + class SeedInspectResponse(BaseModel): dataset_name: str @@ -91,6 +119,15 @@ class SeedInspectResponse(BaseModel): preview_rows: list[dict[str, Any]] = Field(default_factory = list) split: str | None = None subset: str | None = None + resolved_paths: list[str] | None = None + + +class UnstructuredFileUploadResponse(BaseModel): + file_id: str + filename: str + size_bytes: int + status: str # "ok" or "error" + error: str | None = None class McpToolsListRequest(BaseModel): diff --git a/studio/backend/plugins/data-designer-unstructured-seed/pyproject.toml b/studio/backend/plugins/data-designer-unstructured-seed/pyproject.toml index f826ffd992..d4770a0b05 100644 --- a/studio/backend/plugins/data-designer-unstructured-seed/pyproject.toml +++ b/studio/backend/plugins/data-designer-unstructured-seed/pyproject.toml @@ -13,6 +13,9 @@ requires-python = ">=3.11" dependencies = [ "data-designer-engine>=0.5.1,<0.6", "pandas>=2,<3", + "pymupdf>=1.24.0", + "pymupdf4llm>=0.0.17", + "mammoth>=1.8.0", ] [project.entry-points."data_designer.plugins"] diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py index 80f51b2a24..f6fdf74612 100644 --- a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py +++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py @@ -8,6 +8,8 @@ import re from pathlib import Path from typing import Any +import pandas as pd + from utils.paths import ensure_dir, unstructured_seed_cache_root DEFAULT_CHUNK_SIZE = 1200 @@ -59,6 +61,59 @@ def build_unstructured_preview_rows( ] +def build_multi_file_preview_rows( + *, + file_entries: list[tuple[Path, str]], + preview_size: int, + chunk_size: int | None, + chunk_overlap: int | None, +) -> list[dict[str, str]]: + cs = _to_int(chunk_size, DEFAULT_CHUNK_SIZE) + co = _to_int(chunk_overlap, DEFAULT_CHUNK_OVERLAP) + _, rows = materialize_multi_file_unstructured_seed( + file_entries = file_entries, + chunk_size = cs, + chunk_overlap = co, + ) + return _round_robin_preview(rows, preview_size) + + +def _round_robin_preview( + rows: list[dict[str, str]], + preview_size: int, +) -> list[dict[str, str]]: + """Pick preview rows round-robin across source files so every file is represented.""" + if not rows or preview_size <= 0: + return [] + + # Group rows by source_file, preserving order of first appearance + from collections import OrderedDict + + grouped: OrderedDict[str, list[dict[str, str]]] = OrderedDict() + for row in rows: + key = row.get("source_file", "") + if key not in grouped: + grouped[key] = [] + grouped[key].append(row) + + result: list[dict[str, str]] = [] + iterators = [iter(chunks) for chunks in grouped.values()] + while len(result) < preview_size and iterators: + exhausted: list[int] = [] + for i, it in enumerate(iterators): + if len(result) >= preview_size: + break + val = next(it, None) + if val is not None: + result.append(val) + else: + exhausted.append(i) + for i in reversed(exhausted): + iterators.pop(i) + + return result + + def materialize_unstructured_seed_dataset( *, source_path: Path, @@ -103,6 +158,43 @@ def materialize_unstructured_seed_dataset( return parquet_path, rows +def materialize_multi_file_unstructured_seed( + *, + file_entries: list[tuple[Path, str]], # (extracted_txt_path, original_filename) + chunk_size: int, + chunk_overlap: int, +) -> tuple[Path, list[dict[str, str]]]: + """Chunk multiple files and combine into one parquet dataset with source_file column.""" + chunk_size, chunk_overlap = resolve_chunking(chunk_size, chunk_overlap) + cache_key = _compute_multi_file_cache_key(file_entries, chunk_size, chunk_overlap) + cached = _CACHE_DIR / f"{cache_key}.parquet" + if cached.exists(): + df = pd.read_parquet(cached) + rows = df.to_dict(orient = "records") + return cached, rows + + all_rows: list[dict[str, str]] = [] + for txt_path, orig_name in file_entries: + text = load_unstructured_text_file(txt_path) + chunks = split_text_into_chunks( + text = text, + chunk_size = chunk_size, + chunk_overlap = chunk_overlap, + ) + for chunk in chunks: + all_rows.append({"chunk_text": chunk, "source_file": orig_name}) + + if not all_rows: + raise ValueError("No text found in any uploaded files.") + + df = pd.DataFrame(all_rows) + ensure_dir(_CACHE_DIR) + tmp = _CACHE_DIR / f"{cache_key}.tmp.parquet" + df.to_parquet(tmp, index = False) + tmp.replace(cached) + return cached, all_rows + + def load_unstructured_text_file(path: Path) -> str: ext = path.suffix.lower() if ext not in {".txt", ".md"}: @@ -193,3 +285,17 @@ def _compute_cache_key( ] ).encode("utf-8") return hashlib.sha256(payload).hexdigest() + + +def _compute_multi_file_cache_key( + file_entries: list[tuple[Path, str]], + chunk_size: int, + chunk_overlap: int, +) -> str: + parts: list[str] = [] + for path, name in sorted(file_entries, key = lambda e: e[1]): + st = path.stat() + parts.append(f"{path}|{st.st_size}|{st.st_mtime_ns}|{name}") + parts.append(f"cs={chunk_size}|co={chunk_overlap}") + raw = "\n".join(parts) + return hashlib.sha256(raw.encode()).hexdigest() diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/config.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/config.py index e0a0392a69..9bef34cbd6 100644 --- a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/config.py +++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/config.py @@ -6,7 +6,7 @@ from __future__ import annotations from pathlib import Path from typing import Literal -from pydantic import Field, field_validator +from pydantic import Field, field_validator, model_validator from data_designer.config.seed_source import SeedSource @@ -15,27 +15,37 @@ from .chunking import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, resolve_chunkin class UnstructuredSeedSource(SeedSource): seed_type: Literal["unstructured"] = "unstructured" - path: str = Field(..., min_length = 1) + paths: list[str] = Field(min_length = 1) + + @model_validator(mode = "before") + @classmethod + def _normalize_legacy_path(cls, data): + if isinstance(data, dict) and "paths" not in data and data.get("path"): + data = dict(data) + data["paths"] = [data["path"]] + return data + chunk_size: int = DEFAULT_CHUNK_SIZE chunk_overlap: int = DEFAULT_CHUNK_OVERLAP - @field_validator("path", mode = "after") + @field_validator("paths") @classmethod - def _validate_path(cls, value: str) -> str: - path = Path(value).expanduser() - if not path.is_file(): - raise ValueError(f"Unstructured seed path is not a file: {path}") - return value + def _validate_paths(cls, v: list[str]) -> list[str]: + for p in v: + expanded = Path(p).expanduser() + if not expanded.is_file(): + raise ValueError(f"Seed file does not exist: {expanded}") + return v - @field_validator("chunk_size", mode = "after") + @field_validator("chunk_size") @classmethod - def _validate_chunk_size(cls, value: int) -> int: - size, _ = resolve_chunking(value, 0) - return size + def _resolve_chunk_size(cls, v: int) -> int: + cs, _ = resolve_chunking(v, 0) + return cs - @field_validator("chunk_overlap", mode = "after") + @field_validator("chunk_overlap") @classmethod - def _validate_chunk_overlap(cls, value: int, info) -> int: - size = info.data.get("chunk_size", cls.model_fields["chunk_size"].default) - _, overlap = resolve_chunking(size, value) - return overlap + def _resolve_chunk_overlap(cls, v: int, info) -> int: + cs = info.data.get("chunk_size", DEFAULT_CHUNK_SIZE) + _, co = resolve_chunking(cs, v) + return co diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py index 8a3deb9b92..7272e426ad 100644 --- a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py +++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py @@ -8,7 +8,6 @@ from pathlib import Path import data_designer.lazy_heavy_imports as lazy from data_designer.engine.resources.seed_reader import SeedReader -from .chunking import materialize_unstructured_seed_dataset from .config import UnstructuredSeedSource @@ -17,8 +16,25 @@ class UnstructuredSeedReader(SeedReader[UnstructuredSeedSource]): return lazy.duckdb.connect() def get_dataset_uri(self) -> str: - path, _ = materialize_unstructured_seed_dataset( - source_path = Path(self.source.path), + from .chunking import materialize_multi_file_unstructured_seed + import json as json_mod + + file_entries: list[tuple[Path, str]] = [] + for p in self.source.paths: + path_obj = Path(p) + file_id = path_obj.name.replace(".extracted.txt", "") + meta_path = path_obj.parent / f"{file_id}.meta.json" + orig_name = path_obj.name + if meta_path.exists(): + try: + meta = json_mod.loads(meta_path.read_text()) + orig_name = meta.get("original_filename", path_obj.name) + except (json_mod.JSONDecodeError, OSError): + pass + file_entries.append((path_obj, orig_name)) + + path, _ = materialize_multi_file_unstructured_seed( + file_entries = file_entries, chunk_size = self.source.chunk_size, chunk_overlap = self.source.chunk_overlap, ) diff --git a/studio/backend/requirements/single-env/data-designer-deps.txt b/studio/backend/requirements/single-env/data-designer-deps.txt index dfc5b9bf21..e0b3d8b72a 100644 --- a/studio/backend/requirements/single-env/data-designer-deps.txt +++ b/studio/backend/requirements/single-env/data-designer-deps.txt @@ -17,3 +17,6 @@ ruff<1,>=0.14.10 scipy<2,>=1.11.0 sqlfluff<4,>=3.2.0 tiktoken<1,>=0.8.0 +pymupdf>=1.24.0 +pymupdf4llm>=0.0.17 +mammoth>=1.8.0 diff --git a/studio/backend/routes/data_recipe/seed.py b/studio/backend/routes/data_recipe/seed.py index 1765550adc..60fb1d7916 100644 --- a/studio/backend/routes/data_recipe/seed.py +++ b/studio/backend/routes/data_recipe/seed.py @@ -7,23 +7,27 @@ from __future__ import annotations import base64 import binascii +import json +import re from itertools import islice from pathlib import Path from typing import Any from uuid import uuid4 -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, HTTPException, UploadFile, File as FastAPIFile, Form from data_designer_unstructured_seed.chunking import ( build_unstructured_preview_rows, + normalize_unstructured_text, resolve_chunking, ) from core.data_recipe.jsonable import to_preview_jsonable -from utils.paths import ensure_dir, seed_uploads_root +from utils.paths import ensure_dir, seed_uploads_root, unstructured_uploads_root from models.data_recipe import ( SeedInspectRequest, SeedInspectResponse, SeedInspectUploadRequest, + UnstructuredFileUploadResponse, ) router = APIRouter() @@ -31,8 +35,21 @@ router = APIRouter() DATA_EXTS = (".parquet", ".jsonl", ".json", ".csv") DEFAULT_SPLIT = "train" LOCAL_UPLOAD_EXTS = {".csv", ".json", ".jsonl"} -UNSTRUCTURED_UPLOAD_EXTS = {".txt", ".md"} +UNSTRUCTURED_ALLOWED_EXTS = {".pdf", ".docx", ".txt", ".md"} SEED_UPLOAD_DIR = seed_uploads_root() +UNSTRUCTURED_UPLOAD_ROOT = unstructured_uploads_root() +MAX_FILE_SIZE = 50 * 1024 * 1024 # 50MB +MAX_TOTAL_SIZE = 100 * 1024 * 1024 # 100MB + +_SAFE_ID_RE = re.compile(r"^[a-zA-Z0-9_-]+$") + + +def _validate_safe_id(value: str, label: str) -> str: + if not value or not _SAFE_ID_RE.match(value): + raise HTTPException( + 400, f"Invalid {label}: must be alphanumeric/dash/underscore only" + ) + return value def _serialize_preview_value(value: Any) -> Any: @@ -177,7 +194,17 @@ def _read_preview_rows_from_local_file( ext = path.suffix.lower() try: if ext == ".csv": - df = pd.read_csv(path, nrows = preview_size) + df = pd.read_csv(path, nrows = preview_size, encoding = "utf-8-sig") + df.columns = df.columns.str.strip() + unnamed = [c for c in df.columns if c == "" or c.startswith("Unnamed:")] + if unnamed: + df = df.drop(columns = unnamed) + full_df = pd.read_csv(path, encoding = "utf-8-sig") + full_df.columns = full_df.columns.str.strip() + full_df = full_df.drop(columns = unnamed) + tmp_csv = path.with_suffix(".tmp.csv") + full_df.to_csv(tmp_csv, index = False, encoding = "utf-8") + tmp_csv.replace(path) elif ext == ".jsonl": df = pd.read_json(path, lines = True).head(preview_size) elif ext == ".json": @@ -220,6 +247,36 @@ def _read_preview_rows_from_unstructured_file( return _serialize_preview_rows(rows) +def _read_preview_rows_from_multi_files( + *, + block_id: str, + file_ids: list[str], + file_names: list[str], + preview_size: int, + chunk_size: int | None, + chunk_overlap: int | None, +) -> list[dict[str, str]]: + from data_designer_unstructured_seed.chunking import build_multi_file_preview_rows + + _validate_safe_id(block_id, "block_id") + block_dir = UNSTRUCTURED_UPLOAD_ROOT / block_id + file_entries: list[tuple[Path, str]] = [] + for fid, fname in zip(file_ids, file_names): + extracted = block_dir / f"{fid}.extracted.txt" + if not extracted.exists(): + raise HTTPException( + 404, f"Extracted text not found for file: {fname} (id: {fid})" + ) + file_entries.append((extracted, fname)) + + return build_multi_file_preview_rows( + file_entries = file_entries, + preview_size = preview_size, + chunk_size = chunk_size, + chunk_overlap = chunk_overlap, + ) + + @router.post("/seed/inspect", response_model = SeedInspectResponse) def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse: dataset_name = payload.dataset_name.strip() @@ -306,14 +363,202 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse: ) +def _extract_text_from_file(file_path: Path, ext: str) -> str: + """Extract text from uploaded file based on extension, converting to markdown where possible.""" + if ext in {".txt", ".md"}: + raw = file_path.read_text(encoding = "utf-8", errors = "ignore") + elif ext == ".pdf": + import pymupdf4llm + + raw = pymupdf4llm.to_markdown( + str(file_path), write_images = False, show_progress = False + ) + elif ext == ".docx": + import mammoth + + with open(str(file_path), "rb") as f: + result = mammoth.convert_to_markdown(f) + raw = result.value + else: + raise ValueError(f"Unsupported file type: {ext}") + + return normalize_unstructured_text(raw) + + +def _get_block_total_size(block_dir: Path, file_ids: list[str]) -> int: + """Sum raw upload sizes for tracked file IDs only.""" + if not block_dir.exists() or not file_ids: + return 0 + id_set = set(file_ids) + total = 0 + for f in block_dir.iterdir(): + if not f.is_file(): + continue + if f.name.endswith(".extracted.txt") or f.name.endswith(".meta.json"): + continue + stem = f.name.split(".")[0] + if stem in id_set: + total += f.stat().st_size + return total + + +@router.post("/seed/upload-unstructured-file") +async def upload_unstructured_file( + file: UploadFile = FastAPIFile(...), + block_id: str = Form(...), + existing_file_ids: str = Form(""), +) -> UnstructuredFileUploadResponse: + _validate_safe_id(block_id, "block_id") + + tracked_ids = [fid.strip() for fid in existing_file_ids.split(",") if fid.strip()] + + original_filename = file.filename or "upload" + ext = Path(original_filename).suffix.lower() + if ext not in UNSTRUCTURED_ALLOWED_EXTS: + raise HTTPException( + 400, + f"Unsupported file type: {ext}. Allowed: {', '.join(sorted(UNSTRUCTURED_ALLOWED_EXTS))}", + ) + + content = await file.read() + size_bytes = len(content) + + if size_bytes == 0: + raise HTTPException(400, "Empty file not allowed") + + if size_bytes > MAX_FILE_SIZE: + raise HTTPException( + 413, f"File too large ({size_bytes} bytes). Maximum is 50MB." + ) + + block_dir = UNSTRUCTURED_UPLOAD_ROOT / block_id + ensure_dir(block_dir) + current_total = _get_block_total_size(block_dir, file_ids = tracked_ids) + if current_total + size_bytes > MAX_TOTAL_SIZE: + raise HTTPException( + 413, f"Total upload limit ({MAX_TOTAL_SIZE // (1024 * 1024)}MB) exceeded" + ) + + file_id = uuid4().hex + raw_path = block_dir / f"{file_id}{ext}" + raw_path.write_bytes(content) + + extracted_path = block_dir / f"{file_id}.extracted.txt" + try: + extracted_text = _extract_text_from_file(raw_path, ext) + if not extracted_text or not extracted_text.strip(): + raw_path.unlink(missing_ok = True) + return UnstructuredFileUploadResponse( + file_id = file_id, + filename = original_filename, + size_bytes = size_bytes, + status = "error", + error = "No extractable text found in file", + ) + extracted_path.write_text(extracted_text, encoding = "utf-8") + except Exception as e: + raw_path.unlink(missing_ok = True) + extracted_path.unlink(missing_ok = True) + return UnstructuredFileUploadResponse( + file_id = file_id, + filename = original_filename, + size_bytes = size_bytes, + status = "error", + error = f"Text extraction failed: {type(e).__name__}: {e}", + ) + + try: + meta_path = block_dir / f"{file_id}.meta.json" + meta_path.write_text( + json.dumps( + {"original_filename": original_filename, "size_bytes": size_bytes} + ), + encoding = "utf-8", + ) + except OSError: + raw_path.unlink(missing_ok = True) + extracted_path.unlink(missing_ok = True) + return UnstructuredFileUploadResponse( + file_id = file_id, + filename = original_filename, + size_bytes = size_bytes, + status = "error", + error = "Failed to save file metadata", + ) + + return UnstructuredFileUploadResponse( + file_id = file_id, + filename = original_filename, + size_bytes = size_bytes, + status = "ok", + ) + + +@router.delete("/seed/unstructured-file/{block_id}/{file_id}") +async def remove_unstructured_file(block_id: str, file_id: str): + _validate_safe_id(block_id, "block_id") + _validate_safe_id(file_id, "file_id") + + block_dir = UNSTRUCTURED_UPLOAD_ROOT / block_id + if not block_dir.exists(): + raise HTTPException(404, "Block not found") + + deleted = False + for f in block_dir.iterdir(): + stem = f.name.split(".")[0] + if stem == file_id: + f.unlink(missing_ok = True) + deleted = True + + if not deleted: + raise HTTPException(404, "File not found") + try: + if not any(block_dir.iterdir()): + block_dir.rmdir() + except OSError: + pass + + return {"status": "ok"} + + @router.post("/seed/inspect-upload", response_model = SeedInspectResponse) def inspect_seed_upload(payload: SeedInspectUploadRequest) -> SeedInspectResponse: + if payload.file_ids is not None: + if len(payload.file_ids) == 0: + raise HTTPException(400, "file_ids must not be empty") + _validate_safe_id(payload.block_id, "block_id") + for fid in payload.file_ids: + _validate_safe_id(fid, "file_id") + preview_rows = _read_preview_rows_from_multi_files( + block_id = payload.block_id, + file_ids = payload.file_ids, + file_names = payload.file_names, + preview_size = payload.preview_size, + chunk_size = payload.unstructured_chunk_size, + chunk_overlap = payload.unstructured_chunk_overlap, + ) + columns = ["chunk_text", "source_file"] if preview_rows else [] + resolved_paths = [ + str(UNSTRUCTURED_UPLOAD_ROOT / payload.block_id / f"{fid}.extracted.txt") + for fid in payload.file_ids + ] + return SeedInspectResponse( + dataset_name = "unstructured_seed", + resolved_path = resolved_paths[0] if resolved_paths else "", + resolved_paths = resolved_paths, + columns = columns, + preview_rows = _serialize_preview_rows(preview_rows), + ) + seed_source_type = _normalize_optional_text(payload.seed_source_type) or "local" filename = _sanitize_filename(payload.filename) ext = Path(filename).suffix.lower() + # Legacy single-file unstructured path only supports .txt/.md + # PDF/DOCX extraction uses the multi-file upload endpoint instead + _LEGACY_UNSTRUCTURED_EXTS = {".txt", ".md"} if seed_source_type == "unstructured": - if ext not in UNSTRUCTURED_UPLOAD_EXTS: - allowed = ", ".join(sorted(UNSTRUCTURED_UPLOAD_EXTS)) + if ext not in _LEGACY_UNSTRUCTURED_EXTS: + allowed = ", ".join(sorted(_LEGACY_UNSTRUCTURED_EXTS)) raise HTTPException( status_code = 400, detail = f"unsupported file type: {ext}. allowed: {allowed}", @@ -329,8 +574,7 @@ def inspect_seed_upload(payload: SeedInspectUploadRequest) -> SeedInspectRespons file_bytes = _decode_base64_payload(payload.content_base64) if not file_bytes: raise HTTPException(status_code = 400, detail = "empty upload payload") - max_size_bytes = 50 * 1024 * 1024 - if len(file_bytes) > max_size_bytes: + if len(file_bytes) > MAX_FILE_SIZE: raise HTTPException(status_code = 413, detail = "file too large (max 50MB)") ensure_dir(SEED_UPLOAD_DIR) diff --git a/studio/backend/utils/paths/__init__.py b/studio/backend/utils/paths/__init__.py index 507fb1106b..90df216f96 100644 --- a/studio/backend/utils/paths/__init__.py +++ b/studio/backend/utils/paths/__init__.py @@ -19,6 +19,7 @@ from .storage_roots import ( tmp_root, seed_uploads_root, unstructured_seed_cache_root, + unstructured_uploads_root, oxc_validator_tmp_root, tensorboard_root, ensure_dir, @@ -47,6 +48,7 @@ __all__ = [ "tmp_root", "seed_uploads_root", "unstructured_seed_cache_root", + "unstructured_uploads_root", "oxc_validator_tmp_root", "tensorboard_root", "ensure_dir", diff --git a/studio/backend/utils/paths/storage_roots.py b/studio/backend/utils/paths/storage_roots.py index 81763c3095..f14887e119 100644 --- a/studio/backend/utils/paths/storage_roots.py +++ b/studio/backend/utils/paths/storage_roots.py @@ -54,13 +54,17 @@ def tmp_root() -> Path: def seed_uploads_root() -> Path: - return tmp_root() / "seed-uploads" + return datasets_root() / "seed-uploads" def unstructured_seed_cache_root() -> Path: return tmp_root() / "unstructured-seed-cache" +def unstructured_uploads_root() -> Path: + return datasets_root() / "unstructured-uploads" + + def oxc_validator_tmp_root() -> Path: return tmp_root() / "oxc-validator" @@ -104,6 +108,7 @@ def ensure_studio_directories() -> None: datasets_root, dataset_uploads_root, recipe_datasets_root, + unstructured_uploads_root, outputs_root, exports_root, auth_root, diff --git a/studio/frontend/src/features/recipe-studio/api/index.ts b/studio/frontend/src/features/recipe-studio/api/index.ts index 7225450154..9212e4db9b 100644 --- a/studio/frontend/src/features/recipe-studio/api/index.ts +++ b/studio/frontend/src/features/recipe-studio/api/index.ts @@ -103,13 +103,22 @@ export type SeedInspectRequest = { }; export type SeedInspectUploadRequest = { - filename: string; - // base64 payload without data URL prefix - content_base64: string; + // Legacy single-file + filename?: string; + // biome-ignore lint/style/useNamingConvention: api schema + content_base64?: string; + // Multi-file + // biome-ignore lint/style/useNamingConvention: api schema + block_id?: string; + // biome-ignore lint/style/useNamingConvention: api schema + file_ids?: string[]; + // biome-ignore lint/style/useNamingConvention: api schema + file_names?: string[]; + // Shared // biome-ignore lint/style/useNamingConvention: api schema preview_size?: number; // biome-ignore lint/style/useNamingConvention: api schema - seed_source_type?: "local" | "unstructured"; + seed_source_type?: string; // biome-ignore lint/style/useNamingConvention: api schema unstructured_chunk_size?: number; // biome-ignore lint/style/useNamingConvention: api schema @@ -126,6 +135,8 @@ export type SeedInspectResponse = { preview_rows: Record[]; split?: string | null; subset?: string | null; + // biome-ignore lint/style/useNamingConvention: api schema + resolved_paths?: string[] | null; }; export type ValidateError = { @@ -372,3 +383,64 @@ export async function streamRecipeJobEvents(options: { } // NOTE: preview endpoints removed from harness. + +type UnstructuredFileUploadResponse = { + // biome-ignore lint/style/useNamingConvention: api schema + file_id: string; + filename: string; + // biome-ignore lint/style/useNamingConvention: api schema + size_bytes: number; + status: "ok" | "error"; + error?: string; +}; + +export async function uploadUnstructuredFile( + file: File, + blockId: string, + signal?: AbortSignal, + existingFileIds?: string[], +): Promise { + const formData = new FormData(); + formData.append("file", file); + formData.append("block_id", blockId); + if (existingFileIds?.length) { + formData.append("existing_file_ids", existingFileIds.join(",")); + } + + const res = await authFetch(`${DATA_DESIGNER_API_BASE}/seed/upload-unstructured-file`, { + method: "POST", + body: formData, + signal, + }); + + if (res.status === 413) { + const detail = await res.json().catch(() => ({ detail: "File too large" })); + return { + file_id: "", + filename: file.name, + size_bytes: file.size, + status: "error", + error: typeof detail.detail === "string" ? detail.detail : "File too large", + }; + } + + if (!res.ok) { + const detail = await res.json().catch(() => ({ detail: "Upload failed" })); + throw new Error(typeof detail.detail === "string" ? detail.detail : "Upload failed"); + } + + return res.json(); +} + +export async function removeUnstructuredFile( + blockId: string, + fileId: string, +): Promise { + const res = await authFetch( + `${DATA_DESIGNER_API_BASE}/seed/unstructured-file/${encodeURIComponent(blockId)}/${encodeURIComponent(fileId)}`, + { method: "DELETE" }, + ); + if (!res.ok && res.status !== 404) { + throw new Error("Failed to remove file"); + } +} diff --git a/studio/frontend/src/features/recipe-studio/components/controls/layout-controls.tsx b/studio/frontend/src/features/recipe-studio/components/controls/layout-controls.tsx index f4b76c8d2c..2a38f30059 100644 --- a/studio/frontend/src/features/recipe-studio/components/controls/layout-controls.tsx +++ b/studio/frontend/src/features/recipe-studio/components/controls/layout-controls.tsx @@ -8,7 +8,7 @@ import { useUpdateNodeInternals, } from "@xyflow/react"; import { Button } from "@/components/ui/button"; -import { getFitNodeIdsIgnoringNotes } from "../../utils/graph/fit-view"; +import { buildFitViewOptions } from "../../utils/graph/fit-view"; type LayoutControlsProps = { direction: "LR" | "TB"; @@ -36,10 +36,7 @@ export function LayoutControls({ requestAnimationFrame(() => { refreshNodeInternals(); requestAnimationFrame(() => { - fitView({ - duration: 250, - nodes: getFitNodeIdsIgnoringNotes(getNodes()), - }); + fitView(buildFitViewOptions(getNodes())); }); }); }, [fitView, getNodes, onLayout, refreshNodeInternals]); @@ -51,10 +48,7 @@ export function LayoutControls({ requestAnimationFrame(() => { refreshNodeInternals(); requestAnimationFrame(() => { - fitView({ - duration: 250, - nodes: getFitNodeIdsIgnoringNotes(getNodes()), - }); + fitView(buildFitViewOptions(getNodes())); }); }); }); diff --git a/studio/frontend/src/features/recipe-studio/components/controls/viewport-controls.tsx b/studio/frontend/src/features/recipe-studio/components/controls/viewport-controls.tsx index 261ebe1b11..0a0eff849b 100644 --- a/studio/frontend/src/features/recipe-studio/components/controls/viewport-controls.tsx +++ b/studio/frontend/src/features/recipe-studio/components/controls/viewport-controls.tsx @@ -5,7 +5,7 @@ import { type ReactElement, useCallback } from "react"; import { Lock, LockOpen, Maximize2, Minus, Plus } from "lucide-react"; import { Panel, useReactFlow } from "@xyflow/react"; import { Button } from "@/components/ui/button"; -import { getFitNodeIdsIgnoringNotes } from "../../utils/graph/fit-view"; +import { buildFitViewOptions } from "../../utils/graph/fit-view"; import { RECIPE_FLOATING_ICON_BUTTON_CLASS } from "../recipe-floating-icon-button-class"; type ViewportControlsProps = { @@ -30,10 +30,7 @@ export function ViewportControls({ }, [zoomOut]); const handleFitView = useCallback(() => { - fitView({ - duration: 250, - nodes: getFitNodeIdsIgnoringNotes(getNodes()), - }); + fitView(buildFitViewOptions(getNodes())); }, [fitView, getNodes]); return ( diff --git a/studio/frontend/src/features/recipe-studio/components/inline/inline-seed.tsx b/studio/frontend/src/features/recipe-studio/components/inline/inline-seed.tsx index 2162f75ccf..bf022fa028 100644 --- a/studio/frontend/src/features/recipe-studio/components/inline/inline-seed.tsx +++ b/studio/frontend/src/features/recipe-studio/components/inline/inline-seed.tsx @@ -45,7 +45,9 @@ export function InlineSeed({ config, onUpdate }: InlineSeedProps): ReactElement const isLocal = mode === "local"; const fileName = isLocal ? config.local_file_name?.trim() - : config.unstructured_file_name?.trim(); + : config.unstructured_file_names?.length + ? `${config.unstructured_file_names.length} file${config.unstructured_file_names.length !== 1 ? "s" : ""}` + : undefined; return (
diff --git a/studio/frontend/src/features/recipe-studio/components/recipe-graph-node.tsx b/studio/frontend/src/features/recipe-studio/components/recipe-graph-node.tsx index 672c29f4ab..0fcf202190 100644 --- a/studio/frontend/src/features/recipe-studio/components/recipe-graph-node.tsx +++ b/studio/frontend/src/features/recipe-studio/components/recipe-graph-node.tsx @@ -256,9 +256,10 @@ function getConfigSummary(config: NodeConfig | undefined): string { } if ( seedSourceType === "unstructured" && - config.unstructured_file_name?.trim() + config.unstructured_file_names?.length ) { - return config.unstructured_file_name.trim(); + const count = config.unstructured_file_names.length; + return `${count} file${count !== 1 ? "s" : ""} uploaded`; } if (config.hf_path.trim()) { return config.hf_path.trim(); diff --git a/studio/frontend/src/features/recipe-studio/dialogs/expression/expression-dialog.tsx b/studio/frontend/src/features/recipe-studio/dialogs/expression/expression-dialog.tsx index 0947851ac7..0e072d5858 100644 --- a/studio/frontend/src/features/recipe-studio/dialogs/expression/expression-dialog.tsx +++ b/studio/frontend/src/features/recipe-studio/dialogs/expression/expression-dialog.tsx @@ -58,7 +58,7 @@ export function ExpressionDialog({ value={config.name} onChange={(value) => onUpdate({ name: value })} /> -
+
-
+
Import recipe -
+
) : null} -
+
)} {(hasToolProfiles || Boolean(config.tool_alias?.trim())) && ( -
+
)} {config.llm_type === "code" && ( -
+
)} -
+
{imageContext.enabled && ( -
+
)} {config.llm_type === "structured" && ( -
+
-
+
)}
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
updateField("endpoint", event.target.value)} />
-
+
-
+
updateField("api_key_env", event.target.value)} />
-
+
updateField("extra_headers", event.target.value)} />
-
+
+
-
+
{kind === "full" && ( -
+
)} -
+
-
+
updateSchema({ name: event.target.value })} />
-
+
onUpdate({ name: value })} /> -
+
onUpdate({ name: value })} />
-
+
-
+
-
+

Rule weights (optional)

diff --git a/studio/frontend/src/features/recipe-studio/dialogs/samplers/datetime-dialog.tsx b/studio/frontend/src/features/recipe-studio/dialogs/samplers/datetime-dialog.tsx index e75551ec75..930c748585 100644 --- a/studio/frontend/src/features/recipe-studio/dialogs/samplers/datetime-dialog.tsx +++ b/studio/frontend/src/features/recipe-studio/dialogs/samplers/datetime-dialog.tsx @@ -50,7 +50,7 @@ export function DatetimeDialog({ />
-
+
-
+
-
+
onUpdate({ name: value })} />
-
+
onUpdate({ mean: event.target.value })} />
-
+
-
+
Faker

-
+
-
+
-
+
-
+
onUpdate({ name: value })} />
-
+
onUpdate({ name: value })} />
-
+
updateField("dt_min", event.target.value)} />
-
+
-
+
-
+
onUpdate({ name: value })} />
-
+
onUpdate({ low: event.target.value })} />
-
+
-
+
onUpdate({ name: value })} /> -
+
= [ ]; const LOCAL_ACCEPT = ".csv,.json,.jsonl"; -const UNSTRUCTURED_ACCEPT = ".txt,.pdf,.docx"; const MAX_UPLOAD_BYTES = 50 * 1024 * 1024; const DEFAULT_CHUNK_SIZE = 1200; const DEFAULT_CHUNK_OVERLAP = 200; @@ -112,20 +110,20 @@ function getPreviewEmptyStateCopy(mode: SeedConfig["seed_source_type"]): { } { if (mode === "local") { return { - title: "No local preview yet", - description: "Choose a CSV/JSON/JSONL file, then click Load to fetch 10 rows.", + title: "No preview yet", + description: "Upload a CSV, JSON, or JSONL file and click Load to see a sample.", }; } if (mode === "unstructured") { return { - title: "No chunk preview yet", + title: "No preview yet", description: - "Choose a TXT/PDF/DOCX file, then click Load to extract + preview chunk_text rows.", + "Upload your documents and the preview will appear once processing is done.", }; } return { - title: "No dataset preview yet", - description: "Pick a Hugging Face dataset and click Load to fetch 10 sample rows.", + title: "No preview yet", + description: "Select a Hugging Face dataset and click Load to see a sample.", }; } @@ -177,42 +175,6 @@ async function fileToBase64Payload(file: File): Promise { }); } -async function extractUnstructuredText(file: File): Promise { - const lower = file.name.toLowerCase(); - if (lower.endsWith(".txt")) { - return file.text(); - } - if (lower.endsWith(".pdf")) { - const buffer = new Uint8Array(await file.arrayBuffer()); - const pdf = await getDocumentProxy(buffer); - const { text } = await extractText(pdf, { mergePages: true }); - return text; - } - if (lower.endsWith(".docx")) { - const arrayBuffer = await file.arrayBuffer(); - const { value } = await mammoth.extractRawText({ arrayBuffer }); - return value; - } - throw new Error("Unsupported unstructured file type"); -} - -async function toUnstructuredUploadFile(file: File): Promise { - const lower = file.name.toLowerCase(); - if (lower.endsWith(".txt") || lower.endsWith(".md")) { - return file; - } - - const text = (await extractUnstructuredText(file)).trim(); - if (!text) { - throw new Error("No text found in file."); - } - const normalized = text.replace(/\r\n/g, "\n").replace(/\r/g, "\n"); - const stem = file.name.replace(/\.(pdf|docx)$/i, "") || "unstructured_seed"; - return new File([normalized], `${stem}.txt`, { - type: "text/plain", - }); -} - export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactElement { const [inspectError, setInspectError] = useState(null); const [isInspecting, setIsInspecting] = useState(false); @@ -220,16 +182,84 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl const [previewRows, setPreviewRows] = useState[]>([]); const [expandedPreviewRows, setExpandedPreviewRows] = useState>({}); const [localFile, setLocalFile] = useState(null); - const [unstructuredFile, setUnstructuredFile] = useState(null); + const [unstructuredFiles, setUnstructuredFiles] = useState(() => { + if (config.unstructured_file_ids?.length) { + return config.unstructured_file_ids.map((id, i) => ({ + id, + name: config.unstructured_file_names?.[i] ?? "Unknown", + size: config.unstructured_file_sizes?.[i] ?? 0, + status: "ok" as const, + })); + } + return []; + }); const mode = config.seed_source_type ?? "hf"; const previewEmpty = getPreviewEmptyStateCopy(mode); + const prevModeRef = useRef(mode); useEffect(() => { + const prevMode = prevModeRef.current; + prevModeRef.current = mode; setInspectError(null); setLocalFile(null); - setUnstructuredFile(null); - }, [mode]); + if (prevMode === "unstructured" && mode !== "unstructured") { + setUnstructuredFiles([]); + } + if (prevMode !== "unstructured" && mode === "unstructured") { + if (config.unstructured_file_ids?.length) { + setUnstructuredFiles( + config.unstructured_file_ids.map((id, i) => ({ + id, + name: config.unstructured_file_names?.[i] ?? "Unknown", + size: config.unstructured_file_sizes?.[i] ?? 0, + status: "ok" as const, + })), + ); + } else { + setUnstructuredFiles([]); + } + } + }, [mode]); // eslint-disable-line react-hooks/exhaustive-deps + + const didSyncFilesRef = useRef(false); + useEffect(() => { + if (!open) { + didSyncFilesRef.current = false; + return; + } + if (didSyncFilesRef.current) return; + if (mode !== "unstructured") return; + if (unstructuredFiles.length > 0) return; + if (!config.unstructured_file_ids?.length) return; + didSyncFilesRef.current = true; + setUnstructuredFiles( + config.unstructured_file_ids.map((id, i) => ({ + id, + name: config.unstructured_file_names?.[i] ?? "Unknown", + size: config.unstructured_file_sizes?.[i] ?? 0, + status: "ok" as const, + })), + ); + }, [open, mode, unstructuredFiles.length, config.unstructured_file_ids, config.unstructured_file_names, config.unstructured_file_sizes]); + + const handleUnstructuredFilesChange = useCallback( + (updater: FileEntry[] | ((prev: FileEntry[]) => FileEntry[])) => { + setUnstructuredFiles((prev) => { + const next = typeof updater === "function" ? updater(prev) : updater; + const okFiles = next.filter((f) => f.status === "ok"); + queueMicrotask(() => { + onUpdate({ + unstructured_file_ids: okFiles.map((f) => f.id), + unstructured_file_names: okFiles.map((f) => f.name), + unstructured_file_sizes: okFiles.map((f) => f.size), + }); + }); + return next; + }); + }, + [onUpdate], + ); useEffect(() => { setPreviewRows(config.seed_preview_rows ?? []); @@ -256,14 +286,16 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl if (!localFile) return null; return `local:${localFile.name}|${localFile.size}|${localFile.lastModified}`; } - if (!unstructuredFile) return null; + const okFiles = unstructuredFiles.filter((f) => f.status === "ok"); + if (okFiles.length === 0) return null; const { chunkSize, chunkOverlap } = resolveChunking(config); - return `unstructured:${unstructuredFile.name}|${unstructuredFile.size}|${unstructuredFile.lastModified}|${chunkSize}|${chunkOverlap}`; + const fileKey = okFiles.map((f) => `${f.id}|${f.name}`).join(","); + return `unstructured:${fileKey}|${chunkSize}|${chunkOverlap}`; }, [ config, localFile, mode, - unstructuredFile, + unstructuredFiles, ]); const loadSeedMetadata = useCallback(async (opts?: { silent?: boolean }): Promise => { @@ -295,7 +327,9 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl hf_split: response.split ?? "", hf_subset: response.subset ?? "", local_file_name: "", - unstructured_file_name: "", + unstructured_file_ids: [], + unstructured_file_names: [], + unstructured_file_sizes: [], }); setPreviewRows(response.preview_rows ?? []); setLastLoadedKey(loadKey); @@ -326,50 +360,56 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl hf_subset: "", hf_split: "", local_file_name: localFile.name, - unstructured_file_name: "", + unstructured_file_ids: [], + unstructured_file_names: [], + unstructured_file_sizes: [], }); setPreviewRows(response.preview_rows ?? []); setLastLoadedKey(loadKey); return true; } - if (!unstructuredFile) { - throw new Error("Select a PDF/DOCX/TXT file first."); - } - if (unstructuredFile.size > MAX_UPLOAD_BYTES) { - throw new Error("File too large (max 50MB)."); + if (mode === "unstructured") { + const fileIds = unstructuredFiles + .filter((f) => f.status === "ok") + .map((f) => f.id); + const fileNames = unstructuredFiles + .filter((f) => f.status === "ok") + .map((f) => f.name); + + if (fileIds.length === 0) { + setInspectError("No files uploaded"); + return false; + } + + const { chunkSize, chunkOverlap } = resolveChunking(config); + const response = await inspectSeedUpload({ + block_id: config.id, + file_ids: fileIds, + file_names: fileNames, + preview_size: 10, + seed_source_type: "unstructured", + unstructured_chunk_size: chunkSize, + unstructured_chunk_overlap: chunkOverlap, + }); + + onUpdate({ + hf_path: response.resolved_path, + resolved_paths: response.resolved_paths ?? [], + seed_columns: response.columns, + seed_preview_rows: response.preview_rows ?? [], + unstructured_file_ids: fileIds, + unstructured_file_names: fileNames, + unstructured_file_sizes: unstructuredFiles + .filter((f) => f.status === "ok") + .map((f) => f.size), + }); + setPreviewRows(response.preview_rows ?? []); + setLastLoadedKey(loadKey); + return true; } - const { chunkSize, chunkOverlap } = resolveChunking(config); - const uploadFile = await toUnstructuredUploadFile(unstructuredFile); - if (uploadFile.size > MAX_UPLOAD_BYTES) { - throw new Error("Processed text is too large (max 50MB)."); - } - const payload = await fileToBase64Payload(uploadFile); - const response = await inspectSeedUpload({ - filename: uploadFile.name, - content_base64: payload, - preview_size: 10, - seed_source_type: "unstructured", - unstructured_chunk_size: chunkSize, - unstructured_chunk_overlap: chunkOverlap, - }); - onUpdate({ - hf_path: response.resolved_path, - seed_columns: response.columns, - seed_drop_columns: (config.seed_drop_columns ?? []).filter((name) => - response.columns.includes(name), - ), - seed_preview_rows: response.preview_rows ?? [], - hf_repo_id: "", - hf_subset: "", - hf_split: "", - local_file_name: "", - unstructured_file_name: unstructuredFile.name, - }); - setPreviewRows(response.preview_rows ?? []); - setLastLoadedKey(loadKey); - return true; + return false; } catch (error) { if (!opts?.silent) { setInspectError(getErrorMessage(error, "Failed to load seed metadata.")); @@ -385,7 +425,7 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl localFile, mode, onUpdate, - unstructuredFile, + unstructuredFiles, ]); useEffect(() => { @@ -401,6 +441,21 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl void loadSeedMetadata({ silent: true }); }, [getCurrentLoadKey, isInspecting, lastLoadedKey, loadSeedMetadata, open]); + const wasUploadingRef = useRef(false); + useEffect(() => { + if (mode !== "unstructured") return; + const isUploading = unstructuredFiles.some((f) => f.status === "uploading"); + if (isUploading) { + wasUploadingRef.current = true; + } else if (wasUploadingRef.current) { + wasUploadingRef.current = false; + const hasOk = unstructuredFiles.some((f) => f.status === "ok"); + if (hasOk) { + void loadSeedMetadata({ silent: true }); + } + } + }, [mode, unstructuredFiles, loadSeedMetadata]); + const previewColumns = useMemo(() => { const loadedColumns = config.seed_columns ?? []; if (loadedColumns.length > 0) return loadedColumns; @@ -434,10 +489,10 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl -
+
{mode === "hf" && ( <> -
+
-
+
+

- Upload-only. Max 50MB. + Max 50MB per file.

{(localFile?.name || config.local_file_name?.trim()) && (

@@ -537,49 +592,12 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl )} {mode === "unstructured" && ( -

- -
- { - const file = event.target.files?.[0] ?? null; - setUnstructuredFile(file); - onUpdate({ - hf_path: "", - seed_columns: [], - seed_drop_columns: [], - seed_preview_rows: [], - unstructured_file_name: file?.name ?? "", - }); - }} - /> - -
-

- File is converted to text, then chunked server-side into chunk_text rows. Max 50MB. -

- {(unstructuredFile?.name || - config.unstructured_file_name?.trim()) && ( -

- Selected:{" "} - {unstructuredFile?.name ?? config.unstructured_file_name?.trim()} -

- )} -
+ )} {inspectError &&

{inspectError}

} @@ -633,7 +651,7 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl /> -
+
-
+
-
+
-
+
-
+
onUpdate({ selection_start: event.target.value })} />
-
+
-
+
onUpdate({ selection_index: event.target.value })} />
-
+
FileEntry[])) => void; + disabled?: boolean; +}; + +function formatSize(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +function isValidExtension(name: string): boolean { + const ext = name.slice(name.lastIndexOf(".")).toLowerCase(); + return ACCEPTED_EXTENSIONS.includes(ext); +} + +export function UnstructuredDropZone({ + blockId, + files, + onFilesChange, + disabled, +}: UnstructuredDropZoneProps) { + const inputRef = useRef(null); + const filesRef = useRef(files); + filesRef.current = files; + const [isDragOver, setIsDragOver] = useState(false); + + const totalSize = files.reduce((sum, f) => sum + f.size, 0); + + const handleFiles = useCallback( + async (newFiles: File[]) => { + const valid = newFiles.filter((f) => { + if (!isValidExtension(f.name)) return false; + if (f.size > MAX_FILE_SIZE) return false; + return true; + }); + + if (valid.length === 0) return; + + const addedSize = valid.reduce((s, f) => s + f.size, 0); + const currentTotal = filesRef.current.reduce((sum, f) => sum + f.size, 0); + if (currentTotal + addedSize > MAX_TOTAL_SIZE) return; + + const entries: FileEntry[] = valid.map((f) => ({ + id: "", + name: f.name, + size: f.size, + status: "uploading" as const, + abortController: new AbortController(), + })); + + onFilesChange((prev) => [...prev, ...entries]); + + for (let i = 0; i < valid.length; i++) { + const file = valid[i]; + const entry = entries[i]; + let updatedId = ""; + let updatedStatus: FileEntry["status"] = "error"; + let updatedError: string | undefined; + try { + const existingIds = filesRef.current.filter((f) => f.id).map((f) => f.id); + const result = await uploadUnstructuredFile( + file, + blockId, + entry.abortController?.signal, + existingIds, + ); + updatedId = result.file_id; + updatedStatus = result.status === "ok" ? "ok" : "error"; + updatedError = result.error; + } catch (e) { + if (e instanceof DOMException && e.name === "AbortError") { + updatedError = "Cancelled"; + } else { + updatedError = e instanceof Error ? e.message : "Upload failed"; + } + } + onFilesChange((prev) => + prev.map((f) => + f === entry + ? { ...f, id: updatedId, status: updatedStatus, error: updatedError } + : f, + ), + ); + } + + }, + [blockId, onFilesChange], + ); + + const deletedIdsRef = useRef(new Set()); + const handleRemove = useCallback( + (index: number) => { + const entry = filesRef.current[index]; + if (!entry) return; + if (entry.status === "uploading" && entry.abortController) { + entry.abortController.abort(); + } + if (entry.id && entry.status === "ok" && !deletedIdsRef.current.has(entry.id)) { + deletedIdsRef.current.add(entry.id); + void removeUnstructuredFile(blockId, entry.id).catch(() => {}); + } + onFilesChange((prev) => prev.filter((_, i) => i !== index)); + }, + [blockId, onFilesChange], + ); + + const handleDrop = useCallback( + (e: React.DragEvent) => { + e.preventDefault(); + setIsDragOver(false); + if (disabled) return; + const dropped = Array.from(e.dataTransfer.files); + handleFiles(dropped); + }, + [disabled, handleFiles], + ); + + const handleDragOver = useCallback( + (e: React.DragEvent) => { + e.preventDefault(); + if (!disabled) setIsDragOver(true); + }, + [disabled], + ); + + const handleDragLeave = useCallback(() => setIsDragOver(false), []); + + const handleClick = useCallback(() => { + if (!disabled) inputRef.current?.click(); + }, [disabled]); + + const handleInputChange = useCallback( + (e: React.ChangeEvent) => { + const selected = Array.from(e.target.files || []); + handleFiles(selected); + e.target.value = ""; + }, + [handleFiles], + ); + + const successFiles = files.filter((f) => f.status === "ok"); + + return ( +
+
+ +

+ Drop files here or click to browse +

+

+ PDF, DOCX, TXT, MD - up to 50MB each, 100MB total +

+
+ + + + {files.length > 0 && ( +
+ {files.map((entry, i) => ( +
+ {entry.status === "uploading" && ( + + )} + {entry.status === "ok" && ( + + )} + {entry.status === "error" && ( + + )} + {entry.name} + + {formatSize(entry.size)} + + {entry.error && ( + {entry.error} + )} + +
+ ))} +
+ {successFiles.length} file{successFiles.length !== 1 ? "s" : ""} uploaded + {formatSize(totalSize)} / 100MB +
+
+ )} +
+ ); +} + +export type { FileEntry }; diff --git a/studio/frontend/src/features/recipe-studio/dialogs/shared/field-label.tsx b/studio/frontend/src/features/recipe-studio/dialogs/shared/field-label.tsx index 100e9683a0..21ff4d73e7 100644 --- a/studio/frontend/src/features/recipe-studio/dialogs/shared/field-label.tsx +++ b/studio/frontend/src/features/recipe-studio/dialogs/shared/field-label.tsx @@ -18,7 +18,7 @@ export function FieldLabel({ hint, }: FieldLabelProps): ReactElement { return ( -
+
{htmlFor ? (