diff --git a/studio/backend/models/data_recipe.py b/studio/backend/models/data_recipe.py index d49a50d1e3..b382ddb3d0 100644 --- a/studio/backend/models/data_recipe.py +++ b/studio/backend/models/data_recipe.py @@ -9,7 +9,7 @@ from __future__ import annotations from typing import Any -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator class RecipePayload(BaseModel): @@ -76,13 +76,41 @@ class SeedInspectRequest(BaseModel): class SeedInspectUploadRequest(BaseModel): - filename: str = Field(min_length = 1) - content_base64: str = Field(min_length = 1) + # Legacy single-file flow (mutually exclusive with file_ids) + filename: str | None = None + content_base64: str | None = None + # Multi-file flow (mutually exclusive with content_base64) + block_id: str | None = None + file_ids: list[str] | None = None + file_names: list[str] | None = None + # Shared fields preview_size: int = Field(default = 10, ge = 1, le = 50) seed_source_type: str | None = None unstructured_chunk_size: int | None = Field(default = None, ge = 1, le = 20000) unstructured_chunk_overlap: int | None = Field(default = None, ge = 0, le = 20000) + @model_validator(mode = "after") + def _check_mutual_exclusivity(self) -> "SeedInspectUploadRequest": + has_legacy = self.content_base64 is not None + has_multi = self.file_ids is not None + if has_legacy and has_multi: + raise ValueError("Provide either content_base64 or file_ids, not both") + if not has_legacy and not has_multi: + raise ValueError("Provide either content_base64 or file_ids") + if has_multi: + if len(self.file_ids) == 0: + raise ValueError("file_ids must not be empty") + if not self.block_id: + raise ValueError("block_id is required when using file_ids") + if self.file_names is None or len(self.file_ids) != len(self.file_names): + raise ValueError( + "file_names must be provided and same length as file_ids" + ) + if has_legacy: + if not self.filename: + raise ValueError("filename is required when using content_base64") + return self + class SeedInspectResponse(BaseModel): dataset_name: str @@ -91,6 +119,15 @@ class SeedInspectResponse(BaseModel): preview_rows: list[dict[str, Any]] = Field(default_factory = list) split: str | None = None subset: str | None = None + resolved_paths: list[str] | None = None + + +class UnstructuredFileUploadResponse(BaseModel): + file_id: str + filename: str + size_bytes: int + status: str # "ok" or "error" + error: str | None = None class McpToolsListRequest(BaseModel): diff --git a/studio/backend/plugins/data-designer-unstructured-seed/pyproject.toml b/studio/backend/plugins/data-designer-unstructured-seed/pyproject.toml index f826ffd992..d4770a0b05 100644 --- a/studio/backend/plugins/data-designer-unstructured-seed/pyproject.toml +++ b/studio/backend/plugins/data-designer-unstructured-seed/pyproject.toml @@ -13,6 +13,9 @@ requires-python = ">=3.11" dependencies = [ "data-designer-engine>=0.5.1,<0.6", "pandas>=2,<3", + "pymupdf>=1.24.0", + "pymupdf4llm>=0.0.17", + "mammoth>=1.8.0", ] [project.entry-points."data_designer.plugins"] diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py index 80f51b2a24..f6fdf74612 100644 --- a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py +++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py @@ -8,6 +8,8 @@ import re from pathlib import Path from typing import Any +import pandas as pd + from utils.paths import ensure_dir, unstructured_seed_cache_root DEFAULT_CHUNK_SIZE = 1200 @@ -59,6 +61,59 @@ def build_unstructured_preview_rows( ] +def build_multi_file_preview_rows( + *, + file_entries: list[tuple[Path, str]], + preview_size: int, + chunk_size: int | None, + chunk_overlap: int | None, +) -> list[dict[str, str]]: + cs = _to_int(chunk_size, DEFAULT_CHUNK_SIZE) + co = _to_int(chunk_overlap, DEFAULT_CHUNK_OVERLAP) + _, rows = materialize_multi_file_unstructured_seed( + file_entries = file_entries, + chunk_size = cs, + chunk_overlap = co, + ) + return _round_robin_preview(rows, preview_size) + + +def _round_robin_preview( + rows: list[dict[str, str]], + preview_size: int, +) -> list[dict[str, str]]: + """Pick preview rows round-robin across source files so every file is represented.""" + if not rows or preview_size <= 0: + return [] + + # Group rows by source_file, preserving order of first appearance + from collections import OrderedDict + + grouped: OrderedDict[str, list[dict[str, str]]] = OrderedDict() + for row in rows: + key = row.get("source_file", "") + if key not in grouped: + grouped[key] = [] + grouped[key].append(row) + + result: list[dict[str, str]] = [] + iterators = [iter(chunks) for chunks in grouped.values()] + while len(result) < preview_size and iterators: + exhausted: list[int] = [] + for i, it in enumerate(iterators): + if len(result) >= preview_size: + break + val = next(it, None) + if val is not None: + result.append(val) + else: + exhausted.append(i) + for i in reversed(exhausted): + iterators.pop(i) + + return result + + def materialize_unstructured_seed_dataset( *, source_path: Path, @@ -103,6 +158,43 @@ def materialize_unstructured_seed_dataset( return parquet_path, rows +def materialize_multi_file_unstructured_seed( + *, + file_entries: list[tuple[Path, str]], # (extracted_txt_path, original_filename) + chunk_size: int, + chunk_overlap: int, +) -> tuple[Path, list[dict[str, str]]]: + """Chunk multiple files and combine into one parquet dataset with source_file column.""" + chunk_size, chunk_overlap = resolve_chunking(chunk_size, chunk_overlap) + cache_key = _compute_multi_file_cache_key(file_entries, chunk_size, chunk_overlap) + cached = _CACHE_DIR / f"{cache_key}.parquet" + if cached.exists(): + df = pd.read_parquet(cached) + rows = df.to_dict(orient = "records") + return cached, rows + + all_rows: list[dict[str, str]] = [] + for txt_path, orig_name in file_entries: + text = load_unstructured_text_file(txt_path) + chunks = split_text_into_chunks( + text = text, + chunk_size = chunk_size, + chunk_overlap = chunk_overlap, + ) + for chunk in chunks: + all_rows.append({"chunk_text": chunk, "source_file": orig_name}) + + if not all_rows: + raise ValueError("No text found in any uploaded files.") + + df = pd.DataFrame(all_rows) + ensure_dir(_CACHE_DIR) + tmp = _CACHE_DIR / f"{cache_key}.tmp.parquet" + df.to_parquet(tmp, index = False) + tmp.replace(cached) + return cached, all_rows + + def load_unstructured_text_file(path: Path) -> str: ext = path.suffix.lower() if ext not in {".txt", ".md"}: @@ -193,3 +285,17 @@ def _compute_cache_key( ] ).encode("utf-8") return hashlib.sha256(payload).hexdigest() + + +def _compute_multi_file_cache_key( + file_entries: list[tuple[Path, str]], + chunk_size: int, + chunk_overlap: int, +) -> str: + parts: list[str] = [] + for path, name in sorted(file_entries, key = lambda e: e[1]): + st = path.stat() + parts.append(f"{path}|{st.st_size}|{st.st_mtime_ns}|{name}") + parts.append(f"cs={chunk_size}|co={chunk_overlap}") + raw = "\n".join(parts) + return hashlib.sha256(raw.encode()).hexdigest() diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/config.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/config.py index e0a0392a69..9bef34cbd6 100644 --- a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/config.py +++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/config.py @@ -6,7 +6,7 @@ from __future__ import annotations from pathlib import Path from typing import Literal -from pydantic import Field, field_validator +from pydantic import Field, field_validator, model_validator from data_designer.config.seed_source import SeedSource @@ -15,27 +15,37 @@ from .chunking import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, resolve_chunkin class UnstructuredSeedSource(SeedSource): seed_type: Literal["unstructured"] = "unstructured" - path: str = Field(..., min_length = 1) + paths: list[str] = Field(min_length = 1) + + @model_validator(mode = "before") + @classmethod + def _normalize_legacy_path(cls, data): + if isinstance(data, dict) and "paths" not in data and data.get("path"): + data = dict(data) + data["paths"] = [data["path"]] + return data + chunk_size: int = DEFAULT_CHUNK_SIZE chunk_overlap: int = DEFAULT_CHUNK_OVERLAP - @field_validator("path", mode = "after") + @field_validator("paths") @classmethod - def _validate_path(cls, value: str) -> str: - path = Path(value).expanduser() - if not path.is_file(): - raise ValueError(f"Unstructured seed path is not a file: {path}") - return value + def _validate_paths(cls, v: list[str]) -> list[str]: + for p in v: + expanded = Path(p).expanduser() + if not expanded.is_file(): + raise ValueError(f"Seed file does not exist: {expanded}") + return v - @field_validator("chunk_size", mode = "after") + @field_validator("chunk_size") @classmethod - def _validate_chunk_size(cls, value: int) -> int: - size, _ = resolve_chunking(value, 0) - return size + def _resolve_chunk_size(cls, v: int) -> int: + cs, _ = resolve_chunking(v, 0) + return cs - @field_validator("chunk_overlap", mode = "after") + @field_validator("chunk_overlap") @classmethod - def _validate_chunk_overlap(cls, value: int, info) -> int: - size = info.data.get("chunk_size", cls.model_fields["chunk_size"].default) - _, overlap = resolve_chunking(size, value) - return overlap + def _resolve_chunk_overlap(cls, v: int, info) -> int: + cs = info.data.get("chunk_size", DEFAULT_CHUNK_SIZE) + _, co = resolve_chunking(cs, v) + return co diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py index 8a3deb9b92..7272e426ad 100644 --- a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py +++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py @@ -8,7 +8,6 @@ from pathlib import Path import data_designer.lazy_heavy_imports as lazy from data_designer.engine.resources.seed_reader import SeedReader -from .chunking import materialize_unstructured_seed_dataset from .config import UnstructuredSeedSource @@ -17,8 +16,25 @@ class UnstructuredSeedReader(SeedReader[UnstructuredSeedSource]): return lazy.duckdb.connect() def get_dataset_uri(self) -> str: - path, _ = materialize_unstructured_seed_dataset( - source_path = Path(self.source.path), + from .chunking import materialize_multi_file_unstructured_seed + import json as json_mod + + file_entries: list[tuple[Path, str]] = [] + for p in self.source.paths: + path_obj = Path(p) + file_id = path_obj.name.replace(".extracted.txt", "") + meta_path = path_obj.parent / f"{file_id}.meta.json" + orig_name = path_obj.name + if meta_path.exists(): + try: + meta = json_mod.loads(meta_path.read_text()) + orig_name = meta.get("original_filename", path_obj.name) + except (json_mod.JSONDecodeError, OSError): + pass + file_entries.append((path_obj, orig_name)) + + path, _ = materialize_multi_file_unstructured_seed( + file_entries = file_entries, chunk_size = self.source.chunk_size, chunk_overlap = self.source.chunk_overlap, ) diff --git a/studio/backend/requirements/single-env/data-designer-deps.txt b/studio/backend/requirements/single-env/data-designer-deps.txt index dfc5b9bf21..e0b3d8b72a 100644 --- a/studio/backend/requirements/single-env/data-designer-deps.txt +++ b/studio/backend/requirements/single-env/data-designer-deps.txt @@ -17,3 +17,6 @@ ruff<1,>=0.14.10 scipy<2,>=1.11.0 sqlfluff<4,>=3.2.0 tiktoken<1,>=0.8.0 +pymupdf>=1.24.0 +pymupdf4llm>=0.0.17 +mammoth>=1.8.0 diff --git a/studio/backend/routes/data_recipe/seed.py b/studio/backend/routes/data_recipe/seed.py index 1765550adc..60fb1d7916 100644 --- a/studio/backend/routes/data_recipe/seed.py +++ b/studio/backend/routes/data_recipe/seed.py @@ -7,23 +7,27 @@ from __future__ import annotations import base64 import binascii +import json +import re from itertools import islice from pathlib import Path from typing import Any from uuid import uuid4 -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, HTTPException, UploadFile, File as FastAPIFile, Form from data_designer_unstructured_seed.chunking import ( build_unstructured_preview_rows, + normalize_unstructured_text, resolve_chunking, ) from core.data_recipe.jsonable import to_preview_jsonable -from utils.paths import ensure_dir, seed_uploads_root +from utils.paths import ensure_dir, seed_uploads_root, unstructured_uploads_root from models.data_recipe import ( SeedInspectRequest, SeedInspectResponse, SeedInspectUploadRequest, + UnstructuredFileUploadResponse, ) router = APIRouter() @@ -31,8 +35,21 @@ router = APIRouter() DATA_EXTS = (".parquet", ".jsonl", ".json", ".csv") DEFAULT_SPLIT = "train" LOCAL_UPLOAD_EXTS = {".csv", ".json", ".jsonl"} -UNSTRUCTURED_UPLOAD_EXTS = {".txt", ".md"} +UNSTRUCTURED_ALLOWED_EXTS = {".pdf", ".docx", ".txt", ".md"} SEED_UPLOAD_DIR = seed_uploads_root() +UNSTRUCTURED_UPLOAD_ROOT = unstructured_uploads_root() +MAX_FILE_SIZE = 50 * 1024 * 1024 # 50MB +MAX_TOTAL_SIZE = 100 * 1024 * 1024 # 100MB + +_SAFE_ID_RE = re.compile(r"^[a-zA-Z0-9_-]+$") + + +def _validate_safe_id(value: str, label: str) -> str: + if not value or not _SAFE_ID_RE.match(value): + raise HTTPException( + 400, f"Invalid {label}: must be alphanumeric/dash/underscore only" + ) + return value def _serialize_preview_value(value: Any) -> Any: @@ -177,7 +194,17 @@ def _read_preview_rows_from_local_file( ext = path.suffix.lower() try: if ext == ".csv": - df = pd.read_csv(path, nrows = preview_size) + df = pd.read_csv(path, nrows = preview_size, encoding = "utf-8-sig") + df.columns = df.columns.str.strip() + unnamed = [c for c in df.columns if c == "" or c.startswith("Unnamed:")] + if unnamed: + df = df.drop(columns = unnamed) + full_df = pd.read_csv(path, encoding = "utf-8-sig") + full_df.columns = full_df.columns.str.strip() + full_df = full_df.drop(columns = unnamed) + tmp_csv = path.with_suffix(".tmp.csv") + full_df.to_csv(tmp_csv, index = False, encoding = "utf-8") + tmp_csv.replace(path) elif ext == ".jsonl": df = pd.read_json(path, lines = True).head(preview_size) elif ext == ".json": @@ -220,6 +247,36 @@ def _read_preview_rows_from_unstructured_file( return _serialize_preview_rows(rows) +def _read_preview_rows_from_multi_files( + *, + block_id: str, + file_ids: list[str], + file_names: list[str], + preview_size: int, + chunk_size: int | None, + chunk_overlap: int | None, +) -> list[dict[str, str]]: + from data_designer_unstructured_seed.chunking import build_multi_file_preview_rows + + _validate_safe_id(block_id, "block_id") + block_dir = UNSTRUCTURED_UPLOAD_ROOT / block_id + file_entries: list[tuple[Path, str]] = [] + for fid, fname in zip(file_ids, file_names): + extracted = block_dir / f"{fid}.extracted.txt" + if not extracted.exists(): + raise HTTPException( + 404, f"Extracted text not found for file: {fname} (id: {fid})" + ) + file_entries.append((extracted, fname)) + + return build_multi_file_preview_rows( + file_entries = file_entries, + preview_size = preview_size, + chunk_size = chunk_size, + chunk_overlap = chunk_overlap, + ) + + @router.post("/seed/inspect", response_model = SeedInspectResponse) def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse: dataset_name = payload.dataset_name.strip() @@ -306,14 +363,202 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse: ) +def _extract_text_from_file(file_path: Path, ext: str) -> str: + """Extract text from uploaded file based on extension, converting to markdown where possible.""" + if ext in {".txt", ".md"}: + raw = file_path.read_text(encoding = "utf-8", errors = "ignore") + elif ext == ".pdf": + import pymupdf4llm + + raw = pymupdf4llm.to_markdown( + str(file_path), write_images = False, show_progress = False + ) + elif ext == ".docx": + import mammoth + + with open(str(file_path), "rb") as f: + result = mammoth.convert_to_markdown(f) + raw = result.value + else: + raise ValueError(f"Unsupported file type: {ext}") + + return normalize_unstructured_text(raw) + + +def _get_block_total_size(block_dir: Path, file_ids: list[str]) -> int: + """Sum raw upload sizes for tracked file IDs only.""" + if not block_dir.exists() or not file_ids: + return 0 + id_set = set(file_ids) + total = 0 + for f in block_dir.iterdir(): + if not f.is_file(): + continue + if f.name.endswith(".extracted.txt") or f.name.endswith(".meta.json"): + continue + stem = f.name.split(".")[0] + if stem in id_set: + total += f.stat().st_size + return total + + +@router.post("/seed/upload-unstructured-file") +async def upload_unstructured_file( + file: UploadFile = FastAPIFile(...), + block_id: str = Form(...), + existing_file_ids: str = Form(""), +) -> UnstructuredFileUploadResponse: + _validate_safe_id(block_id, "block_id") + + tracked_ids = [fid.strip() for fid in existing_file_ids.split(",") if fid.strip()] + + original_filename = file.filename or "upload" + ext = Path(original_filename).suffix.lower() + if ext not in UNSTRUCTURED_ALLOWED_EXTS: + raise HTTPException( + 400, + f"Unsupported file type: {ext}. Allowed: {', '.join(sorted(UNSTRUCTURED_ALLOWED_EXTS))}", + ) + + content = await file.read() + size_bytes = len(content) + + if size_bytes == 0: + raise HTTPException(400, "Empty file not allowed") + + if size_bytes > MAX_FILE_SIZE: + raise HTTPException( + 413, f"File too large ({size_bytes} bytes). Maximum is 50MB." + ) + + block_dir = UNSTRUCTURED_UPLOAD_ROOT / block_id + ensure_dir(block_dir) + current_total = _get_block_total_size(block_dir, file_ids = tracked_ids) + if current_total + size_bytes > MAX_TOTAL_SIZE: + raise HTTPException( + 413, f"Total upload limit ({MAX_TOTAL_SIZE // (1024 * 1024)}MB) exceeded" + ) + + file_id = uuid4().hex + raw_path = block_dir / f"{file_id}{ext}" + raw_path.write_bytes(content) + + extracted_path = block_dir / f"{file_id}.extracted.txt" + try: + extracted_text = _extract_text_from_file(raw_path, ext) + if not extracted_text or not extracted_text.strip(): + raw_path.unlink(missing_ok = True) + return UnstructuredFileUploadResponse( + file_id = file_id, + filename = original_filename, + size_bytes = size_bytes, + status = "error", + error = "No extractable text found in file", + ) + extracted_path.write_text(extracted_text, encoding = "utf-8") + except Exception as e: + raw_path.unlink(missing_ok = True) + extracted_path.unlink(missing_ok = True) + return UnstructuredFileUploadResponse( + file_id = file_id, + filename = original_filename, + size_bytes = size_bytes, + status = "error", + error = f"Text extraction failed: {type(e).__name__}: {e}", + ) + + try: + meta_path = block_dir / f"{file_id}.meta.json" + meta_path.write_text( + json.dumps( + {"original_filename": original_filename, "size_bytes": size_bytes} + ), + encoding = "utf-8", + ) + except OSError: + raw_path.unlink(missing_ok = True) + extracted_path.unlink(missing_ok = True) + return UnstructuredFileUploadResponse( + file_id = file_id, + filename = original_filename, + size_bytes = size_bytes, + status = "error", + error = "Failed to save file metadata", + ) + + return UnstructuredFileUploadResponse( + file_id = file_id, + filename = original_filename, + size_bytes = size_bytes, + status = "ok", + ) + + +@router.delete("/seed/unstructured-file/{block_id}/{file_id}") +async def remove_unstructured_file(block_id: str, file_id: str): + _validate_safe_id(block_id, "block_id") + _validate_safe_id(file_id, "file_id") + + block_dir = UNSTRUCTURED_UPLOAD_ROOT / block_id + if not block_dir.exists(): + raise HTTPException(404, "Block not found") + + deleted = False + for f in block_dir.iterdir(): + stem = f.name.split(".")[0] + if stem == file_id: + f.unlink(missing_ok = True) + deleted = True + + if not deleted: + raise HTTPException(404, "File not found") + try: + if not any(block_dir.iterdir()): + block_dir.rmdir() + except OSError: + pass + + return {"status": "ok"} + + @router.post("/seed/inspect-upload", response_model = SeedInspectResponse) def inspect_seed_upload(payload: SeedInspectUploadRequest) -> SeedInspectResponse: + if payload.file_ids is not None: + if len(payload.file_ids) == 0: + raise HTTPException(400, "file_ids must not be empty") + _validate_safe_id(payload.block_id, "block_id") + for fid in payload.file_ids: + _validate_safe_id(fid, "file_id") + preview_rows = _read_preview_rows_from_multi_files( + block_id = payload.block_id, + file_ids = payload.file_ids, + file_names = payload.file_names, + preview_size = payload.preview_size, + chunk_size = payload.unstructured_chunk_size, + chunk_overlap = payload.unstructured_chunk_overlap, + ) + columns = ["chunk_text", "source_file"] if preview_rows else [] + resolved_paths = [ + str(UNSTRUCTURED_UPLOAD_ROOT / payload.block_id / f"{fid}.extracted.txt") + for fid in payload.file_ids + ] + return SeedInspectResponse( + dataset_name = "unstructured_seed", + resolved_path = resolved_paths[0] if resolved_paths else "", + resolved_paths = resolved_paths, + columns = columns, + preview_rows = _serialize_preview_rows(preview_rows), + ) + seed_source_type = _normalize_optional_text(payload.seed_source_type) or "local" filename = _sanitize_filename(payload.filename) ext = Path(filename).suffix.lower() + # Legacy single-file unstructured path only supports .txt/.md + # PDF/DOCX extraction uses the multi-file upload endpoint instead + _LEGACY_UNSTRUCTURED_EXTS = {".txt", ".md"} if seed_source_type == "unstructured": - if ext not in UNSTRUCTURED_UPLOAD_EXTS: - allowed = ", ".join(sorted(UNSTRUCTURED_UPLOAD_EXTS)) + if ext not in _LEGACY_UNSTRUCTURED_EXTS: + allowed = ", ".join(sorted(_LEGACY_UNSTRUCTURED_EXTS)) raise HTTPException( status_code = 400, detail = f"unsupported file type: {ext}. allowed: {allowed}", @@ -329,8 +574,7 @@ def inspect_seed_upload(payload: SeedInspectUploadRequest) -> SeedInspectRespons file_bytes = _decode_base64_payload(payload.content_base64) if not file_bytes: raise HTTPException(status_code = 400, detail = "empty upload payload") - max_size_bytes = 50 * 1024 * 1024 - if len(file_bytes) > max_size_bytes: + if len(file_bytes) > MAX_FILE_SIZE: raise HTTPException(status_code = 413, detail = "file too large (max 50MB)") ensure_dir(SEED_UPLOAD_DIR) diff --git a/studio/backend/utils/paths/__init__.py b/studio/backend/utils/paths/__init__.py index 507fb1106b..90df216f96 100644 --- a/studio/backend/utils/paths/__init__.py +++ b/studio/backend/utils/paths/__init__.py @@ -19,6 +19,7 @@ from .storage_roots import ( tmp_root, seed_uploads_root, unstructured_seed_cache_root, + unstructured_uploads_root, oxc_validator_tmp_root, tensorboard_root, ensure_dir, @@ -47,6 +48,7 @@ __all__ = [ "tmp_root", "seed_uploads_root", "unstructured_seed_cache_root", + "unstructured_uploads_root", "oxc_validator_tmp_root", "tensorboard_root", "ensure_dir", diff --git a/studio/backend/utils/paths/storage_roots.py b/studio/backend/utils/paths/storage_roots.py index 81763c3095..f14887e119 100644 --- a/studio/backend/utils/paths/storage_roots.py +++ b/studio/backend/utils/paths/storage_roots.py @@ -54,13 +54,17 @@ def tmp_root() -> Path: def seed_uploads_root() -> Path: - return tmp_root() / "seed-uploads" + return datasets_root() / "seed-uploads" def unstructured_seed_cache_root() -> Path: return tmp_root() / "unstructured-seed-cache" +def unstructured_uploads_root() -> Path: + return datasets_root() / "unstructured-uploads" + + def oxc_validator_tmp_root() -> Path: return tmp_root() / "oxc-validator" @@ -104,6 +108,7 @@ def ensure_studio_directories() -> None: datasets_root, dataset_uploads_root, recipe_datasets_root, + unstructured_uploads_root, outputs_root, exports_root, auth_root, diff --git a/studio/frontend/src/features/recipe-studio/api/index.ts b/studio/frontend/src/features/recipe-studio/api/index.ts index 7225450154..9212e4db9b 100644 --- a/studio/frontend/src/features/recipe-studio/api/index.ts +++ b/studio/frontend/src/features/recipe-studio/api/index.ts @@ -103,13 +103,22 @@ export type SeedInspectRequest = { }; export type SeedInspectUploadRequest = { - filename: string; - // base64 payload without data URL prefix - content_base64: string; + // Legacy single-file + filename?: string; + // biome-ignore lint/style/useNamingConvention: api schema + content_base64?: string; + // Multi-file + // biome-ignore lint/style/useNamingConvention: api schema + block_id?: string; + // biome-ignore lint/style/useNamingConvention: api schema + file_ids?: string[]; + // biome-ignore lint/style/useNamingConvention: api schema + file_names?: string[]; + // Shared // biome-ignore lint/style/useNamingConvention: api schema preview_size?: number; // biome-ignore lint/style/useNamingConvention: api schema - seed_source_type?: "local" | "unstructured"; + seed_source_type?: string; // biome-ignore lint/style/useNamingConvention: api schema unstructured_chunk_size?: number; // biome-ignore lint/style/useNamingConvention: api schema @@ -126,6 +135,8 @@ export type SeedInspectResponse = { preview_rows: Record[]; split?: string | null; subset?: string | null; + // biome-ignore lint/style/useNamingConvention: api schema + resolved_paths?: string[] | null; }; export type ValidateError = { @@ -372,3 +383,64 @@ export async function streamRecipeJobEvents(options: { } // NOTE: preview endpoints removed from harness. + +type UnstructuredFileUploadResponse = { + // biome-ignore lint/style/useNamingConvention: api schema + file_id: string; + filename: string; + // biome-ignore lint/style/useNamingConvention: api schema + size_bytes: number; + status: "ok" | "error"; + error?: string; +}; + +export async function uploadUnstructuredFile( + file: File, + blockId: string, + signal?: AbortSignal, + existingFileIds?: string[], +): Promise { + const formData = new FormData(); + formData.append("file", file); + formData.append("block_id", blockId); + if (existingFileIds?.length) { + formData.append("existing_file_ids", existingFileIds.join(",")); + } + + const res = await authFetch(`${DATA_DESIGNER_API_BASE}/seed/upload-unstructured-file`, { + method: "POST", + body: formData, + signal, + }); + + if (res.status === 413) { + const detail = await res.json().catch(() => ({ detail: "File too large" })); + return { + file_id: "", + filename: file.name, + size_bytes: file.size, + status: "error", + error: typeof detail.detail === "string" ? detail.detail : "File too large", + }; + } + + if (!res.ok) { + const detail = await res.json().catch(() => ({ detail: "Upload failed" })); + throw new Error(typeof detail.detail === "string" ? detail.detail : "Upload failed"); + } + + return res.json(); +} + +export async function removeUnstructuredFile( + blockId: string, + fileId: string, +): Promise { + const res = await authFetch( + `${DATA_DESIGNER_API_BASE}/seed/unstructured-file/${encodeURIComponent(blockId)}/${encodeURIComponent(fileId)}`, + { method: "DELETE" }, + ); + if (!res.ok && res.status !== 404) { + throw new Error("Failed to remove file"); + } +} diff --git a/studio/frontend/src/features/recipe-studio/components/controls/layout-controls.tsx b/studio/frontend/src/features/recipe-studio/components/controls/layout-controls.tsx index f4b76c8d2c..2a38f30059 100644 --- a/studio/frontend/src/features/recipe-studio/components/controls/layout-controls.tsx +++ b/studio/frontend/src/features/recipe-studio/components/controls/layout-controls.tsx @@ -8,7 +8,7 @@ import { useUpdateNodeInternals, } from "@xyflow/react"; import { Button } from "@/components/ui/button"; -import { getFitNodeIdsIgnoringNotes } from "../../utils/graph/fit-view"; +import { buildFitViewOptions } from "../../utils/graph/fit-view"; type LayoutControlsProps = { direction: "LR" | "TB"; @@ -36,10 +36,7 @@ export function LayoutControls({ requestAnimationFrame(() => { refreshNodeInternals(); requestAnimationFrame(() => { - fitView({ - duration: 250, - nodes: getFitNodeIdsIgnoringNotes(getNodes()), - }); + fitView(buildFitViewOptions(getNodes())); }); }); }, [fitView, getNodes, onLayout, refreshNodeInternals]); @@ -51,10 +48,7 @@ export function LayoutControls({ requestAnimationFrame(() => { refreshNodeInternals(); requestAnimationFrame(() => { - fitView({ - duration: 250, - nodes: getFitNodeIdsIgnoringNotes(getNodes()), - }); + fitView(buildFitViewOptions(getNodes())); }); }); }); diff --git a/studio/frontend/src/features/recipe-studio/components/controls/viewport-controls.tsx b/studio/frontend/src/features/recipe-studio/components/controls/viewport-controls.tsx index 261ebe1b11..0a0eff849b 100644 --- a/studio/frontend/src/features/recipe-studio/components/controls/viewport-controls.tsx +++ b/studio/frontend/src/features/recipe-studio/components/controls/viewport-controls.tsx @@ -5,7 +5,7 @@ import { type ReactElement, useCallback } from "react"; import { Lock, LockOpen, Maximize2, Minus, Plus } from "lucide-react"; import { Panel, useReactFlow } from "@xyflow/react"; import { Button } from "@/components/ui/button"; -import { getFitNodeIdsIgnoringNotes } from "../../utils/graph/fit-view"; +import { buildFitViewOptions } from "../../utils/graph/fit-view"; import { RECIPE_FLOATING_ICON_BUTTON_CLASS } from "../recipe-floating-icon-button-class"; type ViewportControlsProps = { @@ -30,10 +30,7 @@ export function ViewportControls({ }, [zoomOut]); const handleFitView = useCallback(() => { - fitView({ - duration: 250, - nodes: getFitNodeIdsIgnoringNotes(getNodes()), - }); + fitView(buildFitViewOptions(getNodes())); }, [fitView, getNodes]); return ( diff --git a/studio/frontend/src/features/recipe-studio/components/inline/inline-seed.tsx b/studio/frontend/src/features/recipe-studio/components/inline/inline-seed.tsx index 2162f75ccf..bf022fa028 100644 --- a/studio/frontend/src/features/recipe-studio/components/inline/inline-seed.tsx +++ b/studio/frontend/src/features/recipe-studio/components/inline/inline-seed.tsx @@ -45,7 +45,9 @@ export function InlineSeed({ config, onUpdate }: InlineSeedProps): ReactElement const isLocal = mode === "local"; const fileName = isLocal ? config.local_file_name?.trim() - : config.unstructured_file_name?.trim(); + : config.unstructured_file_names?.length + ? `${config.unstructured_file_names.length} file${config.unstructured_file_names.length !== 1 ? "s" : ""}` + : undefined; return (
diff --git a/studio/frontend/src/features/recipe-studio/components/recipe-graph-node.tsx b/studio/frontend/src/features/recipe-studio/components/recipe-graph-node.tsx index 672c29f4ab..0fcf202190 100644 --- a/studio/frontend/src/features/recipe-studio/components/recipe-graph-node.tsx +++ b/studio/frontend/src/features/recipe-studio/components/recipe-graph-node.tsx @@ -256,9 +256,10 @@ function getConfigSummary(config: NodeConfig | undefined): string { } if ( seedSourceType === "unstructured" && - config.unstructured_file_name?.trim() + config.unstructured_file_names?.length ) { - return config.unstructured_file_name.trim(); + const count = config.unstructured_file_names.length; + return `${count} file${count !== 1 ? "s" : ""} uploaded`; } if (config.hf_path.trim()) { return config.hf_path.trim(); diff --git a/studio/frontend/src/features/recipe-studio/dialogs/expression/expression-dialog.tsx b/studio/frontend/src/features/recipe-studio/dialogs/expression/expression-dialog.tsx index 0947851ac7..0e072d5858 100644 --- a/studio/frontend/src/features/recipe-studio/dialogs/expression/expression-dialog.tsx +++ b/studio/frontend/src/features/recipe-studio/dialogs/expression/expression-dialog.tsx @@ -58,7 +58,7 @@ export function ExpressionDialog({ value={config.name} onChange={(value) => onUpdate({ name: value })} /> -
+
-
+
Import recipe -
+
) : null} -
+
)} {(hasToolProfiles || Boolean(config.tool_alias?.trim())) && ( -
+
)} {config.llm_type === "code" && ( -
+
)} -
+
{imageContext.enabled && ( -
+
)} {config.llm_type === "structured" && ( -
+
-
+
)}
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
updateField("endpoint", event.target.value)} />
-
+
-
+
updateField("api_key_env", event.target.value)} />
-
+
updateField("extra_headers", event.target.value)} />
-
+
+
-
+
{kind === "full" && ( -
+
)} -
+
-
+
updateSchema({ name: event.target.value })} />
-
+
onUpdate({ name: value })} /> -
+
onUpdate({ name: value })} />
-
+
-
+
-
+

Rule weights (optional)

diff --git a/studio/frontend/src/features/recipe-studio/dialogs/samplers/datetime-dialog.tsx b/studio/frontend/src/features/recipe-studio/dialogs/samplers/datetime-dialog.tsx index e75551ec75..930c748585 100644 --- a/studio/frontend/src/features/recipe-studio/dialogs/samplers/datetime-dialog.tsx +++ b/studio/frontend/src/features/recipe-studio/dialogs/samplers/datetime-dialog.tsx @@ -50,7 +50,7 @@ export function DatetimeDialog({ />
-
+
-
+
-
+
onUpdate({ name: value })} />
-
+
onUpdate({ mean: event.target.value })} />
-
+
-
+
Faker

-
+
-
+
-
+
-
+
onUpdate({ name: value })} />
-
+
onUpdate({ name: value })} />
-
+
updateField("dt_min", event.target.value)} />
-
+
-
+
-
+
onUpdate({ name: value })} />
-
+
onUpdate({ low: event.target.value })} />
-
+
-
+
onUpdate({ name: value })} /> -
+
= [ ]; const LOCAL_ACCEPT = ".csv,.json,.jsonl"; -const UNSTRUCTURED_ACCEPT = ".txt,.pdf,.docx"; const MAX_UPLOAD_BYTES = 50 * 1024 * 1024; const DEFAULT_CHUNK_SIZE = 1200; const DEFAULT_CHUNK_OVERLAP = 200; @@ -112,20 +110,20 @@ function getPreviewEmptyStateCopy(mode: SeedConfig["seed_source_type"]): { } { if (mode === "local") { return { - title: "No local preview yet", - description: "Choose a CSV/JSON/JSONL file, then click Load to fetch 10 rows.", + title: "No preview yet", + description: "Upload a CSV, JSON, or JSONL file and click Load to see a sample.", }; } if (mode === "unstructured") { return { - title: "No chunk preview yet", + title: "No preview yet", description: - "Choose a TXT/PDF/DOCX file, then click Load to extract + preview chunk_text rows.", + "Upload your documents and the preview will appear once processing is done.", }; } return { - title: "No dataset preview yet", - description: "Pick a Hugging Face dataset and click Load to fetch 10 sample rows.", + title: "No preview yet", + description: "Select a Hugging Face dataset and click Load to see a sample.", }; } @@ -177,42 +175,6 @@ async function fileToBase64Payload(file: File): Promise { }); } -async function extractUnstructuredText(file: File): Promise { - const lower = file.name.toLowerCase(); - if (lower.endsWith(".txt")) { - return file.text(); - } - if (lower.endsWith(".pdf")) { - const buffer = new Uint8Array(await file.arrayBuffer()); - const pdf = await getDocumentProxy(buffer); - const { text } = await extractText(pdf, { mergePages: true }); - return text; - } - if (lower.endsWith(".docx")) { - const arrayBuffer = await file.arrayBuffer(); - const { value } = await mammoth.extractRawText({ arrayBuffer }); - return value; - } - throw new Error("Unsupported unstructured file type"); -} - -async function toUnstructuredUploadFile(file: File): Promise { - const lower = file.name.toLowerCase(); - if (lower.endsWith(".txt") || lower.endsWith(".md")) { - return file; - } - - const text = (await extractUnstructuredText(file)).trim(); - if (!text) { - throw new Error("No text found in file."); - } - const normalized = text.replace(/\r\n/g, "\n").replace(/\r/g, "\n"); - const stem = file.name.replace(/\.(pdf|docx)$/i, "") || "unstructured_seed"; - return new File([normalized], `${stem}.txt`, { - type: "text/plain", - }); -} - export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactElement { const [inspectError, setInspectError] = useState(null); const [isInspecting, setIsInspecting] = useState(false); @@ -220,16 +182,84 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl const [previewRows, setPreviewRows] = useState[]>([]); const [expandedPreviewRows, setExpandedPreviewRows] = useState>({}); const [localFile, setLocalFile] = useState(null); - const [unstructuredFile, setUnstructuredFile] = useState(null); + const [unstructuredFiles, setUnstructuredFiles] = useState(() => { + if (config.unstructured_file_ids?.length) { + return config.unstructured_file_ids.map((id, i) => ({ + id, + name: config.unstructured_file_names?.[i] ?? "Unknown", + size: config.unstructured_file_sizes?.[i] ?? 0, + status: "ok" as const, + })); + } + return []; + }); const mode = config.seed_source_type ?? "hf"; const previewEmpty = getPreviewEmptyStateCopy(mode); + const prevModeRef = useRef(mode); useEffect(() => { + const prevMode = prevModeRef.current; + prevModeRef.current = mode; setInspectError(null); setLocalFile(null); - setUnstructuredFile(null); - }, [mode]); + if (prevMode === "unstructured" && mode !== "unstructured") { + setUnstructuredFiles([]); + } + if (prevMode !== "unstructured" && mode === "unstructured") { + if (config.unstructured_file_ids?.length) { + setUnstructuredFiles( + config.unstructured_file_ids.map((id, i) => ({ + id, + name: config.unstructured_file_names?.[i] ?? "Unknown", + size: config.unstructured_file_sizes?.[i] ?? 0, + status: "ok" as const, + })), + ); + } else { + setUnstructuredFiles([]); + } + } + }, [mode]); // eslint-disable-line react-hooks/exhaustive-deps + + const didSyncFilesRef = useRef(false); + useEffect(() => { + if (!open) { + didSyncFilesRef.current = false; + return; + } + if (didSyncFilesRef.current) return; + if (mode !== "unstructured") return; + if (unstructuredFiles.length > 0) return; + if (!config.unstructured_file_ids?.length) return; + didSyncFilesRef.current = true; + setUnstructuredFiles( + config.unstructured_file_ids.map((id, i) => ({ + id, + name: config.unstructured_file_names?.[i] ?? "Unknown", + size: config.unstructured_file_sizes?.[i] ?? 0, + status: "ok" as const, + })), + ); + }, [open, mode, unstructuredFiles.length, config.unstructured_file_ids, config.unstructured_file_names, config.unstructured_file_sizes]); + + const handleUnstructuredFilesChange = useCallback( + (updater: FileEntry[] | ((prev: FileEntry[]) => FileEntry[])) => { + setUnstructuredFiles((prev) => { + const next = typeof updater === "function" ? updater(prev) : updater; + const okFiles = next.filter((f) => f.status === "ok"); + queueMicrotask(() => { + onUpdate({ + unstructured_file_ids: okFiles.map((f) => f.id), + unstructured_file_names: okFiles.map((f) => f.name), + unstructured_file_sizes: okFiles.map((f) => f.size), + }); + }); + return next; + }); + }, + [onUpdate], + ); useEffect(() => { setPreviewRows(config.seed_preview_rows ?? []); @@ -256,14 +286,16 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl if (!localFile) return null; return `local:${localFile.name}|${localFile.size}|${localFile.lastModified}`; } - if (!unstructuredFile) return null; + const okFiles = unstructuredFiles.filter((f) => f.status === "ok"); + if (okFiles.length === 0) return null; const { chunkSize, chunkOverlap } = resolveChunking(config); - return `unstructured:${unstructuredFile.name}|${unstructuredFile.size}|${unstructuredFile.lastModified}|${chunkSize}|${chunkOverlap}`; + const fileKey = okFiles.map((f) => `${f.id}|${f.name}`).join(","); + return `unstructured:${fileKey}|${chunkSize}|${chunkOverlap}`; }, [ config, localFile, mode, - unstructuredFile, + unstructuredFiles, ]); const loadSeedMetadata = useCallback(async (opts?: { silent?: boolean }): Promise => { @@ -295,7 +327,9 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl hf_split: response.split ?? "", hf_subset: response.subset ?? "", local_file_name: "", - unstructured_file_name: "", + unstructured_file_ids: [], + unstructured_file_names: [], + unstructured_file_sizes: [], }); setPreviewRows(response.preview_rows ?? []); setLastLoadedKey(loadKey); @@ -326,50 +360,56 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl hf_subset: "", hf_split: "", local_file_name: localFile.name, - unstructured_file_name: "", + unstructured_file_ids: [], + unstructured_file_names: [], + unstructured_file_sizes: [], }); setPreviewRows(response.preview_rows ?? []); setLastLoadedKey(loadKey); return true; } - if (!unstructuredFile) { - throw new Error("Select a PDF/DOCX/TXT file first."); - } - if (unstructuredFile.size > MAX_UPLOAD_BYTES) { - throw new Error("File too large (max 50MB)."); + if (mode === "unstructured") { + const fileIds = unstructuredFiles + .filter((f) => f.status === "ok") + .map((f) => f.id); + const fileNames = unstructuredFiles + .filter((f) => f.status === "ok") + .map((f) => f.name); + + if (fileIds.length === 0) { + setInspectError("No files uploaded"); + return false; + } + + const { chunkSize, chunkOverlap } = resolveChunking(config); + const response = await inspectSeedUpload({ + block_id: config.id, + file_ids: fileIds, + file_names: fileNames, + preview_size: 10, + seed_source_type: "unstructured", + unstructured_chunk_size: chunkSize, + unstructured_chunk_overlap: chunkOverlap, + }); + + onUpdate({ + hf_path: response.resolved_path, + resolved_paths: response.resolved_paths ?? [], + seed_columns: response.columns, + seed_preview_rows: response.preview_rows ?? [], + unstructured_file_ids: fileIds, + unstructured_file_names: fileNames, + unstructured_file_sizes: unstructuredFiles + .filter((f) => f.status === "ok") + .map((f) => f.size), + }); + setPreviewRows(response.preview_rows ?? []); + setLastLoadedKey(loadKey); + return true; } - const { chunkSize, chunkOverlap } = resolveChunking(config); - const uploadFile = await toUnstructuredUploadFile(unstructuredFile); - if (uploadFile.size > MAX_UPLOAD_BYTES) { - throw new Error("Processed text is too large (max 50MB)."); - } - const payload = await fileToBase64Payload(uploadFile); - const response = await inspectSeedUpload({ - filename: uploadFile.name, - content_base64: payload, - preview_size: 10, - seed_source_type: "unstructured", - unstructured_chunk_size: chunkSize, - unstructured_chunk_overlap: chunkOverlap, - }); - onUpdate({ - hf_path: response.resolved_path, - seed_columns: response.columns, - seed_drop_columns: (config.seed_drop_columns ?? []).filter((name) => - response.columns.includes(name), - ), - seed_preview_rows: response.preview_rows ?? [], - hf_repo_id: "", - hf_subset: "", - hf_split: "", - local_file_name: "", - unstructured_file_name: unstructuredFile.name, - }); - setPreviewRows(response.preview_rows ?? []); - setLastLoadedKey(loadKey); - return true; + return false; } catch (error) { if (!opts?.silent) { setInspectError(getErrorMessage(error, "Failed to load seed metadata.")); @@ -385,7 +425,7 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl localFile, mode, onUpdate, - unstructuredFile, + unstructuredFiles, ]); useEffect(() => { @@ -401,6 +441,21 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl void loadSeedMetadata({ silent: true }); }, [getCurrentLoadKey, isInspecting, lastLoadedKey, loadSeedMetadata, open]); + const wasUploadingRef = useRef(false); + useEffect(() => { + if (mode !== "unstructured") return; + const isUploading = unstructuredFiles.some((f) => f.status === "uploading"); + if (isUploading) { + wasUploadingRef.current = true; + } else if (wasUploadingRef.current) { + wasUploadingRef.current = false; + const hasOk = unstructuredFiles.some((f) => f.status === "ok"); + if (hasOk) { + void loadSeedMetadata({ silent: true }); + } + } + }, [mode, unstructuredFiles, loadSeedMetadata]); + const previewColumns = useMemo(() => { const loadedColumns = config.seed_columns ?? []; if (loadedColumns.length > 0) return loadedColumns; @@ -434,10 +489,10 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl -
+
{mode === "hf" && ( <> -
+
-
+
+

- Upload-only. Max 50MB. + Max 50MB per file.

{(localFile?.name || config.local_file_name?.trim()) && (

@@ -537,49 +592,12 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl )} {mode === "unstructured" && ( -

- -
- { - const file = event.target.files?.[0] ?? null; - setUnstructuredFile(file); - onUpdate({ - hf_path: "", - seed_columns: [], - seed_drop_columns: [], - seed_preview_rows: [], - unstructured_file_name: file?.name ?? "", - }); - }} - /> - -
-

- File is converted to text, then chunked server-side into chunk_text rows. Max 50MB. -

- {(unstructuredFile?.name || - config.unstructured_file_name?.trim()) && ( -

- Selected:{" "} - {unstructuredFile?.name ?? config.unstructured_file_name?.trim()} -

- )} -
+ )} {inspectError &&

{inspectError}

} @@ -633,7 +651,7 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl /> -
+
-
+
-
+
-
+
-
+
onUpdate({ selection_start: event.target.value })} />
-
+
-
+
onUpdate({ selection_index: event.target.value })} />
-
+
FileEntry[])) => void; + disabled?: boolean; +}; + +function formatSize(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +function isValidExtension(name: string): boolean { + const ext = name.slice(name.lastIndexOf(".")).toLowerCase(); + return ACCEPTED_EXTENSIONS.includes(ext); +} + +export function UnstructuredDropZone({ + blockId, + files, + onFilesChange, + disabled, +}: UnstructuredDropZoneProps) { + const inputRef = useRef(null); + const filesRef = useRef(files); + filesRef.current = files; + const [isDragOver, setIsDragOver] = useState(false); + + const totalSize = files.reduce((sum, f) => sum + f.size, 0); + + const handleFiles = useCallback( + async (newFiles: File[]) => { + const valid = newFiles.filter((f) => { + if (!isValidExtension(f.name)) return false; + if (f.size > MAX_FILE_SIZE) return false; + return true; + }); + + if (valid.length === 0) return; + + const addedSize = valid.reduce((s, f) => s + f.size, 0); + const currentTotal = filesRef.current.reduce((sum, f) => sum + f.size, 0); + if (currentTotal + addedSize > MAX_TOTAL_SIZE) return; + + const entries: FileEntry[] = valid.map((f) => ({ + id: "", + name: f.name, + size: f.size, + status: "uploading" as const, + abortController: new AbortController(), + })); + + onFilesChange((prev) => [...prev, ...entries]); + + for (let i = 0; i < valid.length; i++) { + const file = valid[i]; + const entry = entries[i]; + let updatedId = ""; + let updatedStatus: FileEntry["status"] = "error"; + let updatedError: string | undefined; + try { + const existingIds = filesRef.current.filter((f) => f.id).map((f) => f.id); + const result = await uploadUnstructuredFile( + file, + blockId, + entry.abortController?.signal, + existingIds, + ); + updatedId = result.file_id; + updatedStatus = result.status === "ok" ? "ok" : "error"; + updatedError = result.error; + } catch (e) { + if (e instanceof DOMException && e.name === "AbortError") { + updatedError = "Cancelled"; + } else { + updatedError = e instanceof Error ? e.message : "Upload failed"; + } + } + onFilesChange((prev) => + prev.map((f) => + f === entry + ? { ...f, id: updatedId, status: updatedStatus, error: updatedError } + : f, + ), + ); + } + + }, + [blockId, onFilesChange], + ); + + const deletedIdsRef = useRef(new Set()); + const handleRemove = useCallback( + (index: number) => { + const entry = filesRef.current[index]; + if (!entry) return; + if (entry.status === "uploading" && entry.abortController) { + entry.abortController.abort(); + } + if (entry.id && entry.status === "ok" && !deletedIdsRef.current.has(entry.id)) { + deletedIdsRef.current.add(entry.id); + void removeUnstructuredFile(blockId, entry.id).catch(() => {}); + } + onFilesChange((prev) => prev.filter((_, i) => i !== index)); + }, + [blockId, onFilesChange], + ); + + const handleDrop = useCallback( + (e: React.DragEvent) => { + e.preventDefault(); + setIsDragOver(false); + if (disabled) return; + const dropped = Array.from(e.dataTransfer.files); + handleFiles(dropped); + }, + [disabled, handleFiles], + ); + + const handleDragOver = useCallback( + (e: React.DragEvent) => { + e.preventDefault(); + if (!disabled) setIsDragOver(true); + }, + [disabled], + ); + + const handleDragLeave = useCallback(() => setIsDragOver(false), []); + + const handleClick = useCallback(() => { + if (!disabled) inputRef.current?.click(); + }, [disabled]); + + const handleInputChange = useCallback( + (e: React.ChangeEvent) => { + const selected = Array.from(e.target.files || []); + handleFiles(selected); + e.target.value = ""; + }, + [handleFiles], + ); + + const successFiles = files.filter((f) => f.status === "ok"); + + return ( +
+
+ +

+ Drop files here or click to browse +

+

+ PDF, DOCX, TXT, MD - up to 50MB each, 100MB total +

+
+ + + + {files.length > 0 && ( +
+ {files.map((entry, i) => ( +
+ {entry.status === "uploading" && ( + + )} + {entry.status === "ok" && ( + + )} + {entry.status === "error" && ( + + )} + {entry.name} + + {formatSize(entry.size)} + + {entry.error && ( + {entry.error} + )} + +
+ ))} +
+ {successFiles.length} file{successFiles.length !== 1 ? "s" : ""} uploaded + {formatSize(totalSize)} / 100MB +
+
+ )} +
+ ); +} + +export type { FileEntry }; diff --git a/studio/frontend/src/features/recipe-studio/dialogs/shared/field-label.tsx b/studio/frontend/src/features/recipe-studio/dialogs/shared/field-label.tsx index 100e9683a0..21ff4d73e7 100644 --- a/studio/frontend/src/features/recipe-studio/dialogs/shared/field-label.tsx +++ b/studio/frontend/src/features/recipe-studio/dialogs/shared/field-label.tsx @@ -18,7 +18,7 @@ export function FieldLabel({ hint, }: FieldLabelProps): ReactElement { return ( -
+
{htmlFor ? (