From a49638c504cf7161696d84927adf4d8d5b48cfbd Mon Sep 17 00:00:00 2001 From: Manan17 Date: Mon, 9 Mar 2026 05:50:18 +0000 Subject: [PATCH 1/9] dataset upload --- studio/backend/core/training/trainer.py | 61 ++++--- studio/backend/models/datasets.py | 12 ++ studio/backend/routes/datasets.py | 55 +++++++ .../studio/sections/dataset-section.tsx | 149 +++++++++++++----- .../src/features/training/api/datasets-api.ts | 27 ++++ .../frontend/src/features/training/index.ts | 1 + .../training/stores/training-config-store.ts | 3 + .../src/features/training/types/datasets.ts | 5 + 8 files changed, 245 insertions(+), 68 deletions(-) diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py index 7bfb8d4d7a..41d208d9f4 100644 --- a/studio/backend/core/training/trainer.py +++ b/studio/backend/core/training/trainer.py @@ -378,6 +378,9 @@ class UnslothTrainer: self.is_audio = self._audio_type is not None self.is_audio_vlm = False + if not self.is_audio and not self.is_audio_vlm: + self._cuda_audio_used = False + # VLM: vision model with image dataset (mutually exclusive with audio paths) vision = is_vision_model(model_name) if not self.is_audio else False self.is_vlm = not self.is_audio_vlm and vision and is_dataset_image @@ -1786,18 +1789,20 @@ class UnslothTrainer: eval_enabled = eval_steps is not None and eval_steps > 0 if local_datasets: - # Load local datasets - all_data = [] + # Load local datasets using load_dataset() so the result is + # Arrow-backed (has cache files). Dataset.from_list() creates + # an in-memory dataset with no cache, which forces num_proc=1 + # during tokenization/map because sharding requires Arrow files. + all_files: list[str] = [] for dataset_file in local_datasets: # dataset_file may already be an absolute path from routes/training.py if os.path.isabs(dataset_file): file_path = dataset_file else: # Fallback: try relative to assets/datasets - file_path = _ASSETS_DATASETS_ROOT / dataset_file + file_path = str(_ASSETS_DATASETS_ROOT / dataset_file) file_path_obj = Path(file_path) - file_path_str = str(file_path_obj) if file_path_obj.is_dir(): parquet_dir = ( @@ -1807,36 +1812,41 @@ class UnslothTrainer: ) parquet_files = sorted(parquet_dir.glob("*.parquet")) if parquet_files: - for parquet_file in parquet_files: - df = pd.read_parquet(parquet_file) - all_data.extend(df.to_dict("records")) + all_files.extend(str(p) for p in parquet_files) continue + # Fall through to single-file detection for dirs with json/csv + candidates: list[Path] = [] + for ext in ('.json', '.jsonl', '.csv', '.parquet'): + candidates.extend(sorted(file_path_obj.glob(f"*{ext}"))) + if candidates: + all_files.append(str(candidates[0])) + continue + raise ValueError(f"No supported data files in directory: {file_path_obj}") + else: + all_files.append(str(file_path_obj)) - if file_path_str.endswith('.json'): - with open(file_path_obj, 'r', encoding='utf-8') as f: - data = json.load(f) - if isinstance(data, list): - all_data.extend(data) - else: - all_data.append(data) - elif file_path_str.endswith('.csv'): - df = pd.read_csv(file_path_obj) - all_data.extend(df.to_dict('records')) - elif file_path_str.endswith('.parquet'): - df = pd.read_parquet(file_path_obj) - all_data.extend(df.to_dict('records')) - continue + if all_files: + # Determine loader type from the first file extension + first_ext = Path(all_files[0]).suffix.lower() + if first_ext in ('.json', '.jsonl'): + loader = 'json' + elif first_ext == '.csv': + loader = 'csv' + elif first_ext == '.parquet': + loader = 'parquet' + else: + raise ValueError(f"Unsupported local dataset format: {all_files[0]}") - if all_data: - dataset = Dataset.from_list(all_data) + dataset = load_dataset(loader, data_files=all_files, split='train') # Check if stopped during dataset loading if self.should_stop: print("Stopped during dataset loading\n") return None - self._update_progress(status_message=f"Loaded {len(all_data)} samples from local files") - print(f"Loaded {len(all_data)} samples from local files\n") + self._update_progress(status_message=f"Loaded {len(dataset)} samples from local files") + print(f"Loaded {len(dataset)} samples from local files\n") + print(f"[DEBUG] Dataset cache_files: {dataset.cache_files}\n") elif dataset_source: # Load from Hugging Face @@ -2360,6 +2370,7 @@ class UnslothTrainer: "dataset_num_proc": 1 if (self.is_audio or self.is_audio_vlm or self._cuda_audio_used) else safe_num_proc(max(1, os.cpu_count() // 4)), "max_seq_length": training_args.get('max_seq_length', 2048), } + print(f"[DEBUG] dataset_num_proc={config_args['dataset_num_proc']} (is_audio={self.is_audio}, is_audio_vlm={self.is_audio_vlm}, _cuda_audio_used={self._cuda_audio_used})") # On Windows with transformers 5.x, disable DataLoader multiprocessing # to avoid issues with modified sys.path (.venv_t5) in spawned workers. diff --git a/studio/backend/models/datasets.py b/studio/backend/models/datasets.py index 19197d99ad..ca126948bb 100644 --- a/studio/backend/models/datasets.py +++ b/studio/backend/models/datasets.py @@ -41,6 +41,18 @@ class CheckFormatResponse(BaseModel): warning: Optional[str] = None +class UploadDatasetRequest(BaseModel): + """Request for uploading a local training dataset file.""" + filename: str = Field(..., description="Original filename, e.g. my_data.jsonl") + content_base64: str = Field(..., description="Base64-encoded file bytes") + + +class UploadDatasetResponse(BaseModel): + """Response with stored dataset path for training.""" + filename: str = Field(..., description="Original filename") + stored_path: str = Field(..., description="Absolute path stored on backend") + + class LocalDatasetItem(BaseModel): class Metadata(BaseModel): actual_num_records: Optional[int] = None diff --git a/studio/backend/routes/datasets.py b/studio/backend/routes/datasets.py index 04345a944d..e01375d310 100644 --- a/studio/backend/routes/datasets.py +++ b/studio/backend/routes/datasets.py @@ -2,10 +2,12 @@ Datasets API routes """ import base64 +import binascii import io import json import sys from pathlib import Path +from uuid import uuid4 from fastapi import APIRouter, Depends, HTTPException import logging @@ -36,6 +38,8 @@ from models.datasets import ( CheckFormatResponse, LocalDatasetItem, LocalDatasetsResponse, + UploadDatasetRequest, + UploadDatasetResponse, ) @@ -89,6 +93,10 @@ DATA_EXTS = ( '.zip', ) LOCAL_FILE_EXTS = ('.json', '.jsonl', '.csv', '.parquet') +LOCAL_UPLOAD_EXTS = {".csv", ".json", ".jsonl", ".parquet"} +DATASET_UPLOAD_DIR = ( + Path.home() / ".cache" / "unsloth" / "training" / "dataset-uploads" +) BACKEND_ROOT = Path(__file__).resolve().parents[1] LOCAL_DATASETS_ROOT = BACKEND_ROOT / "assets" / "datasets" @@ -252,6 +260,53 @@ def _load_local_preview_slice(*, dataset_path: Path, train_split: str, preview_s return preview_slice, total_rows +def _sanitize_filename(filename: str) -> str: + name = Path(filename).name.strip().replace("\x00", "") + if not name: + return "dataset_upload" + return name + + +def _decode_base64_payload(content_base64: str) -> bytes: + raw = content_base64.strip() + if "," in raw and raw.lower().startswith("data:"): + raw = raw.split(",", 1)[1] + try: + return base64.b64decode(raw, validate=True) + except binascii.Error as exc: + raise HTTPException(status_code=400, detail="Invalid base64 payload") from exc + + +@router.post("/upload", response_model=UploadDatasetResponse) +def upload_dataset( + payload: UploadDatasetRequest, + current_subject: str = Depends(get_current_subject), +) -> UploadDatasetResponse: + filename = _sanitize_filename(payload.filename) + ext = Path(filename).suffix.lower() + if ext not in LOCAL_UPLOAD_EXTS: + allowed = ", ".join(sorted(LOCAL_UPLOAD_EXTS)) + raise HTTPException( + status_code=400, + detail=f"Unsupported file type: {ext}. Allowed: {allowed}", + ) + + file_bytes = _decode_base64_payload(payload.content_base64) + if not file_bytes: + raise HTTPException(status_code=400, detail="Empty upload payload") + + max_size_bytes = 512 * 1024 * 1024 + if len(file_bytes) > max_size_bytes: + raise HTTPException(status_code=413, detail="File too large (max 512MB)") + + DATASET_UPLOAD_DIR.mkdir(parents=True, exist_ok=True) + stored_name = f"{uuid4().hex}_{filename}" + stored_path = DATASET_UPLOAD_DIR / stored_name + stored_path.write_bytes(file_bytes) + + return UploadDatasetResponse(filename=filename, stored_path=str(stored_path)) + + @router.get("/local", response_model=LocalDatasetsResponse) def list_local_datasets( current_subject: str = Depends(get_current_subject), diff --git a/studio/frontend/src/features/studio/sections/dataset-section.tsx b/studio/frontend/src/features/studio/sections/dataset-section.tsx index e522aaca9b..154999ac5b 100644 --- a/studio/frontend/src/features/studio/sections/dataset-section.tsx +++ b/studio/frontend/src/features/studio/sections/dataset-section.tsx @@ -37,6 +37,7 @@ import { } from "@/hooks"; import { HfDatasetSubsetSplitSelectors, + uploadTrainingDataset, useDatasetPreviewDialogStore, useTrainingConfigStore, } from "@/features/training"; @@ -52,7 +53,8 @@ import { ViewIcon, } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; -import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { type ChangeEvent, useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { toast } from "sonner"; import { useShallow } from "zustand/react/shallow"; const SEARCH_INPUT_REASONS = new Set(["input-change", "input-paste", "input-clear"]); @@ -72,7 +74,11 @@ function deriveLocalDatasetName(path: string): string { const parts = normalized.split("/").filter(Boolean); const parquetIndex = parts.lastIndexOf("parquet-files"); if (parquetIndex > 0) return parts[parquetIndex - 1]; - return parts[parts.length - 1] ?? path; + const basename = parts[parts.length - 1] ?? path; + // Strip UUID prefix from uploaded files (format: {32hex}_{original}) + const uuidPrefixMatch = basename.match(/^[a-f0-9]{32}_(.+)$/); + if (uuidPrefixMatch) return uuidPrefixMatch[1]; + return basename; } function formatUpdatedDate(timestamp: number | null): string { @@ -286,6 +292,8 @@ export function DatasetSection() { if (datasetSource !== "upload") return; if (!uploadedFile) return; if (selectedLocalDataset) return; + // Don't clear if this is a direct file upload (not a recipe directory) + if (isLikelyLocalDatasetRef(uploadedFile)) return; selectLocalDataset(null); }, [ datasetSource, @@ -320,11 +328,57 @@ export function DatasetSection() { const selectedLocalUpdatedAt = selectedLocalDataset?.updated_at ?? null; const comboboxAnchorRef = useRef(null); + const fileInputRef = useRef(null); const { scrollRef, sentinelRef } = useInfiniteScroll( fetchMore, hfResults.length, ); + const fileToBase64Payload = (file: File): Promise => + new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + const value = String(reader.result ?? ""); + const parts = value.split(","); + resolve(parts.length > 1 ? parts[1] : value); + }; + reader.onerror = () => reject(new Error("Failed to read file")); + reader.readAsDataURL(file); + }); + + const [isUploading, setIsUploading] = useState(false); + + const handleUploadButtonClick = () => { + fileInputRef.current?.click(); + }; + + const handleDatasetFileChange = async (event: ChangeEvent) => { + const file = event.target.files?.[0]; + event.target.value = ""; + if (!file) return; + + setIsUploading(true); + try { + const contentBase64 = await fileToBase64Payload(file); + const uploaded = await uploadTrainingDataset({ + filename: file.name, + contentBase64, + }); + + selectLocalDataset(uploaded.stored_path); + + toast.success("Dataset uploaded", { + description: uploaded.filename, + }); + } catch (error) { + toast.error("Upload failed", { + description: error instanceof Error ? error.message : "Unknown error", + }); + } finally { + setIsUploading(false); + } + }; + return (
- ) : datasetSource === "upload" ? ( + ) : datasetSource === "upload" && selectedLocalDataset ? (
@@ -609,45 +663,39 @@ export function DatasetSection() {
- {uploadedFile ? ( -
-
- - 0 - ? String(selectedLocalColumns.length) - : "--" - } - /> - - -
+
+
+ + 0 + ? String(selectedLocalColumns.length) + : "--" + } + /> + +
- ) : ( -

- Select a local dataset to view metadata. -

- )} +
) : null} @@ -839,9 +887,15 @@ export function DatasetSection() { variant="outline" size="sm" className="cursor-pointer gap-1.5" + disabled={isUploading} + onClick={handleUploadButtonClick} > - - Upload + {isUploading ? ( + + ) : ( + + )} + {isUploading ? "Uploading..." : "Upload"}
+ { + void handleDatasetFileChange(event); + }} + /> diff --git a/studio/frontend/src/features/training/api/datasets-api.ts b/studio/frontend/src/features/training/api/datasets-api.ts index d056d92189..0f37ac53bf 100644 --- a/studio/frontend/src/features/training/api/datasets-api.ts +++ b/studio/frontend/src/features/training/api/datasets-api.ts @@ -1,6 +1,7 @@ import type { CheckFormatResponse, LocalDatasetsResponse, + UploadDatasetResponse, } from "../types/datasets"; import { authFetch } from "@/features/auth"; @@ -39,6 +40,32 @@ export async function checkDatasetFormat({ return res.json(); } +type UploadDatasetArgs = { + filename: string; + contentBase64: string; +}; + +export async function uploadTrainingDataset({ + filename, + contentBase64, +}: UploadDatasetArgs): Promise { + const res = await authFetch("/api/datasets/upload", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + filename, + content_base64: contentBase64, + }), + }); + + if (!res.ok) { + const body = await res.json().catch(() => null); + throw new Error(body?.detail || `Upload failed (${res.status})`); + } + + return res.json(); +} + export async function listLocalDatasets(): Promise { const res = await authFetch("/api/datasets/local"); if (!res.ok) { diff --git a/studio/frontend/src/features/training/index.ts b/studio/frontend/src/features/training/index.ts index d2fd4f2e43..9302ffe7dd 100644 --- a/studio/frontend/src/features/training/index.ts +++ b/studio/frontend/src/features/training/index.ts @@ -7,6 +7,7 @@ export { useTrainingActions } from "./hooks/use-training-actions"; export { useTrainingRuntimeLifecycle } from "./hooks/use-training-runtime-lifecycle"; export { HfDatasetSubsetSplitSelectors } from "./components/hf-dataset-subset-split-selectors"; export { useDatasetPreviewDialogStore } from "./stores/dataset-preview-dialog-store"; +export { uploadTrainingDataset } from "./api/datasets-api"; export { listLocalModels } from "./api/models-api"; export type { LocalModelInfo } from "./api/models-api"; export type { TrainingPhase } from "./types/runtime"; diff --git a/studio/frontend/src/features/training/stores/training-config-store.ts b/studio/frontend/src/features/training/stores/training-config-store.ts index 42f6b2a33b..3318a192d8 100644 --- a/studio/frontend/src/features/training/stores/training-config-store.ts +++ b/studio/frontend/src/features/training/stores/training-config-store.ts @@ -285,6 +285,9 @@ export const useTrainingConfigStore = create()( uploadedFile, ...resetDatasetState(), }); + if (uploadedFile) { + runDatasetCheck(uploadedFile, "train"); + } }, setDatasetFormat: (datasetFormat) => set({ datasetFormat }), setDataset: (dataset) => { diff --git a/studio/frontend/src/features/training/types/datasets.ts b/studio/frontend/src/features/training/types/datasets.ts index cee77f683b..41a71affaf 100644 --- a/studio/frontend/src/features/training/types/datasets.ts +++ b/studio/frontend/src/features/training/types/datasets.ts @@ -15,6 +15,11 @@ export type CheckFormatResponse = { warning?: string | null; }; +export type UploadDatasetResponse = { + filename: string; + stored_path: string; +}; + export type LocalDatasetInfo = { metadata?: { actual_num_records?: number | null; From a08b73e38573edf9166bc493263ef4d33d816dbf Mon Sep 17 00:00:00 2001 From: Manan17 Date: Mon, 9 Mar 2026 07:04:02 +0000 Subject: [PATCH 2/9] remove file size limit --- studio/backend/routes/datasets.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/studio/backend/routes/datasets.py b/studio/backend/routes/datasets.py index e01375d310..2357cd2c0e 100644 --- a/studio/backend/routes/datasets.py +++ b/studio/backend/routes/datasets.py @@ -295,10 +295,6 @@ def upload_dataset( if not file_bytes: raise HTTPException(status_code=400, detail="Empty upload payload") - max_size_bytes = 512 * 1024 * 1024 - if len(file_bytes) > max_size_bytes: - raise HTTPException(status_code=413, detail="File too large (max 512MB)") - DATASET_UPLOAD_DIR.mkdir(parents=True, exist_ok=True) stored_name = f"{uuid4().hex}_{filename}" stored_path = DATASET_UPLOAD_DIR / stored_name From 4c5ded4c52b89b8a146e157f068e66a3f3325e9f Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Mon, 9 Mar 2026 13:35:55 +0000 Subject: [PATCH 3/9] normalize uploaded filename extension to lowercase for consistent downstream checks --- studio/backend/routes/datasets.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/studio/backend/routes/datasets.py b/studio/backend/routes/datasets.py index 2357cd2c0e..2faa7121ac 100644 --- a/studio/backend/routes/datasets.py +++ b/studio/backend/routes/datasets.py @@ -296,7 +296,9 @@ def upload_dataset( raise HTTPException(status_code=400, detail="Empty upload payload") DATASET_UPLOAD_DIR.mkdir(parents=True, exist_ok=True) - stored_name = f"{uuid4().hex}_{filename}" + # Normalize extension to lowercase so downstream suffix checks work + stem = Path(filename).stem + stored_name = f"{uuid4().hex}_{stem}{ext}" stored_path = DATASET_UPLOAD_DIR / stored_name stored_path.write_bytes(file_bytes) From c998227fec6d907e6f175aefd662ce1f1ff440da Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Mon, 9 Mar 2026 13:38:00 +0000 Subject: [PATCH 4/9] add client-side file size validation before upload --- .../src/features/studio/sections/dataset-section.tsx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/studio/frontend/src/features/studio/sections/dataset-section.tsx b/studio/frontend/src/features/studio/sections/dataset-section.tsx index 154999ac5b..b81dd058f6 100644 --- a/studio/frontend/src/features/studio/sections/dataset-section.tsx +++ b/studio/frontend/src/features/studio/sections/dataset-section.tsx @@ -357,6 +357,14 @@ export function DatasetSection() { event.target.value = ""; if (!file) return; + const MAX_SIZE_BYTES = 512 * 1024 * 1024; + if (file.size > MAX_SIZE_BYTES) { + toast.error("File too large", { + description: "Maximum upload size is 512 MB.", + }); + return; + } + setIsUploading(true); try { const contentBase64 = await fileToBase64Payload(file); From 56412f23625637354e265804fe6ad2d79980271f Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Mon, 9 Mar 2026 13:52:45 +0000 Subject: [PATCH 5/9] include all candidate files when scanning a directory, not just the first --- studio/backend/core/training/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py index 41d208d9f4..5e20dfb43f 100644 --- a/studio/backend/core/training/trainer.py +++ b/studio/backend/core/training/trainer.py @@ -1819,7 +1819,7 @@ class UnslothTrainer: for ext in ('.json', '.jsonl', '.csv', '.parquet'): candidates.extend(sorted(file_path_obj.glob(f"*{ext}"))) if candidates: - all_files.append(str(candidates[0])) + all_files.extend(str(c) for c in candidates) continue raise ValueError(f"No supported data files in directory: {file_path_obj}") else: From 1d06e2f54cbfd8fbaa407c046647fda95aa22e3a Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Mon, 9 Mar 2026 13:55:45 +0000 Subject: [PATCH 6/9] switch dataset upload from base64 JSON to multipart/form-data with streamed writes --- studio/backend/models/datasets.py | 6 --- studio/backend/routes/datasets.py | 41 +++++++++---------- .../studio/sections/dataset-section.tsx | 18 +------- .../src/features/training/api/datasets-api.ts | 19 +++------ 4 files changed, 26 insertions(+), 58 deletions(-) diff --git a/studio/backend/models/datasets.py b/studio/backend/models/datasets.py index ca126948bb..49d8b4c419 100644 --- a/studio/backend/models/datasets.py +++ b/studio/backend/models/datasets.py @@ -41,12 +41,6 @@ class CheckFormatResponse(BaseModel): warning: Optional[str] = None -class UploadDatasetRequest(BaseModel): - """Request for uploading a local training dataset file.""" - filename: str = Field(..., description="Original filename, e.g. my_data.jsonl") - content_base64: str = Field(..., description="Base64-encoded file bytes") - - class UploadDatasetResponse(BaseModel): """Response with stored dataset path for training.""" filename: str = Field(..., description="Original filename") diff --git a/studio/backend/routes/datasets.py b/studio/backend/routes/datasets.py index 2faa7121ac..302ac0aa9d 100644 --- a/studio/backend/routes/datasets.py +++ b/studio/backend/routes/datasets.py @@ -2,13 +2,12 @@ Datasets API routes """ import base64 -import binascii import io import json import sys from pathlib import Path from uuid import uuid4 -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, UploadFile import logging # Add backend directory to path @@ -38,7 +37,6 @@ from models.datasets import ( CheckFormatResponse, LocalDatasetItem, LocalDatasetsResponse, - UploadDatasetRequest, UploadDatasetResponse, ) @@ -267,22 +265,12 @@ def _sanitize_filename(filename: str) -> str: return name -def _decode_base64_payload(content_base64: str) -> bytes: - raw = content_base64.strip() - if "," in raw and raw.lower().startswith("data:"): - raw = raw.split(",", 1)[1] - try: - return base64.b64decode(raw, validate=True) - except binascii.Error as exc: - raise HTTPException(status_code=400, detail="Invalid base64 payload") from exc - - @router.post("/upload", response_model=UploadDatasetResponse) -def upload_dataset( - payload: UploadDatasetRequest, +async def upload_dataset( + file: UploadFile, current_subject: str = Depends(get_current_subject), ) -> UploadDatasetResponse: - filename = _sanitize_filename(payload.filename) + filename = _sanitize_filename(file.filename or "dataset_upload") ext = Path(filename).suffix.lower() if ext not in LOCAL_UPLOAD_EXTS: allowed = ", ".join(sorted(LOCAL_UPLOAD_EXTS)) @@ -291,16 +279,25 @@ def upload_dataset( detail=f"Unsupported file type: {ext}. Allowed: {allowed}", ) - file_bytes = _decode_base64_payload(payload.content_base64) - if not file_bytes: - raise HTTPException(status_code=400, detail="Empty upload payload") - + max_size_bytes = 512 * 1024 * 1024 DATASET_UPLOAD_DIR.mkdir(parents=True, exist_ok=True) - # Normalize extension to lowercase so downstream suffix checks work stem = Path(filename).stem stored_name = f"{uuid4().hex}_{stem}{ext}" stored_path = DATASET_UPLOAD_DIR / stored_name - stored_path.write_bytes(file_bytes) + + # Stream file to disk in chunks to avoid holding entire file in memory + size = 0 + with open(stored_path, "wb") as f: + while chunk := await file.read(1024 * 1024): + size += len(chunk) + if size > max_size_bytes: + stored_path.unlink(missing_ok=True) + raise HTTPException(status_code=413, detail="File too large (max 512MB)") + f.write(chunk) + + if size == 0: + stored_path.unlink(missing_ok=True) + raise HTTPException(status_code=400, detail="Empty upload payload") return UploadDatasetResponse(filename=filename, stored_path=str(stored_path)) diff --git a/studio/frontend/src/features/studio/sections/dataset-section.tsx b/studio/frontend/src/features/studio/sections/dataset-section.tsx index b81dd058f6..a72c21100a 100644 --- a/studio/frontend/src/features/studio/sections/dataset-section.tsx +++ b/studio/frontend/src/features/studio/sections/dataset-section.tsx @@ -334,18 +334,6 @@ export function DatasetSection() { hfResults.length, ); - const fileToBase64Payload = (file: File): Promise => - new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onload = () => { - const value = String(reader.result ?? ""); - const parts = value.split(","); - resolve(parts.length > 1 ? parts[1] : value); - }; - reader.onerror = () => reject(new Error("Failed to read file")); - reader.readAsDataURL(file); - }); - const [isUploading, setIsUploading] = useState(false); const handleUploadButtonClick = () => { @@ -367,11 +355,7 @@ export function DatasetSection() { setIsUploading(true); try { - const contentBase64 = await fileToBase64Payload(file); - const uploaded = await uploadTrainingDataset({ - filename: file.name, - contentBase64, - }); + const uploaded = await uploadTrainingDataset(file); selectLocalDataset(uploaded.stored_path); diff --git a/studio/frontend/src/features/training/api/datasets-api.ts b/studio/frontend/src/features/training/api/datasets-api.ts index 0f37ac53bf..f3098acc80 100644 --- a/studio/frontend/src/features/training/api/datasets-api.ts +++ b/studio/frontend/src/features/training/api/datasets-api.ts @@ -40,22 +40,15 @@ export async function checkDatasetFormat({ return res.json(); } -type UploadDatasetArgs = { - filename: string; - contentBase64: string; -}; +export async function uploadTrainingDataset( + file: File, +): Promise { + const form = new FormData(); + form.append("file", file); -export async function uploadTrainingDataset({ - filename, - contentBase64, -}: UploadDatasetArgs): Promise { const res = await authFetch("/api/datasets/upload", { method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - filename, - content_base64: contentBase64, - }), + body: form, }); if (!res.ok) { From fbcd111a701ebeabbbec89b51d2bac4ded4066a2 Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Mon, 9 Mar 2026 14:01:02 +0000 Subject: [PATCH 7/9] narrow stale selection guard to only skip clearing for uploaded files --- .../frontend/src/features/studio/sections/dataset-section.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/studio/frontend/src/features/studio/sections/dataset-section.tsx b/studio/frontend/src/features/studio/sections/dataset-section.tsx index a72c21100a..576b45610b 100644 --- a/studio/frontend/src/features/studio/sections/dataset-section.tsx +++ b/studio/frontend/src/features/studio/sections/dataset-section.tsx @@ -293,7 +293,7 @@ export function DatasetSection() { if (!uploadedFile) return; if (selectedLocalDataset) return; // Don't clear if this is a direct file upload (not a recipe directory) - if (isLikelyLocalDatasetRef(uploadedFile)) return; + if (uploadedFile.includes("/dataset-uploads/")) return; selectLocalDataset(null); }, [ datasetSource, From ae89101e810124afcd560115be5b9945f81c4c2f Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Mon, 9 Mar 2026 16:51:30 +0000 Subject: [PATCH 8/9] Revert "narrow stale selection guard to only skip clearing for uploaded files" This reverts commit fbcd111a701ebeabbbec89b51d2bac4ded4066a2. --- .../frontend/src/features/studio/sections/dataset-section.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/studio/frontend/src/features/studio/sections/dataset-section.tsx b/studio/frontend/src/features/studio/sections/dataset-section.tsx index 576b45610b..a72c21100a 100644 --- a/studio/frontend/src/features/studio/sections/dataset-section.tsx +++ b/studio/frontend/src/features/studio/sections/dataset-section.tsx @@ -293,7 +293,7 @@ export function DatasetSection() { if (!uploadedFile) return; if (selectedLocalDataset) return; // Don't clear if this is a direct file upload (not a recipe directory) - if (uploadedFile.includes("/dataset-uploads/")) return; + if (isLikelyLocalDatasetRef(uploadedFile)) return; selectLocalDataset(null); }, [ datasetSource, From 022bafaf92f3f199f2d9b8c009ca7071f0e2e5dc Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Mon, 9 Mar 2026 17:06:36 +0000 Subject: [PATCH 9/9] store uploaded datasets under assets/datasets/uploads instead of ~/.cache --- studio/backend/routes/datasets.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/studio/backend/routes/datasets.py b/studio/backend/routes/datasets.py index 302ac0aa9d..b06fa33942 100644 --- a/studio/backend/routes/datasets.py +++ b/studio/backend/routes/datasets.py @@ -92,11 +92,9 @@ DATA_EXTS = ( ) LOCAL_FILE_EXTS = ('.json', '.jsonl', '.csv', '.parquet') LOCAL_UPLOAD_EXTS = {".csv", ".json", ".jsonl", ".parquet"} -DATASET_UPLOAD_DIR = ( - Path.home() / ".cache" / "unsloth" / "training" / "dataset-uploads" -) BACKEND_ROOT = Path(__file__).resolve().parents[1] LOCAL_DATASETS_ROOT = BACKEND_ROOT / "assets" / "datasets" +DATASET_UPLOAD_DIR = LOCAL_DATASETS_ROOT / "uploads" def _safe_read_metadata(path: Path) -> dict | None: