diff --git a/studio/backend/auth/storage.py b/studio/backend/auth/storage.py index f72a7ed943..cb5df6e3f9 100644 --- a/studio/backend/auth/storage.py +++ b/studio/backend/auth/storage.py @@ -7,10 +7,11 @@ SQLite storage for authentication data (user credentials + JWT secret). import hashlib import sqlite3 from datetime import datetime, timezone -from pathlib import Path from typing import Optional, Tuple -DB_PATH = Path(__file__).parent / "auth.db" +from utils.paths import auth_db_path, ensure_dir + +DB_PATH = auth_db_path() def _hash_token(token: str) -> str: @@ -20,6 +21,7 @@ def _hash_token(token: str) -> str: def get_connection() -> sqlite3.Connection: """Get a connection to the auth database, creating tables if needed.""" + ensure_dir(DB_PATH.parent) conn = sqlite3.connect(DB_PATH) conn.row_factory = sqlite3.Row conn.execute( @@ -256,4 +258,3 @@ def revoke_user_refresh_tokens(username: str) -> None: conn.commit() finally: conn.close() - diff --git a/studio/backend/core/data_recipe/jobs/worker.py b/studio/backend/core/data_recipe/jobs/worker.py index fbe9d91b6d..1d898f866f 100644 --- a/studio/backend/core/data_recipe/jobs/worker.py +++ b/studio/backend/core/data_recipe/jobs/worker.py @@ -16,9 +16,9 @@ from typing import Any from ..jsonable import to_jsonable, to_preview_jsonable from .constants import EVENT_JOB_COMPLETED, EVENT_JOB_ERROR, EVENT_JOB_STARTED from ..service import build_config_builder, create_data_designer +from utils.paths import ensure_dir, recipe_datasets_root -_PROJECT_ROOT = Path(__file__).resolve().parents[5] -_ARTIFACT_ROOT = _PROJECT_ROOT / "studio" / "backend" / "assets" / "datasets" +_ARTIFACT_ROOT = recipe_datasets_root() class _QueueLogHandler(logging.Handler): @@ -88,7 +88,7 @@ def run_job_process( artifact_root=_ARTIFACT_ROOT, ) merge_batches = bool(run.get("merge_batches")) - _ARTIFACT_ROOT.mkdir(parents=True, exist_ok=True) + ensure_dir(_ARTIFACT_ROOT) run_config_raw = run.get("run_config") or {} builder = build_config_builder(recipe) diff --git a/studio/backend/core/data_recipe/local_callable_validators.py b/studio/backend/core/data_recipe/local_callable_validators.py index 5e2749d45d..a7e211eca5 100644 --- a/studio/backend/core/data_recipe/local_callable_validators.py +++ b/studio/backend/core/data_recipe/local_callable_validators.py @@ -5,6 +5,7 @@ from __future__ import annotations import json import logging +import os import subprocess from copy import deepcopy from dataclasses import dataclass @@ -12,6 +13,8 @@ from functools import lru_cache from pathlib import Path from typing import Any +from utils.paths import ensure_dir, oxc_validator_tmp_root + logger = logging.getLogger(__name__) OXC_VALIDATION_FN_MARKER = "unsloth_oxc_validator" @@ -235,6 +238,12 @@ def _run_oxc_batch( "codes": code_values, } try: + tmp_dir = ensure_dir(oxc_validator_tmp_root()) + env = dict(os.environ) + tmp_dir_str = str(tmp_dir) + env["TMPDIR"] = tmp_dir_str + env["TMP"] = tmp_dir_str + env["TEMP"] = tmp_dir_str proc = subprocess.run( ["node", str(_OXC_RUNNER_PATH)], cwd=str(_OXC_TOOL_DIR), @@ -242,6 +251,7 @@ def _run_oxc_batch( text=True, capture_output=True, check=False, + env=env, ) except (OSError, ValueError) as exc: logger.warning("OXC subprocess launch failed: %s", exc) diff --git a/studio/backend/core/export/export.py b/studio/backend/core/export/export.py index 986da0def2..ef19530a2c 100644 --- a/studio/backend/core/export/export.py +++ b/studio/backend/core/export/export.py @@ -21,6 +21,7 @@ from utils.hardware import clear_gpu_cache from utils.models import is_vision_model, get_base_model_from_lora from utils.models.model_config import detect_audio_type +from utils.paths import ensure_dir, outputs_root, resolve_export_dir, resolve_output_dir from core.inference import get_inference_backend logger = logging.getLogger(__name__) @@ -128,7 +129,7 @@ class ExportBackend: logger.error(f"Error during memory cleanup: {e}") return False - def scan_checkpoints(self, outputs_dir: str = "./outputs") -> List[Tuple[str, List[Tuple[str, str]]]]: + def scan_checkpoints(self, outputs_dir: str = str(outputs_root())) -> List[Tuple[str, List[Tuple[str, str]]]]: """ Scan outputs folder for training runs and their checkpoints. @@ -325,8 +326,9 @@ class ExportBackend: # Save locally if requested if save_directory: + save_directory = str(resolve_export_dir(save_directory)) logger.info(f"Saving merged model locally to: {save_directory}") - os.makedirs(save_directory, exist_ok=True) + ensure_dir(Path(save_directory)) self.current_model.save_pretrained_merged( save_directory, @@ -386,8 +388,9 @@ class ExportBackend: try: # Save locally if requested if save_directory: + save_directory = str(resolve_export_dir(save_directory)) logger.info(f"Saving base model locally to: {save_directory}") - os.makedirs(save_directory, exist_ok=True) + ensure_dir(Path(save_directory)) self.current_model.save_pretrained(save_directory) self.current_tokenizer.save_pretrained(save_directory) @@ -475,6 +478,7 @@ class ExportBackend: # Save locally if requested if save_directory: + save_directory = str(resolve_export_dir(save_directory)) # Resolve to absolute path so unsloth's relative-path internals # (check_llama_cpp, use_local_gguf, _download_convert_hf_to_gguf) # all resolve against the repo root cwd, NOT the export directory. @@ -482,7 +486,7 @@ class ExportBackend: logger.info(f"Saving GGUF model locally to: {abs_save_dir}") # Create the directory if it doesn't exist - os.makedirs(abs_save_dir, exist_ok=True) + ensure_dir(Path(abs_save_dir)) # On WSL, patch out sudo check before llama.cpp build _apply_wsl_sudo_patch() @@ -582,8 +586,9 @@ class ExportBackend: try: # Save locally if requested if save_directory: + save_directory = str(resolve_export_dir(save_directory)) logger.info(f"Saving LoRA adapter locally to: {save_directory}") - os.makedirs(save_directory, exist_ok=True) + ensure_dir(Path(save_directory)) self.current_model.save_pretrained(save_directory) self.current_tokenizer.save_pretrained(save_directory) diff --git a/studio/backend/core/export/orchestrator.py b/studio/backend/core/export/orchestrator.py index d6a04a3c1b..37604c81df 100644 --- a/studio/backend/core/export/orchestrator.py +++ b/studio/backend/core/export/orchestrator.py @@ -21,6 +21,7 @@ import threading import time from pathlib import Path from typing import Any, List, Optional, Tuple +from utils.paths import outputs_root logger = logging.getLogger(__name__) @@ -378,7 +379,7 @@ class ExportOrchestrator: return success def scan_checkpoints( - self, outputs_dir: str = "./outputs" + self, outputs_dir: str = str(outputs_root()) ) -> List[Tuple[str, list]]: """Scan for checkpoints — no ML imports needed, runs locally.""" from utils.models.checkpoints import scan_checkpoints diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py index 31e05d0bda..5a03cd6653 100644 --- a/studio/backend/core/training/trainer.py +++ b/studio/backend/core/training/trainer.py @@ -30,13 +30,21 @@ from datasets import Dataset, load_dataset from utils.models import is_vision_model, detect_audio_type from utils.datasets import format_and_template_dataset from utils.datasets import MODEL_TO_TEMPLATE_MAPPER, TEMPLATE_TO_RESPONSES_MAPPER +from utils.paths import ensure_dir, resolve_dataset_path, resolve_output_dir, resolve_tensorboard_dir from trl import SFTTrainer, SFTConfig logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -_BACKEND_ROOT = Path(__file__).resolve().parents[2] -_ASSETS_DATASETS_ROOT = _BACKEND_ROOT / "assets" / "datasets" + + +def _build_report_targets(training_args) -> list[str] | str: + report_to: list[str] = [] + if training_args.get("enable_wandb", False): + report_to.append("wandb") + if training_args.get("enable_tensorboard", False): + report_to.append("tensorboard") + return report_to or "none" @dataclass @@ -205,9 +213,14 @@ class UnslothTrainer: "lr_scheduler_type": lr_scheduler_type, "seed": random_seed, "output_dir": output_dir, - "report_to": ["wandb"] if training_args.get('enable_wandb', False) else "none", + "report_to": _build_report_targets(training_args), } + if training_args.get("enable_tensorboard", False): + config["logging_dir"] = str( + resolve_tensorboard_dir(training_args.get("tensorboard_dir")) + ) + # max_steps vs epochs if max_steps_val and max_steps_val > 0: config["max_steps"] = max_steps_val @@ -1529,6 +1542,7 @@ class UnslothTrainer: import numpy as np import soundfile as sf from datasets import Dataset as HFDataset + from utils.paths import ensure_dir, tmp_root device = "cuda" if torch.cuda.is_available() else "cpu" @@ -1625,10 +1639,18 @@ class UnslothTrainer: audio_bytes = buf.getvalue() # 1. Get word timings from Whisper - with tempfile.NamedTemporaryFile(suffix=".wav", delete=True) as tmp: + with tempfile.NamedTemporaryFile( + suffix=".wav", + delete=False, + dir=str(ensure_dir(tmp_root())), + ) as tmp: tmp.write(audio_bytes) tmp.flush() - whisper_result = whisper_model.transcribe(tmp.name, word_timestamps=True) + tmp_path = tmp.name + try: + whisper_result = whisper_model.transcribe(tmp_path, word_timestamps=True) + finally: + Path(tmp_path).unlink(missing_ok=True) normalized_transcript = text_normalizations(text) words_with_timings = [] @@ -1816,7 +1838,7 @@ class UnslothTrainer: file_path = dataset_file else: # Fallback: try relative to assets/datasets - file_path = str(_ASSETS_DATASETS_ROOT / dataset_file) + file_path = str(resolve_dataset_path(dataset_file)) file_path_obj = Path(file_path) @@ -2069,7 +2091,7 @@ class UnslothTrainer: dataset: Dataset, eval_dataset: Dataset = None, eval_steps: float = 0.00, - output_dir: str = "./outputs", + output_dir: str | None = None, num_epochs: int = 3, learning_rate: float = 5e-5, batch_size: int = 2, @@ -2086,7 +2108,7 @@ class UnslothTrainer: wandb_project: str = "unsloth-training", wandb_token: str = None, enable_tensorboard: bool = False, - tensorboard_dir: str = "runs", + tensorboard_dir: str | None = None, **kwargs) -> bool: """Start training in a separate thread""" @@ -2174,8 +2196,8 @@ class UnslothTrainer: wandb.init(project=training_args.get('wandb_project', 'unsloth-training')) # Create output directory - output_dir = training_args.get('output_dir', './outputs') - os.makedirs(output_dir, exist_ok=True) + output_dir = str(resolve_output_dir(training_args.get("output_dir"))) + ensure_dir(Path(output_dir)) # ========== AUDIO TRAINER BRANCH ========== if self._audio_type == 'csm': @@ -2385,11 +2407,15 @@ class UnslothTrainer: "weight_decay": training_args.get('weight_decay', 0.01), "seed": training_args.get('random_seed', 3407), "output_dir": output_dir, - "report_to": ["wandb"] if training_args.get('enable_wandb', False) else "none", + "report_to": _build_report_targets(training_args), "include_num_input_tokens_seen": True, # Enable token counting "dataset_num_proc": 1 if (self.is_audio or self.is_audio_vlm or self._cuda_audio_used) else safe_num_proc(max(1, os.cpu_count() // 4)), "max_seq_length": training_args.get('max_seq_length', 2048), } + if training_args.get("enable_tensorboard", False): + config_args["logging_dir"] = str( + resolve_tensorboard_dir(training_args.get("tensorboard_dir")) + ) print(f"[DEBUG] dataset_num_proc={config_args['dataset_num_proc']} (is_audio={self.is_audio}, is_audio_vlm={self.is_audio_vlm}, _cuda_audio_used={self._cuda_audio_used})") # On Windows with transformers 5.x, disable DataLoader multiprocessing diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py index 7b98865eea..509f99d3ec 100644 --- a/studio/backend/core/training/worker.py +++ b/studio/backend/core/training/worker.py @@ -121,6 +121,7 @@ def run_training_process( sys.path.insert(0, backend_path) from core.training.trainer import UnslothTrainer, TrainingProgress + from utils.paths import ensure_dir, resolve_output_dir, resolve_tensorboard_dir import transformers logger.info("Subprocess loaded transformers %s", transformers.__version__) @@ -305,7 +306,14 @@ def run_training_process( # Generate output dir output_dir = config.get("output_dir") if not output_dir: - output_dir = f"./outputs/{model_name.replace('/', '_')}_{int(time.time())}" + output_dir = f"{model_name.replace('/', '_')}_{int(time.time())}" + output_dir = str(resolve_output_dir(output_dir)) + ensure_dir(Path(output_dir)) + + tensorboard_dir = config.get("tensorboard_dir") + if config.get("enable_tensorboard", False): + tensorboard_dir = str(resolve_tensorboard_dir(tensorboard_dir)) + ensure_dir(Path(tensorboard_dir)) # Start training (directly — no inner thread, we ARE the subprocess) _send_status(event_queue, "Starting training...") @@ -331,7 +339,7 @@ def run_training_process( wandb_project=config.get("wandb_project", "unsloth-training"), wandb_token=config.get("wandb_token"), enable_tensorboard=config.get("enable_tensorboard", False), - tensorboard_dir=config.get("tensorboard_dir", "runs"), + tensorboard_dir=tensorboard_dir, eval_dataset=eval_dataset, eval_steps=eval_steps, max_seq_length=config.get("max_seq_length", 2048), diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py index 358e7ea701..e226f4f27d 100644 --- a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py +++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py @@ -8,11 +8,13 @@ import re from pathlib import Path from typing import Any +from utils.paths import ensure_dir, unstructured_seed_cache_root + DEFAULT_CHUNK_SIZE = 1200 DEFAULT_CHUNK_OVERLAP = 200 MAX_CHUNK_SIZE = 20000 _MIN_BREAK_RATIO = 0.6 -_CACHE_DIR = Path.home() / ".cache" / "unsloth" / "data-recipe" / "unstructured-seed-cache" +_CACHE_DIR = unstructured_seed_cache_root() def resolve_chunking( @@ -85,7 +87,7 @@ def materialize_unstructured_seed_dataset( raise ValueError("No text found in unstructured seed source.") rows = [{"chunk_text": chunk} for chunk in chunks] - _CACHE_DIR.mkdir(parents=True, exist_ok=True) + ensure_dir(_CACHE_DIR) try: import pandas as pd except ImportError as exc: # pragma: no cover diff --git a/studio/backend/routes/data_recipe/seed.py b/studio/backend/routes/data_recipe/seed.py index e803c054c2..f388ce2323 100644 --- a/studio/backend/routes/data_recipe/seed.py +++ b/studio/backend/routes/data_recipe/seed.py @@ -18,6 +18,7 @@ from data_designer_unstructured_seed.chunking import ( resolve_chunking, ) from core.data_recipe.jsonable import to_preview_jsonable +from utils.paths import ensure_dir, seed_uploads_root from models.data_recipe import ( SeedInspectRequest, @@ -31,7 +32,7 @@ DATA_EXTS = (".parquet", ".jsonl", ".json", ".csv") DEFAULT_SPLIT = "train" LOCAL_UPLOAD_EXTS = {".csv", ".json", ".jsonl"} UNSTRUCTURED_UPLOAD_EXTS = {".txt", ".md"} -SEED_UPLOAD_DIR = Path.home() / ".cache" / "unsloth" / "data-recipe" / "seed-uploads" +SEED_UPLOAD_DIR = seed_uploads_root() def _serialize_preview_value(value: Any) -> Any: @@ -304,7 +305,7 @@ def inspect_seed_upload(payload: SeedInspectUploadRequest) -> SeedInspectRespons if len(file_bytes) > max_size_bytes: raise HTTPException(status_code=413, detail="file too large (max 50MB)") - SEED_UPLOAD_DIR.mkdir(parents=True, exist_ok=True) + ensure_dir(SEED_UPLOAD_DIR) stored_name = f"{uuid4().hex}_{filename}" stored_path = SEED_UPLOAD_DIR / stored_name stored_path.write_bytes(file_bytes) diff --git a/studio/backend/routes/datasets.py b/studio/backend/routes/datasets.py index 25965a847e..b8793fd9d0 100644 --- a/studio/backend/routes/datasets.py +++ b/studio/backend/routes/datasets.py @@ -42,6 +42,12 @@ from models.datasets import ( LocalDatasetsResponse, UploadDatasetResponse, ) +from utils.paths import ( + dataset_uploads_root, + ensure_dir, + recipe_datasets_root, + resolve_dataset_path, +) def _serialize_preview_value(value): @@ -91,9 +97,8 @@ _ARCHIVE_EXTS = ('.tar', '.tar.gz', '.tgz', '.gz', '.zst', '.zip', '.txt') DATA_EXTS = _TABULAR_EXTS + _ARCHIVE_EXTS LOCAL_FILE_EXTS = ('.json', '.jsonl', '.csv', '.parquet') LOCAL_UPLOAD_EXTS = {".csv", ".json", ".jsonl", ".parquet"} -BACKEND_ROOT = Path(__file__).resolve().parents[1] -LOCAL_DATASETS_ROOT = BACKEND_ROOT / "assets" / "datasets" -DATASET_UPLOAD_DIR = LOCAL_DATASETS_ROOT / "uploads" +LOCAL_DATASETS_ROOT = recipe_datasets_root() +DATASET_UPLOAD_DIR = dataset_uploads_root() def _safe_read_metadata(path: Path) -> dict | None: @@ -277,7 +282,7 @@ async def upload_dataset( ) max_size_bytes = 512 * 1024 * 1024 - DATASET_UPLOAD_DIR.mkdir(parents=True, exist_ok=True) + ensure_dir(DATASET_UPLOAD_DIR) stem = Path(filename).stem stored_name = f"{uuid4().hex}_{stem}{ext}" stored_path = DATASET_UPLOAD_DIR / stored_name @@ -333,7 +338,7 @@ def check_format( logger.info(f"Checking format for dataset: {request.dataset_name}") - dataset_path = Path(request.dataset_name) + dataset_path = resolve_dataset_path(request.dataset_name) total_rows = None if dataset_path.exists(): diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 387268edbb..4e7b1e6646 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -576,11 +576,16 @@ def _decode_audio_base64(b64: str) -> np.ndarray: import torchaudio import tempfile import os + from utils.paths import ensure_dir, tmp_root raw = base64.b64decode(b64) # torchaudio.load needs a file path or file-like object with format hint # Write to a temp file so torchaudio can auto-detect the format - with tempfile.NamedTemporaryFile(suffix=".audio", delete=False) as tmp: + with tempfile.NamedTemporaryFile( + suffix=".audio", + delete=False, + dir=str(ensure_dir(tmp_root())), + ) as tmp: tmp.write(raw) tmp_path = tmp.name try: diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py index e217ff63d8..190924493e 100644 --- a/studio/backend/routes/models.py +++ b/studio/backend/routes/models.py @@ -31,6 +31,7 @@ try: ) from utils.models.model_config import _pick_best_gguf, _extract_quant_label, is_audio_input_type from core.inference import get_inference_backend + from utils.paths import outputs_root, exports_root, resolve_output_dir, resolve_export_dir except ImportError: # Fallback: try to import from parent directory parent_backend = backend_path.parent / "backend" @@ -48,6 +49,7 @@ except ImportError: ) from utils.models.model_config import _pick_best_gguf, _extract_quant_label, is_audio_input_type from core.inference import get_inference_backend + from utils.paths import outputs_root, exports_root, resolve_output_dir, resolve_export_dir from models import ( CheckpointInfo, @@ -322,8 +324,8 @@ async def get_model_config( @router.get("/loras") async def scan_loras( - outputs_dir: str = Query(default="./outputs", description="Directory to scan for LoRA adapters"), - exports_dir: str = Query(default="./exports", description="Directory to scan for exported models"), + outputs_dir: str = Query(default=str(outputs_root()), description="Directory to scan for LoRA adapters"), + exports_dir: str = Query(default=str(exports_root()), description="Directory to scan for exported models"), current_subject: str = Depends(get_current_subject), ): """ @@ -333,10 +335,12 @@ async def scan_loras( (from exports_dir) in a single list, distinguished by source field. """ try: + resolved_outputs_dir = str(resolve_output_dir(outputs_dir)) + resolved_exports_dir = str(resolve_export_dir(exports_dir)) lora_list = [] # Scan training outputs - trained_loras = scan_trained_loras(outputs_dir=outputs_dir) + trained_loras = scan_trained_loras(outputs_dir=resolved_outputs_dir) for display_name, adapter_path in trained_loras: base_model = get_base_model_from_lora(adapter_path) lora_list.append(LoRAInfo( @@ -347,7 +351,7 @@ async def scan_loras( )) # Scan exported models (merged, LoRA, base — skips GGUF) - exported = scan_exported_models(exports_dir=exports_dir) + exported = scan_exported_models(exports_dir=resolved_exports_dir) for display_name, model_path, export_type, base_model in exported: lora_list.append(LoRAInfo( display_name=display_name, @@ -359,7 +363,7 @@ async def scan_loras( return LoRAScanResponse( loras=lora_list, - outputs_dir=outputs_dir + outputs_dir=resolved_outputs_dir ) except Exception as e: @@ -477,7 +481,7 @@ async def get_gguf_variants( @router.get("/checkpoints", response_model=CheckpointListResponse) async def list_checkpoints( outputs_dir: str = Query( - default="./outputs", + default=str(outputs_root()), description="Directory to scan for checkpoints", ), current_subject: str = Depends(get_current_subject), @@ -488,7 +492,8 @@ async def list_checkpoints( Scans the outputs folder for training runs and their checkpoints. """ try: - raw_models = scan_checkpoints(outputs_dir=outputs_dir) + resolved_outputs_dir = str(resolve_output_dir(outputs_dir)) + raw_models = scan_checkpoints(outputs_dir=resolved_outputs_dir) models = [ ModelCheckpoints( @@ -505,7 +510,7 @@ async def list_checkpoints( ] return CheckpointListResponse( - outputs_dir=outputs_dir, + outputs_dir=resolved_outputs_dir, models=models, ) except Exception as e: diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py index 3deabb5253..45fbff0172 100644 --- a/studio/backend/routes/training.py +++ b/studio/backend/routes/training.py @@ -23,6 +23,7 @@ if str(backend_path) not in sys.path: try: from core.training import get_training_backend from utils.models.model_config import load_model_defaults + from utils.paths import resolve_dataset_path except ImportError: # Fallback: try to import from parent directory parent_backend = backend_path.parent / "backend" @@ -30,6 +31,7 @@ except ImportError: sys.path.insert(0, str(parent_backend)) from core.training import get_training_backend from utils.models.model_config import load_model_defaults + from utils.paths import resolve_dataset_path # Auth from auth.authentication import get_current_subject @@ -117,24 +119,8 @@ async def start_training( if request.local_datasets: validated_datasets = [] missing_datasets = [] - # Get the backend directory (where this file is located) - backend_dir = Path(__file__).parent.parent - assets_datasets_dir = backend_dir / "assets" / "datasets" - for dataset_path in request.local_datasets: - dataset_file = Path(dataset_path) - - # If not absolute, try multiple locations - if not dataset_file.is_absolute(): - # First try: relative to current working directory - candidate = Path.cwd() / dataset_path - if not candidate.exists(): - # Second try: relative to assets/datasets folder - candidate = assets_datasets_dir / dataset_path - if not candidate.exists(): - # Third try: just the filename in assets/datasets - candidate = assets_datasets_dir / dataset_file.name - dataset_file = candidate + dataset_file = resolve_dataset_path(dataset_path) if not dataset_file.exists(): missing_datasets.append( diff --git a/studio/backend/utils/models/checkpoints.py b/studio/backend/utils/models/checkpoints.py index c601cf64db..c1cc0d0b78 100644 --- a/studio/backend/utils/models/checkpoints.py +++ b/studio/backend/utils/models/checkpoints.py @@ -8,6 +8,7 @@ import json import logging from pathlib import Path from typing import List, Optional, Tuple +from utils.paths import outputs_root, resolve_output_dir logger = logging.getLogger(__name__) @@ -33,7 +34,7 @@ def _read_checkpoint_loss(checkpoint_path: Path) -> Optional[float]: def scan_checkpoints( - outputs_dir: str = "./outputs", + outputs_dir: str = str(outputs_root()), ) -> List[Tuple[str, List[Tuple[str, str, Optional[float]]], dict]]: """ Scan outputs folder for training runs and their checkpoints. @@ -45,7 +46,7 @@ def scan_checkpoints( set to the loss of the last (highest-step) intermediate checkpoint. """ models = [] - outputs_path = Path(outputs_dir) + outputs_path = resolve_output_dir(outputs_dir) if not outputs_path.exists(): logger.warning(f"Outputs directory not found: {outputs_dir}") diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 8724f4b00b..472f994e2e 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -7,7 +7,15 @@ Model and LoRA configuration handling from transformers import AutoConfig from dataclasses import dataclass from typing import Optional, Dict, Any -from utils.paths import normalize_path, is_local_path, is_model_cached +from utils.paths import ( + normalize_path, + is_local_path, + is_model_cached, + outputs_root, + exports_root, + resolve_output_dir, + resolve_export_dir, +) from utils.utils import without_hf_auth import logging import os @@ -894,7 +902,7 @@ def download_gguf_file( return local_path -def scan_trained_loras(outputs_dir: str = "./outputs") -> List[Tuple[str, str]]: +def scan_trained_loras(outputs_dir: str = str(outputs_root())) -> List[Tuple[str, str]]: """ Scan outputs folder for trained LoRA adapters. @@ -908,7 +916,7 @@ def scan_trained_loras(outputs_dir: str = "./outputs") -> List[Tuple[str, str]]: ] """ trained_loras = [] - outputs_path = Path(outputs_dir) + outputs_path = resolve_output_dir(outputs_dir) if not outputs_path.exists(): logger.warning(f"Outputs directory not found: {outputs_dir}") @@ -937,7 +945,7 @@ def scan_trained_loras(outputs_dir: str = "./outputs") -> List[Tuple[str, str]]: logger.error(f"Error scanning outputs folder: {e}") return [] -def scan_exported_models(exports_dir: str = "./exports") -> List[Tuple[str, str, str, Optional[str]]]: +def scan_exported_models(exports_dir: str = str(exports_root())) -> List[Tuple[str, str, str, Optional[str]]]: """ Scan exports folder for exported models (merged, LoRA, GGUF). @@ -950,7 +958,7 @@ def scan_exported_models(exports_dir: str = "./exports") -> List[Tuple[str, str, export_type: "lora" | "merged" | "gguf" """ results = [] - exports_path = Path(exports_dir) + exports_path = resolve_export_dir(exports_dir) if not exports_path.exists(): return results @@ -1038,7 +1046,7 @@ def scan_exported_models(exports_dir: str = "./exports") -> List[Tuple[str, str, # Fallback: read base model from the original training run's # adapter_config.json in ./outputs/{run_name}/ if not base_model: - outputs_adapter_cfg = Path("./outputs") / run_dir.name / "adapter_config.json" + outputs_adapter_cfg = resolve_output_dir(run_dir.name) / "adapter_config.json" try: if outputs_adapter_cfg.exists(): cfg = json.loads(outputs_adapter_cfg.read_text()) diff --git a/studio/backend/utils/paths/__init__.py b/studio/backend/utils/paths/__init__.py index d196f8ce14..b13d949924 100644 --- a/studio/backend/utils/paths/__init__.py +++ b/studio/backend/utils/paths/__init__.py @@ -5,10 +5,52 @@ Path utilities for model and dataset handling """ from .path_utils import normalize_path, is_local_path, is_model_cached, get_cache_path +from .storage_roots import ( + studio_root, + assets_root, + datasets_root, + dataset_uploads_root, + recipe_datasets_root, + outputs_root, + exports_root, + auth_root, + auth_db_path, + tmp_root, + seed_uploads_root, + unstructured_seed_cache_root, + oxc_validator_tmp_root, + tensorboard_root, + ensure_dir, + resolve_under_root, + resolve_output_dir, + resolve_export_dir, + resolve_tensorboard_dir, + resolve_dataset_path, +) __all__ = [ 'normalize_path', 'is_local_path', 'is_model_cached', 'get_cache_path', + 'studio_root', + 'assets_root', + 'datasets_root', + 'dataset_uploads_root', + 'recipe_datasets_root', + 'outputs_root', + 'exports_root', + 'auth_root', + 'auth_db_path', + 'tmp_root', + 'seed_uploads_root', + 'unstructured_seed_cache_root', + 'oxc_validator_tmp_root', + 'tensorboard_root', + 'ensure_dir', + 'resolve_under_root', + 'resolve_output_dir', + 'resolve_export_dir', + 'resolve_tensorboard_dir', + 'resolve_dataset_path', ] diff --git a/studio/backend/utils/paths/storage_roots.py b/studio/backend/utils/paths/storage_roots.py new file mode 100644 index 0000000000..a005d604a8 --- /dev/null +++ b/studio/backend/utils/paths/storage_roots.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +from pathlib import Path +import tempfile + + +def studio_root() -> Path: + return Path.home() / ".unsloth" / "studio" + + +def assets_root() -> Path: + return studio_root() / "assets" + + +def datasets_root() -> Path: + return assets_root() / "datasets" + + +def dataset_uploads_root() -> Path: + return datasets_root() / "uploads" + + +def recipe_datasets_root() -> Path: + return datasets_root() / "recipes" + + +def outputs_root() -> Path: + return studio_root() / "outputs" + + +def exports_root() -> Path: + return studio_root() / "exports" + + +def auth_root() -> Path: + return studio_root() / "auth" + + +def auth_db_path() -> Path: + return auth_root() / "auth.db" + + +def tmp_root() -> Path: + return Path(tempfile.gettempdir()) / "unsloth-studio" + + +def seed_uploads_root() -> Path: + return tmp_root() / "seed-uploads" + + +def unstructured_seed_cache_root() -> Path: + return tmp_root() / "unstructured-seed-cache" + + +def oxc_validator_tmp_root() -> Path: + return tmp_root() / "oxc-validator" + + +def tensorboard_root() -> Path: + return studio_root() / "runs" + + +def ensure_dir(path: Path) -> Path: + path.mkdir(parents=True, exist_ok=True) + return path + + +def _clean_relative_path(path_value: str, *, strip_prefixes: tuple[str, ...] = ()) -> Path: + path = Path(path_value).expanduser() + parts = [part for part in path.parts if part not in ("", ".")] + while parts and parts[0] in strip_prefixes: + parts = parts[1:] + return Path(*parts) if parts else Path() + + +def resolve_under_root( + path_value: str | None, + *, + root: Path, + strip_prefixes: tuple[str, ...] = (), +) -> Path: + if not path_value or not str(path_value).strip(): + return root + + path = Path(str(path_value).strip()).expanduser() + if path.is_absolute(): + return path + + cleaned = _clean_relative_path(str(path), strip_prefixes=strip_prefixes) + return root / cleaned + + +def resolve_output_dir(path_value: str | None = None) -> Path: + return resolve_under_root( + path_value, + root=outputs_root(), + strip_prefixes=("outputs",), + ) + + +def resolve_export_dir(path_value: str | None = None) -> Path: + return resolve_under_root( + path_value, + root=exports_root(), + strip_prefixes=("exports",), + ) + + +def resolve_tensorboard_dir(path_value: str | None = None) -> Path: + return resolve_under_root( + path_value, + root=tensorboard_root(), + strip_prefixes=("runs", "tensorboard"), + ) + + +def resolve_dataset_path(path_value: str) -> Path: + path = Path(path_value).expanduser() + if path.is_absolute(): + return path + + parts = [part for part in Path(path_value).parts if part not in ("", ".")] + if parts[:2] == ["assets", "datasets"]: + parts = parts[2:] + if parts and parts[0] == "uploads": + cleaned = Path(*parts[1:]) if len(parts) > 1 else Path() + return dataset_uploads_root() / cleaned + if parts and parts[0] == "recipes": + cleaned = Path(*parts[1:]) if len(parts) > 1 else Path() + return recipe_datasets_root() / cleaned + + cleaned = Path(*parts) if parts else Path() + candidates = [ + dataset_uploads_root() / cleaned, + recipe_datasets_root() / cleaned, + datasets_root() / cleaned, + dataset_uploads_root() / cleaned.name, + recipe_datasets_root() / cleaned.name, + ] + for candidate in candidates: + if candidate.exists(): + return candidate + return candidates[0] diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts index c73b60d440..379ee5a8c9 100644 --- a/studio/frontend/src/features/chat/api/chat-api.ts +++ b/studio/frontend/src/features/chat/api/chat-api.ts @@ -49,9 +49,11 @@ export async function listModels(): Promise { return parseJsonOrThrow(response); } -export async function listLoras(outputsDir = "./outputs"): Promise { - const query = new URLSearchParams({ outputs_dir: outputsDir }).toString(); - const response = await authFetch(`/api/models/loras?${query}`); +export async function listLoras(outputsDir?: string): Promise { + const query = outputsDir + ? `?${new URLSearchParams({ outputs_dir: outputsDir }).toString()}` + : ""; + const response = await authFetch(`/api/models/loras${query}`); return parseJsonOrThrow(response); } diff --git a/studio/frontend/src/features/export/export-page.tsx b/studio/frontend/src/features/export/export-page.tsx index 05d697bd5e..c9bf251520 100644 --- a/studio/frontend/src/features/export/export-page.tsx +++ b/studio/frontend/src/features/export/export-page.tsx @@ -162,8 +162,8 @@ export function ExportPage() { // For other formats, nest under training-run/checkpoint const saveDir = exportMethod === "gguf" - ? `./exports/${(baseModelName.split("/").pop() ?? selectedModelIdx ?? "model")}-finetune-gguf` - : `./exports/${selectedModelIdx ?? "model"}/${checkpoint}`; + ? `${baseModelName.split("/").pop() ?? selectedModelIdx ?? "model"}-finetune-gguf` + : `${selectedModelIdx ?? "model"}/${checkpoint}`; const pushToHub = destination === "hub"; const repoId = pushToHub && hfUsername && modelName ? `${hfUsername}/${modelName}` diff --git a/install_python_stack.py b/studio/install_python_stack.py similarity index 98% rename from install_python_stack.py rename to studio/install_python_stack.py index 78ddc571d0..927dc625b7 100644 --- a/install_python_stack.py +++ b/studio/install_python_stack.py @@ -23,11 +23,11 @@ IS_WINDOWS = sys.platform == "win32" # ── Paths ────────────────────────────────────────────────────────────── SCRIPT_DIR = Path(__file__).resolve().parent -REQ_ROOT = SCRIPT_DIR / "studio" / "backend" / "requirements" +REQ_ROOT = SCRIPT_DIR / "backend" / "requirements" SINGLE_ENV = REQ_ROOT / "single-env" CONSTRAINTS = SINGLE_ENV / "constraints.txt" LOCAL_DD_UNSTRUCTURED_PLUGIN = ( - SCRIPT_DIR / "studio" / "backend" / "plugins" / "data-designer-unstructured-seed" + SCRIPT_DIR / "backend" / "plugins" / "data-designer-unstructured-seed" ) # ── Color support ────────────────────────────────────────────────────── diff --git a/setup.bat b/studio/setup.bat similarity index 100% rename from setup.bat rename to studio/setup.bat diff --git a/setup.ps1 b/studio/setup.ps1 similarity index 98% rename from setup.ps1 rename to studio/setup.ps1 index 5a573126aa..6597fc6746 100644 --- a/setup.ps1 +++ b/studio/setup.ps1 @@ -17,9 +17,9 @@ $ErrorActionPreference = "Stop" $ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path $PackageDir = Split-Path -Parent $ScriptDir -# Detect if running from pip install (no studio/frontend/ dir in repo) -$FrontendDir = Join-Path $ScriptDir "studio\frontend" -$OxcValidatorDir = Join-Path $ScriptDir "studio\backend\core\data_recipe\oxc-validator" +# Detect if running from pip install (no frontend/ dir in studio) +$FrontendDir = Join-Path $ScriptDir "frontend" +$OxcValidatorDir = Join-Path $ScriptDir "backend\core\data_recipe\oxc-validator" $IsPipInstall = -not (Test-Path $FrontendDir) # ───────────────────────────────────────────── @@ -587,7 +587,7 @@ if ($IsPipInstall) { Pop-Location $ErrorActionPreference = $prevEAP_npm Write-Host "[ERROR] npm install failed (exit code $LASTEXITCODE)" -ForegroundColor Red - Write-Host " Try running 'npm install' manually in studio/frontend/ to see errors" -ForegroundColor Yellow + Write-Host " Try running 'npm install' manually in frontend/ to see errors" -ForegroundColor Yellow exit 1 } npm run build 2>&1 | Out-Null @@ -599,7 +599,7 @@ if ($IsPipInstall) { } Pop-Location $ErrorActionPreference = $prevEAP_npm - Write-Host "[OK] Frontend built to studio/frontend/dist" -ForegroundColor Green + Write-Host "[OK] Frontend built to frontend/dist" -ForegroundColor Green } if (Test-Path $OxcValidatorDir) { @@ -648,8 +648,8 @@ if (-not $PythonCmd) { Write-Host "[OK] Using $PythonCmd ($(& $PythonCmd --version 2>&1))" -ForegroundColor Green # Always create a .venv for isolation -- even for pip installs. -# Created in the current working directory (where user ran the command). -$VenvDir = Join-Path (Get-Location) ".venv" +# Created in the repo root (parent of studio/). +$VenvDir = Join-Path (Split-Path -Parent $PSScriptRoot) ".venv" if (-not (Test-Path $VenvDir)) { Write-Host " Creating virtual environment at $VenvDir..." -ForegroundColor Cyan & $PythonCmd -m venv $VenvDir @@ -722,7 +722,7 @@ $ErrorActionPreference = $prevEAP # The training subprocess just prepends .venv_t5/ to sys.path — instant switch. Write-Host "" Write-Host " Pre-installing transformers 5.x for newer model support..." -ForegroundColor Cyan -$VenvT5Dir = Join-Path $PSScriptRoot ".venv_t5" +$VenvT5Dir = Join-Path (Split-Path -Parent $PSScriptRoot) ".venv_t5" if (Test-Path $VenvT5Dir) { Remove-Item -Recurse -Force $VenvT5Dir } New-Item -ItemType Directory -Path $VenvT5Dir -Force | Out-Null $prevEAP_t5 = $ErrorActionPreference @@ -956,10 +956,10 @@ if (Test-Path $LlamaServerBin) { # Add shell aliases (PowerShell profile + cmd batch files) # ============================================ Write-Host "" -$RepoDir = $PSScriptRoot +$RepoDir = Split-Path -Parent $PSScriptRoot $VenvPython = Join-Path $RepoDir ".venv\Scripts\python.exe" $CliScript = Join-Path $RepoDir "cli.py" -$FrontendDist = Join-Path $RepoDir "studio\frontend\dist" +$FrontendDist = Join-Path $PSScriptRoot "frontend\dist" $AliasAdded = $false # --- PowerShell profile: add functions --- diff --git a/setup.sh b/studio/setup.sh similarity index 93% rename from setup.sh rename to studio/setup.sh index b1ff403fbe..a1595fd935 100755 --- a/setup.sh +++ b/studio/setup.sh @@ -5,6 +5,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" # ── Helper: run command quietly, show output only on failure ── run_quiet() { @@ -28,9 +29,9 @@ echo "║ Unsloth Studio Setup Script ║" echo "╚══════════════════════════════════════╝" # ── Clean up stale Unsloth compiled caches ── -rm -rf "$SCRIPT_DIR/unsloth_compiled_cache" -rm -rf "$SCRIPT_DIR/studio/backend/unsloth_compiled_cache" -rm -rf "$SCRIPT_DIR/studio/tmp/unsloth_compiled_cache" +rm -rf "$REPO_ROOT/unsloth_compiled_cache" +rm -rf "$SCRIPT_DIR/backend/unsloth_compiled_cache" +rm -rf "$SCRIPT_DIR/tmp/unsloth_compiled_cache" # ── Detect Colab (like unsloth does) ── IS_COLAB=false @@ -99,13 +100,13 @@ echo "✅ Node $(node -v) | npm $(npm -v)" # ── 5. Build frontend ── echo "" echo "Building frontend..." -cd "$SCRIPT_DIR/studio/frontend" +cd "$SCRIPT_DIR/frontend" run_quiet "npm install" npm install run_quiet "npm run build" npm run build -cd "$SCRIPT_DIR/studio/backend/core/data_recipe/oxc-validator" +cd "$SCRIPT_DIR/backend/core/data_recipe/oxc-validator" run_quiet "npm install (oxc validator runtime)" npm install cd "$SCRIPT_DIR" -echo "✅ Frontend built to studio/frontend/dist" +echo "✅ Frontend built to frontend/dist" # ── 6. Python venv + deps ── echo "" @@ -168,7 +169,7 @@ fi BEST_VER=$("$BEST_PY" --version 2>&1 | awk '{print $2}') echo "✅ Using $BEST_PY ($BEST_VER) — compatible (3.${MIN_PY_MINOR}.x – 3.${MAX_PY_MINOR}.x)" -REQ_ROOT="$SCRIPT_DIR/studio/backend/requirements" +REQ_ROOT="$SCRIPT_DIR/backend/requirements" SINGLE_ENV_CONSTRAINTS="$REQ_ROOT/single-env/constraints.txt" SINGLE_ENV_DATA_DESIGNER="$REQ_ROOT/single-env/data-designer.txt" SINGLE_ENV_DATA_DESIGNER_DEPS="$REQ_ROOT/single-env/data-designer-deps.txt" @@ -183,11 +184,13 @@ if [ "$IS_COLAB" = true ]; then install_python_stack else # Local: create venv (always start fresh to preserve correct install order) + cd "$REPO_ROOT" rm -rf .venv rm -rf .venv_overlay # Remove legacy overlay (no longer used) rm -rf .venv_t5 # Will be rebuilt below "$BEST_PY" -m venv .venv source .venv/bin/activate + cd "$SCRIPT_DIR" install_python_stack # ── 6b. Pre-install transformers 5.x into .venv_t5/ ── @@ -196,7 +199,7 @@ else # The training subprocess just prepends .venv_t5/ to sys.path — instant switch. echo "" echo " Pre-installing transformers 5.x for newer model support..." - VENV_T5_DIR="$SCRIPT_DIR/.venv_t5" + VENV_T5_DIR="$REPO_ROOT/.venv_t5" mkdir -p "$VENV_T5_DIR" run_quiet "pip install transformers 5.x" pip install --target "$VENV_T5_DIR" --no-deps "transformers==5.2.0" run_quiet "pip install huggingface_hub for t5" pip install --target "$VENV_T5_DIR" --no-deps "huggingface_hub==1.3.0" @@ -306,32 +309,30 @@ rm -rf "$LLAMA_CPP_DIR" # This alias hardcodes the venv python path so users don't need to activate. if [ "$IS_COLAB" = false ]; then echo "" -REPO_DIR="$SCRIPT_DIR" +REPO_DIR="$REPO_ROOT" # Detect the user's default shell and pick the right rc file USER_SHELL="$(basename "${SHELL:-/bin/bash}")" case "$USER_SHELL" in zsh) SHELL_RC="$HOME/.zshrc" - ALIAS_BLOCK="alias unsloth-studio='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist' -alias unsloth-ui='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist'" + ALIAS_BLOCK="alias unsloth-studio='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist' +alias unsloth-ui='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist'" ;; fish) SHELL_RC="$HOME/.config/fish/config.fish" - # fish uses 'abbr' or 'function'; a simple alias works via 'alias' in config.fish - ALIAS_BLOCK="alias unsloth-studio '${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist' -alias unsloth-ui '${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist'" + ALIAS_BLOCK="alias unsloth-studio '${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist' +alias unsloth-ui '${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist'" ;; ksh) SHELL_RC="$HOME/.kshrc" - ALIAS_BLOCK="alias unsloth-studio='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist' -alias unsloth-ui='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist'" + ALIAS_BLOCK="alias unsloth-studio='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist' +alias unsloth-ui='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist'" ;; *) - # Default to bash for bash and any other POSIX-compatible shell SHELL_RC="$HOME/.bashrc" - ALIAS_BLOCK="alias unsloth-studio='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist' -alias unsloth-ui='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist'" + ALIAS_BLOCK="alias unsloth-studio='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist' +alias unsloth-ui='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist'" ;; esac