diff --git a/studio/install_python_stack.py b/install_python_stack.py similarity index 98% rename from studio/install_python_stack.py rename to install_python_stack.py index 927dc625b7..78ddc571d0 100644 --- a/studio/install_python_stack.py +++ b/install_python_stack.py @@ -23,11 +23,11 @@ IS_WINDOWS = sys.platform == "win32" # ── Paths ────────────────────────────────────────────────────────────── SCRIPT_DIR = Path(__file__).resolve().parent -REQ_ROOT = SCRIPT_DIR / "backend" / "requirements" +REQ_ROOT = SCRIPT_DIR / "studio" / "backend" / "requirements" SINGLE_ENV = REQ_ROOT / "single-env" CONSTRAINTS = SINGLE_ENV / "constraints.txt" LOCAL_DD_UNSTRUCTURED_PLUGIN = ( - SCRIPT_DIR / "backend" / "plugins" / "data-designer-unstructured-seed" + SCRIPT_DIR / "studio" / "backend" / "plugins" / "data-designer-unstructured-seed" ) # ── Color support ────────────────────────────────────────────────────── diff --git a/studio/setup.bat b/setup.bat similarity index 100% rename from studio/setup.bat rename to setup.bat diff --git a/studio/setup.ps1 b/setup.ps1 similarity index 98% rename from studio/setup.ps1 rename to setup.ps1 index 6597fc6746..5a573126aa 100644 --- a/studio/setup.ps1 +++ b/setup.ps1 @@ -17,9 +17,9 @@ $ErrorActionPreference = "Stop" $ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path $PackageDir = Split-Path -Parent $ScriptDir -# Detect if running from pip install (no frontend/ dir in studio) -$FrontendDir = Join-Path $ScriptDir "frontend" -$OxcValidatorDir = Join-Path $ScriptDir "backend\core\data_recipe\oxc-validator" +# Detect if running from pip install (no studio/frontend/ dir in repo) +$FrontendDir = Join-Path $ScriptDir "studio\frontend" +$OxcValidatorDir = Join-Path $ScriptDir "studio\backend\core\data_recipe\oxc-validator" $IsPipInstall = -not (Test-Path $FrontendDir) # ───────────────────────────────────────────── @@ -587,7 +587,7 @@ if ($IsPipInstall) { Pop-Location $ErrorActionPreference = $prevEAP_npm Write-Host "[ERROR] npm install failed (exit code $LASTEXITCODE)" -ForegroundColor Red - Write-Host " Try running 'npm install' manually in frontend/ to see errors" -ForegroundColor Yellow + Write-Host " Try running 'npm install' manually in studio/frontend/ to see errors" -ForegroundColor Yellow exit 1 } npm run build 2>&1 | Out-Null @@ -599,7 +599,7 @@ if ($IsPipInstall) { } Pop-Location $ErrorActionPreference = $prevEAP_npm - Write-Host "[OK] Frontend built to frontend/dist" -ForegroundColor Green + Write-Host "[OK] Frontend built to studio/frontend/dist" -ForegroundColor Green } if (Test-Path $OxcValidatorDir) { @@ -648,8 +648,8 @@ if (-not $PythonCmd) { Write-Host "[OK] Using $PythonCmd ($(& $PythonCmd --version 2>&1))" -ForegroundColor Green # Always create a .venv for isolation -- even for pip installs. -# Created in the repo root (parent of studio/). -$VenvDir = Join-Path (Split-Path -Parent $PSScriptRoot) ".venv" +# Created in the current working directory (where user ran the command). +$VenvDir = Join-Path (Get-Location) ".venv" if (-not (Test-Path $VenvDir)) { Write-Host " Creating virtual environment at $VenvDir..." -ForegroundColor Cyan & $PythonCmd -m venv $VenvDir @@ -722,7 +722,7 @@ $ErrorActionPreference = $prevEAP # The training subprocess just prepends .venv_t5/ to sys.path — instant switch. Write-Host "" Write-Host " Pre-installing transformers 5.x for newer model support..." -ForegroundColor Cyan -$VenvT5Dir = Join-Path (Split-Path -Parent $PSScriptRoot) ".venv_t5" +$VenvT5Dir = Join-Path $PSScriptRoot ".venv_t5" if (Test-Path $VenvT5Dir) { Remove-Item -Recurse -Force $VenvT5Dir } New-Item -ItemType Directory -Path $VenvT5Dir -Force | Out-Null $prevEAP_t5 = $ErrorActionPreference @@ -956,10 +956,10 @@ if (Test-Path $LlamaServerBin) { # Add shell aliases (PowerShell profile + cmd batch files) # ============================================ Write-Host "" -$RepoDir = Split-Path -Parent $PSScriptRoot +$RepoDir = $PSScriptRoot $VenvPython = Join-Path $RepoDir ".venv\Scripts\python.exe" $CliScript = Join-Path $RepoDir "cli.py" -$FrontendDist = Join-Path $PSScriptRoot "frontend\dist" +$FrontendDist = Join-Path $RepoDir "studio\frontend\dist" $AliasAdded = $false # --- PowerShell profile: add functions --- diff --git a/studio/setup.sh b/setup.sh similarity index 93% rename from studio/setup.sh rename to setup.sh index a1595fd935..b1ff403fbe 100755 --- a/studio/setup.sh +++ b/setup.sh @@ -5,7 +5,6 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" # ── Helper: run command quietly, show output only on failure ── run_quiet() { @@ -29,9 +28,9 @@ echo "║ Unsloth Studio Setup Script ║" echo "╚══════════════════════════════════════╝" # ── Clean up stale Unsloth compiled caches ── -rm -rf "$REPO_ROOT/unsloth_compiled_cache" -rm -rf "$SCRIPT_DIR/backend/unsloth_compiled_cache" -rm -rf "$SCRIPT_DIR/tmp/unsloth_compiled_cache" +rm -rf "$SCRIPT_DIR/unsloth_compiled_cache" +rm -rf "$SCRIPT_DIR/studio/backend/unsloth_compiled_cache" +rm -rf "$SCRIPT_DIR/studio/tmp/unsloth_compiled_cache" # ── Detect Colab (like unsloth does) ── IS_COLAB=false @@ -100,13 +99,13 @@ echo "✅ Node $(node -v) | npm $(npm -v)" # ── 5. Build frontend ── echo "" echo "Building frontend..." -cd "$SCRIPT_DIR/frontend" +cd "$SCRIPT_DIR/studio/frontend" run_quiet "npm install" npm install run_quiet "npm run build" npm run build -cd "$SCRIPT_DIR/backend/core/data_recipe/oxc-validator" +cd "$SCRIPT_DIR/studio/backend/core/data_recipe/oxc-validator" run_quiet "npm install (oxc validator runtime)" npm install cd "$SCRIPT_DIR" -echo "✅ Frontend built to frontend/dist" +echo "✅ Frontend built to studio/frontend/dist" # ── 6. Python venv + deps ── echo "" @@ -169,7 +168,7 @@ fi BEST_VER=$("$BEST_PY" --version 2>&1 | awk '{print $2}') echo "✅ Using $BEST_PY ($BEST_VER) — compatible (3.${MIN_PY_MINOR}.x – 3.${MAX_PY_MINOR}.x)" -REQ_ROOT="$SCRIPT_DIR/backend/requirements" +REQ_ROOT="$SCRIPT_DIR/studio/backend/requirements" SINGLE_ENV_CONSTRAINTS="$REQ_ROOT/single-env/constraints.txt" SINGLE_ENV_DATA_DESIGNER="$REQ_ROOT/single-env/data-designer.txt" SINGLE_ENV_DATA_DESIGNER_DEPS="$REQ_ROOT/single-env/data-designer-deps.txt" @@ -184,13 +183,11 @@ if [ "$IS_COLAB" = true ]; then install_python_stack else # Local: create venv (always start fresh to preserve correct install order) - cd "$REPO_ROOT" rm -rf .venv rm -rf .venv_overlay # Remove legacy overlay (no longer used) rm -rf .venv_t5 # Will be rebuilt below "$BEST_PY" -m venv .venv source .venv/bin/activate - cd "$SCRIPT_DIR" install_python_stack # ── 6b. Pre-install transformers 5.x into .venv_t5/ ── @@ -199,7 +196,7 @@ else # The training subprocess just prepends .venv_t5/ to sys.path — instant switch. echo "" echo " Pre-installing transformers 5.x for newer model support..." - VENV_T5_DIR="$REPO_ROOT/.venv_t5" + VENV_T5_DIR="$SCRIPT_DIR/.venv_t5" mkdir -p "$VENV_T5_DIR" run_quiet "pip install transformers 5.x" pip install --target "$VENV_T5_DIR" --no-deps "transformers==5.2.0" run_quiet "pip install huggingface_hub for t5" pip install --target "$VENV_T5_DIR" --no-deps "huggingface_hub==1.3.0" @@ -309,30 +306,32 @@ rm -rf "$LLAMA_CPP_DIR" # This alias hardcodes the venv python path so users don't need to activate. if [ "$IS_COLAB" = false ]; then echo "" -REPO_DIR="$REPO_ROOT" +REPO_DIR="$SCRIPT_DIR" # Detect the user's default shell and pick the right rc file USER_SHELL="$(basename "${SHELL:-/bin/bash}")" case "$USER_SHELL" in zsh) SHELL_RC="$HOME/.zshrc" - ALIAS_BLOCK="alias unsloth-studio='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist' -alias unsloth-ui='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist'" + ALIAS_BLOCK="alias unsloth-studio='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist' +alias unsloth-ui='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist'" ;; fish) SHELL_RC="$HOME/.config/fish/config.fish" - ALIAS_BLOCK="alias unsloth-studio '${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist' -alias unsloth-ui '${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist'" + # fish uses 'abbr' or 'function'; a simple alias works via 'alias' in config.fish + ALIAS_BLOCK="alias unsloth-studio '${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist' +alias unsloth-ui '${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist'" ;; ksh) SHELL_RC="$HOME/.kshrc" - ALIAS_BLOCK="alias unsloth-studio='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist' -alias unsloth-ui='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist'" + ALIAS_BLOCK="alias unsloth-studio='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist' +alias unsloth-ui='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist'" ;; *) + # Default to bash for bash and any other POSIX-compatible shell SHELL_RC="$HOME/.bashrc" - ALIAS_BLOCK="alias unsloth-studio='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist' -alias unsloth-ui='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist'" + ALIAS_BLOCK="alias unsloth-studio='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist' +alias unsloth-ui='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist'" ;; esac diff --git a/studio/backend/auth/storage.py b/studio/backend/auth/storage.py index cb5df6e3f9..f72a7ed943 100644 --- a/studio/backend/auth/storage.py +++ b/studio/backend/auth/storage.py @@ -7,11 +7,10 @@ SQLite storage for authentication data (user credentials + JWT secret). import hashlib import sqlite3 from datetime import datetime, timezone +from pathlib import Path from typing import Optional, Tuple -from utils.paths import auth_db_path, ensure_dir - -DB_PATH = auth_db_path() +DB_PATH = Path(__file__).parent / "auth.db" def _hash_token(token: str) -> str: @@ -21,7 +20,6 @@ def _hash_token(token: str) -> str: def get_connection() -> sqlite3.Connection: """Get a connection to the auth database, creating tables if needed.""" - ensure_dir(DB_PATH.parent) conn = sqlite3.connect(DB_PATH) conn.row_factory = sqlite3.Row conn.execute( @@ -258,3 +256,4 @@ def revoke_user_refresh_tokens(username: str) -> None: conn.commit() finally: conn.close() + diff --git a/studio/backend/core/data_recipe/jobs/worker.py b/studio/backend/core/data_recipe/jobs/worker.py index 1d898f866f..fbe9d91b6d 100644 --- a/studio/backend/core/data_recipe/jobs/worker.py +++ b/studio/backend/core/data_recipe/jobs/worker.py @@ -16,9 +16,9 @@ from typing import Any from ..jsonable import to_jsonable, to_preview_jsonable from .constants import EVENT_JOB_COMPLETED, EVENT_JOB_ERROR, EVENT_JOB_STARTED from ..service import build_config_builder, create_data_designer -from utils.paths import ensure_dir, recipe_datasets_root -_ARTIFACT_ROOT = recipe_datasets_root() +_PROJECT_ROOT = Path(__file__).resolve().parents[5] +_ARTIFACT_ROOT = _PROJECT_ROOT / "studio" / "backend" / "assets" / "datasets" class _QueueLogHandler(logging.Handler): @@ -88,7 +88,7 @@ def run_job_process( artifact_root=_ARTIFACT_ROOT, ) merge_batches = bool(run.get("merge_batches")) - ensure_dir(_ARTIFACT_ROOT) + _ARTIFACT_ROOT.mkdir(parents=True, exist_ok=True) run_config_raw = run.get("run_config") or {} builder = build_config_builder(recipe) diff --git a/studio/backend/core/data_recipe/local_callable_validators.py b/studio/backend/core/data_recipe/local_callable_validators.py index a7e211eca5..5e2749d45d 100644 --- a/studio/backend/core/data_recipe/local_callable_validators.py +++ b/studio/backend/core/data_recipe/local_callable_validators.py @@ -5,7 +5,6 @@ from __future__ import annotations import json import logging -import os import subprocess from copy import deepcopy from dataclasses import dataclass @@ -13,8 +12,6 @@ from functools import lru_cache from pathlib import Path from typing import Any -from utils.paths import ensure_dir, oxc_validator_tmp_root - logger = logging.getLogger(__name__) OXC_VALIDATION_FN_MARKER = "unsloth_oxc_validator" @@ -238,12 +235,6 @@ def _run_oxc_batch( "codes": code_values, } try: - tmp_dir = ensure_dir(oxc_validator_tmp_root()) - env = dict(os.environ) - tmp_dir_str = str(tmp_dir) - env["TMPDIR"] = tmp_dir_str - env["TMP"] = tmp_dir_str - env["TEMP"] = tmp_dir_str proc = subprocess.run( ["node", str(_OXC_RUNNER_PATH)], cwd=str(_OXC_TOOL_DIR), @@ -251,7 +242,6 @@ def _run_oxc_batch( text=True, capture_output=True, check=False, - env=env, ) except (OSError, ValueError) as exc: logger.warning("OXC subprocess launch failed: %s", exc) diff --git a/studio/backend/core/export/export.py b/studio/backend/core/export/export.py index ef19530a2c..986da0def2 100644 --- a/studio/backend/core/export/export.py +++ b/studio/backend/core/export/export.py @@ -21,7 +21,6 @@ from utils.hardware import clear_gpu_cache from utils.models import is_vision_model, get_base_model_from_lora from utils.models.model_config import detect_audio_type -from utils.paths import ensure_dir, outputs_root, resolve_export_dir, resolve_output_dir from core.inference import get_inference_backend logger = logging.getLogger(__name__) @@ -129,7 +128,7 @@ class ExportBackend: logger.error(f"Error during memory cleanup: {e}") return False - def scan_checkpoints(self, outputs_dir: str = str(outputs_root())) -> List[Tuple[str, List[Tuple[str, str]]]]: + def scan_checkpoints(self, outputs_dir: str = "./outputs") -> List[Tuple[str, List[Tuple[str, str]]]]: """ Scan outputs folder for training runs and their checkpoints. @@ -326,9 +325,8 @@ class ExportBackend: # Save locally if requested if save_directory: - save_directory = str(resolve_export_dir(save_directory)) logger.info(f"Saving merged model locally to: {save_directory}") - ensure_dir(Path(save_directory)) + os.makedirs(save_directory, exist_ok=True) self.current_model.save_pretrained_merged( save_directory, @@ -388,9 +386,8 @@ class ExportBackend: try: # Save locally if requested if save_directory: - save_directory = str(resolve_export_dir(save_directory)) logger.info(f"Saving base model locally to: {save_directory}") - ensure_dir(Path(save_directory)) + os.makedirs(save_directory, exist_ok=True) self.current_model.save_pretrained(save_directory) self.current_tokenizer.save_pretrained(save_directory) @@ -478,7 +475,6 @@ class ExportBackend: # Save locally if requested if save_directory: - save_directory = str(resolve_export_dir(save_directory)) # Resolve to absolute path so unsloth's relative-path internals # (check_llama_cpp, use_local_gguf, _download_convert_hf_to_gguf) # all resolve against the repo root cwd, NOT the export directory. @@ -486,7 +482,7 @@ class ExportBackend: logger.info(f"Saving GGUF model locally to: {abs_save_dir}") # Create the directory if it doesn't exist - ensure_dir(Path(abs_save_dir)) + os.makedirs(abs_save_dir, exist_ok=True) # On WSL, patch out sudo check before llama.cpp build _apply_wsl_sudo_patch() @@ -586,9 +582,8 @@ class ExportBackend: try: # Save locally if requested if save_directory: - save_directory = str(resolve_export_dir(save_directory)) logger.info(f"Saving LoRA adapter locally to: {save_directory}") - ensure_dir(Path(save_directory)) + os.makedirs(save_directory, exist_ok=True) self.current_model.save_pretrained(save_directory) self.current_tokenizer.save_pretrained(save_directory) diff --git a/studio/backend/core/export/orchestrator.py b/studio/backend/core/export/orchestrator.py index 37604c81df..d6a04a3c1b 100644 --- a/studio/backend/core/export/orchestrator.py +++ b/studio/backend/core/export/orchestrator.py @@ -21,7 +21,6 @@ import threading import time from pathlib import Path from typing import Any, List, Optional, Tuple -from utils.paths import outputs_root logger = logging.getLogger(__name__) @@ -379,7 +378,7 @@ class ExportOrchestrator: return success def scan_checkpoints( - self, outputs_dir: str = str(outputs_root()) + self, outputs_dir: str = "./outputs" ) -> List[Tuple[str, list]]: """Scan for checkpoints — no ML imports needed, runs locally.""" from utils.models.checkpoints import scan_checkpoints diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py index 5a03cd6653..31e05d0bda 100644 --- a/studio/backend/core/training/trainer.py +++ b/studio/backend/core/training/trainer.py @@ -30,21 +30,13 @@ from datasets import Dataset, load_dataset from utils.models import is_vision_model, detect_audio_type from utils.datasets import format_and_template_dataset from utils.datasets import MODEL_TO_TEMPLATE_MAPPER, TEMPLATE_TO_RESPONSES_MAPPER -from utils.paths import ensure_dir, resolve_dataset_path, resolve_output_dir, resolve_tensorboard_dir from trl import SFTTrainer, SFTConfig logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) - - -def _build_report_targets(training_args) -> list[str] | str: - report_to: list[str] = [] - if training_args.get("enable_wandb", False): - report_to.append("wandb") - if training_args.get("enable_tensorboard", False): - report_to.append("tensorboard") - return report_to or "none" +_BACKEND_ROOT = Path(__file__).resolve().parents[2] +_ASSETS_DATASETS_ROOT = _BACKEND_ROOT / "assets" / "datasets" @dataclass @@ -213,14 +205,9 @@ class UnslothTrainer: "lr_scheduler_type": lr_scheduler_type, "seed": random_seed, "output_dir": output_dir, - "report_to": _build_report_targets(training_args), + "report_to": ["wandb"] if training_args.get('enable_wandb', False) else "none", } - if training_args.get("enable_tensorboard", False): - config["logging_dir"] = str( - resolve_tensorboard_dir(training_args.get("tensorboard_dir")) - ) - # max_steps vs epochs if max_steps_val and max_steps_val > 0: config["max_steps"] = max_steps_val @@ -1542,7 +1529,6 @@ class UnslothTrainer: import numpy as np import soundfile as sf from datasets import Dataset as HFDataset - from utils.paths import ensure_dir, tmp_root device = "cuda" if torch.cuda.is_available() else "cpu" @@ -1639,18 +1625,10 @@ class UnslothTrainer: audio_bytes = buf.getvalue() # 1. Get word timings from Whisper - with tempfile.NamedTemporaryFile( - suffix=".wav", - delete=False, - dir=str(ensure_dir(tmp_root())), - ) as tmp: + with tempfile.NamedTemporaryFile(suffix=".wav", delete=True) as tmp: tmp.write(audio_bytes) tmp.flush() - tmp_path = tmp.name - try: - whisper_result = whisper_model.transcribe(tmp_path, word_timestamps=True) - finally: - Path(tmp_path).unlink(missing_ok=True) + whisper_result = whisper_model.transcribe(tmp.name, word_timestamps=True) normalized_transcript = text_normalizations(text) words_with_timings = [] @@ -1838,7 +1816,7 @@ class UnslothTrainer: file_path = dataset_file else: # Fallback: try relative to assets/datasets - file_path = str(resolve_dataset_path(dataset_file)) + file_path = str(_ASSETS_DATASETS_ROOT / dataset_file) file_path_obj = Path(file_path) @@ -2091,7 +2069,7 @@ class UnslothTrainer: dataset: Dataset, eval_dataset: Dataset = None, eval_steps: float = 0.00, - output_dir: str | None = None, + output_dir: str = "./outputs", num_epochs: int = 3, learning_rate: float = 5e-5, batch_size: int = 2, @@ -2108,7 +2086,7 @@ class UnslothTrainer: wandb_project: str = "unsloth-training", wandb_token: str = None, enable_tensorboard: bool = False, - tensorboard_dir: str | None = None, + tensorboard_dir: str = "runs", **kwargs) -> bool: """Start training in a separate thread""" @@ -2196,8 +2174,8 @@ class UnslothTrainer: wandb.init(project=training_args.get('wandb_project', 'unsloth-training')) # Create output directory - output_dir = str(resolve_output_dir(training_args.get("output_dir"))) - ensure_dir(Path(output_dir)) + output_dir = training_args.get('output_dir', './outputs') + os.makedirs(output_dir, exist_ok=True) # ========== AUDIO TRAINER BRANCH ========== if self._audio_type == 'csm': @@ -2407,15 +2385,11 @@ class UnslothTrainer: "weight_decay": training_args.get('weight_decay', 0.01), "seed": training_args.get('random_seed', 3407), "output_dir": output_dir, - "report_to": _build_report_targets(training_args), + "report_to": ["wandb"] if training_args.get('enable_wandb', False) else "none", "include_num_input_tokens_seen": True, # Enable token counting "dataset_num_proc": 1 if (self.is_audio or self.is_audio_vlm or self._cuda_audio_used) else safe_num_proc(max(1, os.cpu_count() // 4)), "max_seq_length": training_args.get('max_seq_length', 2048), } - if training_args.get("enable_tensorboard", False): - config_args["logging_dir"] = str( - resolve_tensorboard_dir(training_args.get("tensorboard_dir")) - ) print(f"[DEBUG] dataset_num_proc={config_args['dataset_num_proc']} (is_audio={self.is_audio}, is_audio_vlm={self.is_audio_vlm}, _cuda_audio_used={self._cuda_audio_used})") # On Windows with transformers 5.x, disable DataLoader multiprocessing diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py index 509f99d3ec..7b98865eea 100644 --- a/studio/backend/core/training/worker.py +++ b/studio/backend/core/training/worker.py @@ -121,7 +121,6 @@ def run_training_process( sys.path.insert(0, backend_path) from core.training.trainer import UnslothTrainer, TrainingProgress - from utils.paths import ensure_dir, resolve_output_dir, resolve_tensorboard_dir import transformers logger.info("Subprocess loaded transformers %s", transformers.__version__) @@ -306,14 +305,7 @@ def run_training_process( # Generate output dir output_dir = config.get("output_dir") if not output_dir: - output_dir = f"{model_name.replace('/', '_')}_{int(time.time())}" - output_dir = str(resolve_output_dir(output_dir)) - ensure_dir(Path(output_dir)) - - tensorboard_dir = config.get("tensorboard_dir") - if config.get("enable_tensorboard", False): - tensorboard_dir = str(resolve_tensorboard_dir(tensorboard_dir)) - ensure_dir(Path(tensorboard_dir)) + output_dir = f"./outputs/{model_name.replace('/', '_')}_{int(time.time())}" # Start training (directly — no inner thread, we ARE the subprocess) _send_status(event_queue, "Starting training...") @@ -339,7 +331,7 @@ def run_training_process( wandb_project=config.get("wandb_project", "unsloth-training"), wandb_token=config.get("wandb_token"), enable_tensorboard=config.get("enable_tensorboard", False), - tensorboard_dir=tensorboard_dir, + tensorboard_dir=config.get("tensorboard_dir", "runs"), eval_dataset=eval_dataset, eval_steps=eval_steps, max_seq_length=config.get("max_seq_length", 2048), diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py index e226f4f27d..358e7ea701 100644 --- a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py +++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py @@ -8,13 +8,11 @@ import re from pathlib import Path from typing import Any -from utils.paths import ensure_dir, unstructured_seed_cache_root - DEFAULT_CHUNK_SIZE = 1200 DEFAULT_CHUNK_OVERLAP = 200 MAX_CHUNK_SIZE = 20000 _MIN_BREAK_RATIO = 0.6 -_CACHE_DIR = unstructured_seed_cache_root() +_CACHE_DIR = Path.home() / ".cache" / "unsloth" / "data-recipe" / "unstructured-seed-cache" def resolve_chunking( @@ -87,7 +85,7 @@ def materialize_unstructured_seed_dataset( raise ValueError("No text found in unstructured seed source.") rows = [{"chunk_text": chunk} for chunk in chunks] - ensure_dir(_CACHE_DIR) + _CACHE_DIR.mkdir(parents=True, exist_ok=True) try: import pandas as pd except ImportError as exc: # pragma: no cover diff --git a/studio/backend/routes/data_recipe/seed.py b/studio/backend/routes/data_recipe/seed.py index f388ce2323..e803c054c2 100644 --- a/studio/backend/routes/data_recipe/seed.py +++ b/studio/backend/routes/data_recipe/seed.py @@ -18,7 +18,6 @@ from data_designer_unstructured_seed.chunking import ( resolve_chunking, ) from core.data_recipe.jsonable import to_preview_jsonable -from utils.paths import ensure_dir, seed_uploads_root from models.data_recipe import ( SeedInspectRequest, @@ -32,7 +31,7 @@ DATA_EXTS = (".parquet", ".jsonl", ".json", ".csv") DEFAULT_SPLIT = "train" LOCAL_UPLOAD_EXTS = {".csv", ".json", ".jsonl"} UNSTRUCTURED_UPLOAD_EXTS = {".txt", ".md"} -SEED_UPLOAD_DIR = seed_uploads_root() +SEED_UPLOAD_DIR = Path.home() / ".cache" / "unsloth" / "data-recipe" / "seed-uploads" def _serialize_preview_value(value: Any) -> Any: @@ -305,7 +304,7 @@ def inspect_seed_upload(payload: SeedInspectUploadRequest) -> SeedInspectRespons if len(file_bytes) > max_size_bytes: raise HTTPException(status_code=413, detail="file too large (max 50MB)") - ensure_dir(SEED_UPLOAD_DIR) + SEED_UPLOAD_DIR.mkdir(parents=True, exist_ok=True) stored_name = f"{uuid4().hex}_{filename}" stored_path = SEED_UPLOAD_DIR / stored_name stored_path.write_bytes(file_bytes) diff --git a/studio/backend/routes/datasets.py b/studio/backend/routes/datasets.py index b8793fd9d0..25965a847e 100644 --- a/studio/backend/routes/datasets.py +++ b/studio/backend/routes/datasets.py @@ -42,12 +42,6 @@ from models.datasets import ( LocalDatasetsResponse, UploadDatasetResponse, ) -from utils.paths import ( - dataset_uploads_root, - ensure_dir, - recipe_datasets_root, - resolve_dataset_path, -) def _serialize_preview_value(value): @@ -97,8 +91,9 @@ _ARCHIVE_EXTS = ('.tar', '.tar.gz', '.tgz', '.gz', '.zst', '.zip', '.txt') DATA_EXTS = _TABULAR_EXTS + _ARCHIVE_EXTS LOCAL_FILE_EXTS = ('.json', '.jsonl', '.csv', '.parquet') LOCAL_UPLOAD_EXTS = {".csv", ".json", ".jsonl", ".parquet"} -LOCAL_DATASETS_ROOT = recipe_datasets_root() -DATASET_UPLOAD_DIR = dataset_uploads_root() +BACKEND_ROOT = Path(__file__).resolve().parents[1] +LOCAL_DATASETS_ROOT = BACKEND_ROOT / "assets" / "datasets" +DATASET_UPLOAD_DIR = LOCAL_DATASETS_ROOT / "uploads" def _safe_read_metadata(path: Path) -> dict | None: @@ -282,7 +277,7 @@ async def upload_dataset( ) max_size_bytes = 512 * 1024 * 1024 - ensure_dir(DATASET_UPLOAD_DIR) + DATASET_UPLOAD_DIR.mkdir(parents=True, exist_ok=True) stem = Path(filename).stem stored_name = f"{uuid4().hex}_{stem}{ext}" stored_path = DATASET_UPLOAD_DIR / stored_name @@ -338,7 +333,7 @@ def check_format( logger.info(f"Checking format for dataset: {request.dataset_name}") - dataset_path = resolve_dataset_path(request.dataset_name) + dataset_path = Path(request.dataset_name) total_rows = None if dataset_path.exists(): diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 4e7b1e6646..387268edbb 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -576,16 +576,11 @@ def _decode_audio_base64(b64: str) -> np.ndarray: import torchaudio import tempfile import os - from utils.paths import ensure_dir, tmp_root raw = base64.b64decode(b64) # torchaudio.load needs a file path or file-like object with format hint # Write to a temp file so torchaudio can auto-detect the format - with tempfile.NamedTemporaryFile( - suffix=".audio", - delete=False, - dir=str(ensure_dir(tmp_root())), - ) as tmp: + with tempfile.NamedTemporaryFile(suffix=".audio", delete=False) as tmp: tmp.write(raw) tmp_path = tmp.name try: diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py index 190924493e..e217ff63d8 100644 --- a/studio/backend/routes/models.py +++ b/studio/backend/routes/models.py @@ -31,7 +31,6 @@ try: ) from utils.models.model_config import _pick_best_gguf, _extract_quant_label, is_audio_input_type from core.inference import get_inference_backend - from utils.paths import outputs_root, exports_root, resolve_output_dir, resolve_export_dir except ImportError: # Fallback: try to import from parent directory parent_backend = backend_path.parent / "backend" @@ -49,7 +48,6 @@ except ImportError: ) from utils.models.model_config import _pick_best_gguf, _extract_quant_label, is_audio_input_type from core.inference import get_inference_backend - from utils.paths import outputs_root, exports_root, resolve_output_dir, resolve_export_dir from models import ( CheckpointInfo, @@ -324,8 +322,8 @@ async def get_model_config( @router.get("/loras") async def scan_loras( - outputs_dir: str = Query(default=str(outputs_root()), description="Directory to scan for LoRA adapters"), - exports_dir: str = Query(default=str(exports_root()), description="Directory to scan for exported models"), + outputs_dir: str = Query(default="./outputs", description="Directory to scan for LoRA adapters"), + exports_dir: str = Query(default="./exports", description="Directory to scan for exported models"), current_subject: str = Depends(get_current_subject), ): """ @@ -335,12 +333,10 @@ async def scan_loras( (from exports_dir) in a single list, distinguished by source field. """ try: - resolved_outputs_dir = str(resolve_output_dir(outputs_dir)) - resolved_exports_dir = str(resolve_export_dir(exports_dir)) lora_list = [] # Scan training outputs - trained_loras = scan_trained_loras(outputs_dir=resolved_outputs_dir) + trained_loras = scan_trained_loras(outputs_dir=outputs_dir) for display_name, adapter_path in trained_loras: base_model = get_base_model_from_lora(adapter_path) lora_list.append(LoRAInfo( @@ -351,7 +347,7 @@ async def scan_loras( )) # Scan exported models (merged, LoRA, base — skips GGUF) - exported = scan_exported_models(exports_dir=resolved_exports_dir) + exported = scan_exported_models(exports_dir=exports_dir) for display_name, model_path, export_type, base_model in exported: lora_list.append(LoRAInfo( display_name=display_name, @@ -363,7 +359,7 @@ async def scan_loras( return LoRAScanResponse( loras=lora_list, - outputs_dir=resolved_outputs_dir + outputs_dir=outputs_dir ) except Exception as e: @@ -481,7 +477,7 @@ async def get_gguf_variants( @router.get("/checkpoints", response_model=CheckpointListResponse) async def list_checkpoints( outputs_dir: str = Query( - default=str(outputs_root()), + default="./outputs", description="Directory to scan for checkpoints", ), current_subject: str = Depends(get_current_subject), @@ -492,8 +488,7 @@ async def list_checkpoints( Scans the outputs folder for training runs and their checkpoints. """ try: - resolved_outputs_dir = str(resolve_output_dir(outputs_dir)) - raw_models = scan_checkpoints(outputs_dir=resolved_outputs_dir) + raw_models = scan_checkpoints(outputs_dir=outputs_dir) models = [ ModelCheckpoints( @@ -510,7 +505,7 @@ async def list_checkpoints( ] return CheckpointListResponse( - outputs_dir=resolved_outputs_dir, + outputs_dir=outputs_dir, models=models, ) except Exception as e: diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py index 45fbff0172..3deabb5253 100644 --- a/studio/backend/routes/training.py +++ b/studio/backend/routes/training.py @@ -23,7 +23,6 @@ if str(backend_path) not in sys.path: try: from core.training import get_training_backend from utils.models.model_config import load_model_defaults - from utils.paths import resolve_dataset_path except ImportError: # Fallback: try to import from parent directory parent_backend = backend_path.parent / "backend" @@ -31,7 +30,6 @@ except ImportError: sys.path.insert(0, str(parent_backend)) from core.training import get_training_backend from utils.models.model_config import load_model_defaults - from utils.paths import resolve_dataset_path # Auth from auth.authentication import get_current_subject @@ -119,8 +117,24 @@ async def start_training( if request.local_datasets: validated_datasets = [] missing_datasets = [] + # Get the backend directory (where this file is located) + backend_dir = Path(__file__).parent.parent + assets_datasets_dir = backend_dir / "assets" / "datasets" + for dataset_path in request.local_datasets: - dataset_file = resolve_dataset_path(dataset_path) + dataset_file = Path(dataset_path) + + # If not absolute, try multiple locations + if not dataset_file.is_absolute(): + # First try: relative to current working directory + candidate = Path.cwd() / dataset_path + if not candidate.exists(): + # Second try: relative to assets/datasets folder + candidate = assets_datasets_dir / dataset_path + if not candidate.exists(): + # Third try: just the filename in assets/datasets + candidate = assets_datasets_dir / dataset_file.name + dataset_file = candidate if not dataset_file.exists(): missing_datasets.append( diff --git a/studio/backend/utils/models/checkpoints.py b/studio/backend/utils/models/checkpoints.py index c1cc0d0b78..c601cf64db 100644 --- a/studio/backend/utils/models/checkpoints.py +++ b/studio/backend/utils/models/checkpoints.py @@ -8,7 +8,6 @@ import json import logging from pathlib import Path from typing import List, Optional, Tuple -from utils.paths import outputs_root, resolve_output_dir logger = logging.getLogger(__name__) @@ -34,7 +33,7 @@ def _read_checkpoint_loss(checkpoint_path: Path) -> Optional[float]: def scan_checkpoints( - outputs_dir: str = str(outputs_root()), + outputs_dir: str = "./outputs", ) -> List[Tuple[str, List[Tuple[str, str, Optional[float]]], dict]]: """ Scan outputs folder for training runs and their checkpoints. @@ -46,7 +45,7 @@ def scan_checkpoints( set to the loss of the last (highest-step) intermediate checkpoint. """ models = [] - outputs_path = resolve_output_dir(outputs_dir) + outputs_path = Path(outputs_dir) if not outputs_path.exists(): logger.warning(f"Outputs directory not found: {outputs_dir}") diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 472f994e2e..8724f4b00b 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -7,15 +7,7 @@ Model and LoRA configuration handling from transformers import AutoConfig from dataclasses import dataclass from typing import Optional, Dict, Any -from utils.paths import ( - normalize_path, - is_local_path, - is_model_cached, - outputs_root, - exports_root, - resolve_output_dir, - resolve_export_dir, -) +from utils.paths import normalize_path, is_local_path, is_model_cached from utils.utils import without_hf_auth import logging import os @@ -902,7 +894,7 @@ def download_gguf_file( return local_path -def scan_trained_loras(outputs_dir: str = str(outputs_root())) -> List[Tuple[str, str]]: +def scan_trained_loras(outputs_dir: str = "./outputs") -> List[Tuple[str, str]]: """ Scan outputs folder for trained LoRA adapters. @@ -916,7 +908,7 @@ def scan_trained_loras(outputs_dir: str = str(outputs_root())) -> List[Tuple[str ] """ trained_loras = [] - outputs_path = resolve_output_dir(outputs_dir) + outputs_path = Path(outputs_dir) if not outputs_path.exists(): logger.warning(f"Outputs directory not found: {outputs_dir}") @@ -945,7 +937,7 @@ def scan_trained_loras(outputs_dir: str = str(outputs_root())) -> List[Tuple[str logger.error(f"Error scanning outputs folder: {e}") return [] -def scan_exported_models(exports_dir: str = str(exports_root())) -> List[Tuple[str, str, str, Optional[str]]]: +def scan_exported_models(exports_dir: str = "./exports") -> List[Tuple[str, str, str, Optional[str]]]: """ Scan exports folder for exported models (merged, LoRA, GGUF). @@ -958,7 +950,7 @@ def scan_exported_models(exports_dir: str = str(exports_root())) -> List[Tuple[s export_type: "lora" | "merged" | "gguf" """ results = [] - exports_path = resolve_export_dir(exports_dir) + exports_path = Path(exports_dir) if not exports_path.exists(): return results @@ -1046,7 +1038,7 @@ def scan_exported_models(exports_dir: str = str(exports_root())) -> List[Tuple[s # Fallback: read base model from the original training run's # adapter_config.json in ./outputs/{run_name}/ if not base_model: - outputs_adapter_cfg = resolve_output_dir(run_dir.name) / "adapter_config.json" + outputs_adapter_cfg = Path("./outputs") / run_dir.name / "adapter_config.json" try: if outputs_adapter_cfg.exists(): cfg = json.loads(outputs_adapter_cfg.read_text()) diff --git a/studio/backend/utils/paths/__init__.py b/studio/backend/utils/paths/__init__.py index b13d949924..d196f8ce14 100644 --- a/studio/backend/utils/paths/__init__.py +++ b/studio/backend/utils/paths/__init__.py @@ -5,52 +5,10 @@ Path utilities for model and dataset handling """ from .path_utils import normalize_path, is_local_path, is_model_cached, get_cache_path -from .storage_roots import ( - studio_root, - assets_root, - datasets_root, - dataset_uploads_root, - recipe_datasets_root, - outputs_root, - exports_root, - auth_root, - auth_db_path, - tmp_root, - seed_uploads_root, - unstructured_seed_cache_root, - oxc_validator_tmp_root, - tensorboard_root, - ensure_dir, - resolve_under_root, - resolve_output_dir, - resolve_export_dir, - resolve_tensorboard_dir, - resolve_dataset_path, -) __all__ = [ 'normalize_path', 'is_local_path', 'is_model_cached', 'get_cache_path', - 'studio_root', - 'assets_root', - 'datasets_root', - 'dataset_uploads_root', - 'recipe_datasets_root', - 'outputs_root', - 'exports_root', - 'auth_root', - 'auth_db_path', - 'tmp_root', - 'seed_uploads_root', - 'unstructured_seed_cache_root', - 'oxc_validator_tmp_root', - 'tensorboard_root', - 'ensure_dir', - 'resolve_under_root', - 'resolve_output_dir', - 'resolve_export_dir', - 'resolve_tensorboard_dir', - 'resolve_dataset_path', ] diff --git a/studio/backend/utils/paths/storage_roots.py b/studio/backend/utils/paths/storage_roots.py deleted file mode 100644 index a005d604a8..0000000000 --- a/studio/backend/utils/paths/storage_roots.py +++ /dev/null @@ -1,143 +0,0 @@ -from __future__ import annotations - -from pathlib import Path -import tempfile - - -def studio_root() -> Path: - return Path.home() / ".unsloth" / "studio" - - -def assets_root() -> Path: - return studio_root() / "assets" - - -def datasets_root() -> Path: - return assets_root() / "datasets" - - -def dataset_uploads_root() -> Path: - return datasets_root() / "uploads" - - -def recipe_datasets_root() -> Path: - return datasets_root() / "recipes" - - -def outputs_root() -> Path: - return studio_root() / "outputs" - - -def exports_root() -> Path: - return studio_root() / "exports" - - -def auth_root() -> Path: - return studio_root() / "auth" - - -def auth_db_path() -> Path: - return auth_root() / "auth.db" - - -def tmp_root() -> Path: - return Path(tempfile.gettempdir()) / "unsloth-studio" - - -def seed_uploads_root() -> Path: - return tmp_root() / "seed-uploads" - - -def unstructured_seed_cache_root() -> Path: - return tmp_root() / "unstructured-seed-cache" - - -def oxc_validator_tmp_root() -> Path: - return tmp_root() / "oxc-validator" - - -def tensorboard_root() -> Path: - return studio_root() / "runs" - - -def ensure_dir(path: Path) -> Path: - path.mkdir(parents=True, exist_ok=True) - return path - - -def _clean_relative_path(path_value: str, *, strip_prefixes: tuple[str, ...] = ()) -> Path: - path = Path(path_value).expanduser() - parts = [part for part in path.parts if part not in ("", ".")] - while parts and parts[0] in strip_prefixes: - parts = parts[1:] - return Path(*parts) if parts else Path() - - -def resolve_under_root( - path_value: str | None, - *, - root: Path, - strip_prefixes: tuple[str, ...] = (), -) -> Path: - if not path_value or not str(path_value).strip(): - return root - - path = Path(str(path_value).strip()).expanduser() - if path.is_absolute(): - return path - - cleaned = _clean_relative_path(str(path), strip_prefixes=strip_prefixes) - return root / cleaned - - -def resolve_output_dir(path_value: str | None = None) -> Path: - return resolve_under_root( - path_value, - root=outputs_root(), - strip_prefixes=("outputs",), - ) - - -def resolve_export_dir(path_value: str | None = None) -> Path: - return resolve_under_root( - path_value, - root=exports_root(), - strip_prefixes=("exports",), - ) - - -def resolve_tensorboard_dir(path_value: str | None = None) -> Path: - return resolve_under_root( - path_value, - root=tensorboard_root(), - strip_prefixes=("runs", "tensorboard"), - ) - - -def resolve_dataset_path(path_value: str) -> Path: - path = Path(path_value).expanduser() - if path.is_absolute(): - return path - - parts = [part for part in Path(path_value).parts if part not in ("", ".")] - if parts[:2] == ["assets", "datasets"]: - parts = parts[2:] - if parts and parts[0] == "uploads": - cleaned = Path(*parts[1:]) if len(parts) > 1 else Path() - return dataset_uploads_root() / cleaned - if parts and parts[0] == "recipes": - cleaned = Path(*parts[1:]) if len(parts) > 1 else Path() - return recipe_datasets_root() / cleaned - - cleaned = Path(*parts) if parts else Path() - candidates = [ - dataset_uploads_root() / cleaned, - recipe_datasets_root() / cleaned, - datasets_root() / cleaned, - dataset_uploads_root() / cleaned.name, - recipe_datasets_root() / cleaned.name, - ] - for candidate in candidates: - if candidate.exists(): - return candidate - return candidates[0] diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts index 379ee5a8c9..c73b60d440 100644 --- a/studio/frontend/src/features/chat/api/chat-api.ts +++ b/studio/frontend/src/features/chat/api/chat-api.ts @@ -49,11 +49,9 @@ export async function listModels(): Promise { return parseJsonOrThrow(response); } -export async function listLoras(outputsDir?: string): Promise { - const query = outputsDir - ? `?${new URLSearchParams({ outputs_dir: outputsDir }).toString()}` - : ""; - const response = await authFetch(`/api/models/loras${query}`); +export async function listLoras(outputsDir = "./outputs"): Promise { + const query = new URLSearchParams({ outputs_dir: outputsDir }).toString(); + const response = await authFetch(`/api/models/loras?${query}`); return parseJsonOrThrow(response); } diff --git a/studio/frontend/src/features/export/export-page.tsx b/studio/frontend/src/features/export/export-page.tsx index c9bf251520..05d697bd5e 100644 --- a/studio/frontend/src/features/export/export-page.tsx +++ b/studio/frontend/src/features/export/export-page.tsx @@ -162,8 +162,8 @@ export function ExportPage() { // For other formats, nest under training-run/checkpoint const saveDir = exportMethod === "gguf" - ? `${baseModelName.split("/").pop() ?? selectedModelIdx ?? "model"}-finetune-gguf` - : `${selectedModelIdx ?? "model"}/${checkpoint}`; + ? `./exports/${(baseModelName.split("/").pop() ?? selectedModelIdx ?? "model")}-finetune-gguf` + : `./exports/${selectedModelIdx ?? "model"}/${checkpoint}`; const pushToHub = destination === "hub"; const repoId = pushToHub && hfUsername && modelName ? `${hfUsername}/${modelName}`