Revert "Merge pull request #347 from unslothai/feature/studio-storage-roots"
This reverts commit6b43e33ff1, reversing changes made to9edadaf21f.
This commit is contained in:
Roland Tannous
parent
6b43e33ff1
commit
daa50d0756
23 changed files with 104 additions and 356 deletions
|
|
@ -23,11 +23,11 @@ IS_WINDOWS = sys.platform == "win32"
|
|||
|
||||
# ── Paths ──────────────────────────────────────────────────────────────
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
REQ_ROOT = SCRIPT_DIR / "backend" / "requirements"
|
||||
REQ_ROOT = SCRIPT_DIR / "studio" / "backend" / "requirements"
|
||||
SINGLE_ENV = REQ_ROOT / "single-env"
|
||||
CONSTRAINTS = SINGLE_ENV / "constraints.txt"
|
||||
LOCAL_DD_UNSTRUCTURED_PLUGIN = (
|
||||
SCRIPT_DIR / "backend" / "plugins" / "data-designer-unstructured-seed"
|
||||
SCRIPT_DIR / "studio" / "backend" / "plugins" / "data-designer-unstructured-seed"
|
||||
)
|
||||
|
||||
# ── Color support ──────────────────────────────────────────────────────
|
||||
|
|
@ -17,9 +17,9 @@ $ErrorActionPreference = "Stop"
|
|||
$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$PackageDir = Split-Path -Parent $ScriptDir
|
||||
|
||||
# Detect if running from pip install (no frontend/ dir in studio)
|
||||
$FrontendDir = Join-Path $ScriptDir "frontend"
|
||||
$OxcValidatorDir = Join-Path $ScriptDir "backend\core\data_recipe\oxc-validator"
|
||||
# Detect if running from pip install (no studio/frontend/ dir in repo)
|
||||
$FrontendDir = Join-Path $ScriptDir "studio\frontend"
|
||||
$OxcValidatorDir = Join-Path $ScriptDir "studio\backend\core\data_recipe\oxc-validator"
|
||||
$IsPipInstall = -not (Test-Path $FrontendDir)
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
|
|
@ -587,7 +587,7 @@ if ($IsPipInstall) {
|
|||
Pop-Location
|
||||
$ErrorActionPreference = $prevEAP_npm
|
||||
Write-Host "[ERROR] npm install failed (exit code $LASTEXITCODE)" -ForegroundColor Red
|
||||
Write-Host " Try running 'npm install' manually in frontend/ to see errors" -ForegroundColor Yellow
|
||||
Write-Host " Try running 'npm install' manually in studio/frontend/ to see errors" -ForegroundColor Yellow
|
||||
exit 1
|
||||
}
|
||||
npm run build 2>&1 | Out-Null
|
||||
|
|
@ -599,7 +599,7 @@ if ($IsPipInstall) {
|
|||
}
|
||||
Pop-Location
|
||||
$ErrorActionPreference = $prevEAP_npm
|
||||
Write-Host "[OK] Frontend built to frontend/dist" -ForegroundColor Green
|
||||
Write-Host "[OK] Frontend built to studio/frontend/dist" -ForegroundColor Green
|
||||
}
|
||||
|
||||
if (Test-Path $OxcValidatorDir) {
|
||||
|
|
@ -648,8 +648,8 @@ if (-not $PythonCmd) {
|
|||
Write-Host "[OK] Using $PythonCmd ($(& $PythonCmd --version 2>&1))" -ForegroundColor Green
|
||||
|
||||
# Always create a .venv for isolation -- even for pip installs.
|
||||
# Created in the repo root (parent of studio/).
|
||||
$VenvDir = Join-Path (Split-Path -Parent $PSScriptRoot) ".venv"
|
||||
# Created in the current working directory (where user ran the command).
|
||||
$VenvDir = Join-Path (Get-Location) ".venv"
|
||||
if (-not (Test-Path $VenvDir)) {
|
||||
Write-Host " Creating virtual environment at $VenvDir..." -ForegroundColor Cyan
|
||||
& $PythonCmd -m venv $VenvDir
|
||||
|
|
@ -722,7 +722,7 @@ $ErrorActionPreference = $prevEAP
|
|||
# The training subprocess just prepends .venv_t5/ to sys.path — instant switch.
|
||||
Write-Host ""
|
||||
Write-Host " Pre-installing transformers 5.x for newer model support..." -ForegroundColor Cyan
|
||||
$VenvT5Dir = Join-Path (Split-Path -Parent $PSScriptRoot) ".venv_t5"
|
||||
$VenvT5Dir = Join-Path $PSScriptRoot ".venv_t5"
|
||||
if (Test-Path $VenvT5Dir) { Remove-Item -Recurse -Force $VenvT5Dir }
|
||||
New-Item -ItemType Directory -Path $VenvT5Dir -Force | Out-Null
|
||||
$prevEAP_t5 = $ErrorActionPreference
|
||||
|
|
@ -956,10 +956,10 @@ if (Test-Path $LlamaServerBin) {
|
|||
# Add shell aliases (PowerShell profile + cmd batch files)
|
||||
# ============================================
|
||||
Write-Host ""
|
||||
$RepoDir = Split-Path -Parent $PSScriptRoot
|
||||
$RepoDir = $PSScriptRoot
|
||||
$VenvPython = Join-Path $RepoDir ".venv\Scripts\python.exe"
|
||||
$CliScript = Join-Path $RepoDir "cli.py"
|
||||
$FrontendDist = Join-Path $PSScriptRoot "frontend\dist"
|
||||
$FrontendDist = Join-Path $RepoDir "studio\frontend\dist"
|
||||
$AliasAdded = $false
|
||||
|
||||
# --- PowerShell profile: add functions ---
|
||||
|
|
@ -5,7 +5,6 @@
|
|||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
# ── Helper: run command quietly, show output only on failure ──
|
||||
run_quiet() {
|
||||
|
|
@ -29,9 +28,9 @@ echo "║ Unsloth Studio Setup Script ║"
|
|||
echo "╚══════════════════════════════════════╝"
|
||||
|
||||
# ── Clean up stale Unsloth compiled caches ──
|
||||
rm -rf "$REPO_ROOT/unsloth_compiled_cache"
|
||||
rm -rf "$SCRIPT_DIR/backend/unsloth_compiled_cache"
|
||||
rm -rf "$SCRIPT_DIR/tmp/unsloth_compiled_cache"
|
||||
rm -rf "$SCRIPT_DIR/unsloth_compiled_cache"
|
||||
rm -rf "$SCRIPT_DIR/studio/backend/unsloth_compiled_cache"
|
||||
rm -rf "$SCRIPT_DIR/studio/tmp/unsloth_compiled_cache"
|
||||
|
||||
# ── Detect Colab (like unsloth does) ──
|
||||
IS_COLAB=false
|
||||
|
|
@ -100,13 +99,13 @@ echo "✅ Node $(node -v) | npm $(npm -v)"
|
|||
# ── 5. Build frontend ──
|
||||
echo ""
|
||||
echo "Building frontend..."
|
||||
cd "$SCRIPT_DIR/frontend"
|
||||
cd "$SCRIPT_DIR/studio/frontend"
|
||||
run_quiet "npm install" npm install
|
||||
run_quiet "npm run build" npm run build
|
||||
cd "$SCRIPT_DIR/backend/core/data_recipe/oxc-validator"
|
||||
cd "$SCRIPT_DIR/studio/backend/core/data_recipe/oxc-validator"
|
||||
run_quiet "npm install (oxc validator runtime)" npm install
|
||||
cd "$SCRIPT_DIR"
|
||||
echo "✅ Frontend built to frontend/dist"
|
||||
echo "✅ Frontend built to studio/frontend/dist"
|
||||
|
||||
# ── 6. Python venv + deps ──
|
||||
echo ""
|
||||
|
|
@ -169,7 +168,7 @@ fi
|
|||
BEST_VER=$("$BEST_PY" --version 2>&1 | awk '{print $2}')
|
||||
echo "✅ Using $BEST_PY ($BEST_VER) — compatible (3.${MIN_PY_MINOR}.x – 3.${MAX_PY_MINOR}.x)"
|
||||
|
||||
REQ_ROOT="$SCRIPT_DIR/backend/requirements"
|
||||
REQ_ROOT="$SCRIPT_DIR/studio/backend/requirements"
|
||||
SINGLE_ENV_CONSTRAINTS="$REQ_ROOT/single-env/constraints.txt"
|
||||
SINGLE_ENV_DATA_DESIGNER="$REQ_ROOT/single-env/data-designer.txt"
|
||||
SINGLE_ENV_DATA_DESIGNER_DEPS="$REQ_ROOT/single-env/data-designer-deps.txt"
|
||||
|
|
@ -184,13 +183,11 @@ if [ "$IS_COLAB" = true ]; then
|
|||
install_python_stack
|
||||
else
|
||||
# Local: create venv (always start fresh to preserve correct install order)
|
||||
cd "$REPO_ROOT"
|
||||
rm -rf .venv
|
||||
rm -rf .venv_overlay # Remove legacy overlay (no longer used)
|
||||
rm -rf .venv_t5 # Will be rebuilt below
|
||||
"$BEST_PY" -m venv .venv
|
||||
source .venv/bin/activate
|
||||
cd "$SCRIPT_DIR"
|
||||
install_python_stack
|
||||
|
||||
# ── 6b. Pre-install transformers 5.x into .venv_t5/ ──
|
||||
|
|
@ -199,7 +196,7 @@ else
|
|||
# The training subprocess just prepends .venv_t5/ to sys.path — instant switch.
|
||||
echo ""
|
||||
echo " Pre-installing transformers 5.x for newer model support..."
|
||||
VENV_T5_DIR="$REPO_ROOT/.venv_t5"
|
||||
VENV_T5_DIR="$SCRIPT_DIR/.venv_t5"
|
||||
mkdir -p "$VENV_T5_DIR"
|
||||
run_quiet "pip install transformers 5.x" pip install --target "$VENV_T5_DIR" --no-deps "transformers==5.2.0"
|
||||
run_quiet "pip install huggingface_hub for t5" pip install --target "$VENV_T5_DIR" --no-deps "huggingface_hub==1.3.0"
|
||||
|
|
@ -309,30 +306,32 @@ rm -rf "$LLAMA_CPP_DIR"
|
|||
# This alias hardcodes the venv python path so users don't need to activate.
|
||||
if [ "$IS_COLAB" = false ]; then
|
||||
echo ""
|
||||
REPO_DIR="$REPO_ROOT"
|
||||
REPO_DIR="$SCRIPT_DIR"
|
||||
|
||||
# Detect the user's default shell and pick the right rc file
|
||||
USER_SHELL="$(basename "${SHELL:-/bin/bash}")"
|
||||
case "$USER_SHELL" in
|
||||
zsh)
|
||||
SHELL_RC="$HOME/.zshrc"
|
||||
ALIAS_BLOCK="alias unsloth-studio='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist'
|
||||
alias unsloth-ui='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist'"
|
||||
ALIAS_BLOCK="alias unsloth-studio='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist'
|
||||
alias unsloth-ui='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist'"
|
||||
;;
|
||||
fish)
|
||||
SHELL_RC="$HOME/.config/fish/config.fish"
|
||||
ALIAS_BLOCK="alias unsloth-studio '${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist'
|
||||
alias unsloth-ui '${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist'"
|
||||
# fish uses 'abbr' or 'function'; a simple alias works via 'alias' in config.fish
|
||||
ALIAS_BLOCK="alias unsloth-studio '${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist'
|
||||
alias unsloth-ui '${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist'"
|
||||
;;
|
||||
ksh)
|
||||
SHELL_RC="$HOME/.kshrc"
|
||||
ALIAS_BLOCK="alias unsloth-studio='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist'
|
||||
alias unsloth-ui='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist'"
|
||||
ALIAS_BLOCK="alias unsloth-studio='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist'
|
||||
alias unsloth-ui='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist'"
|
||||
;;
|
||||
*)
|
||||
# Default to bash for bash and any other POSIX-compatible shell
|
||||
SHELL_RC="$HOME/.bashrc"
|
||||
ALIAS_BLOCK="alias unsloth-studio='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist'
|
||||
alias unsloth-ui='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${SCRIPT_DIR}/frontend/dist'"
|
||||
ALIAS_BLOCK="alias unsloth-studio='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist'
|
||||
alias unsloth-ui='${REPO_DIR}/.venv/bin/python ${REPO_DIR}/cli.py studio -f ${REPO_DIR}/studio/frontend/dist'"
|
||||
;;
|
||||
esac
|
||||
|
||||
|
|
@ -7,11 +7,10 @@ SQLite storage for authentication data (user credentials + JWT secret).
|
|||
import hashlib
|
||||
import sqlite3
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from utils.paths import auth_db_path, ensure_dir
|
||||
|
||||
DB_PATH = auth_db_path()
|
||||
DB_PATH = Path(__file__).parent / "auth.db"
|
||||
|
||||
|
||||
def _hash_token(token: str) -> str:
|
||||
|
|
@ -21,7 +20,6 @@ def _hash_token(token: str) -> str:
|
|||
|
||||
def get_connection() -> sqlite3.Connection:
|
||||
"""Get a connection to the auth database, creating tables if needed."""
|
||||
ensure_dir(DB_PATH.parent)
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute(
|
||||
|
|
@ -258,3 +256,4 @@ def revoke_user_refresh_tokens(username: str) -> None:
|
|||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
|
|
|||
|
|
@ -16,9 +16,9 @@ from typing import Any
|
|||
from ..jsonable import to_jsonable, to_preview_jsonable
|
||||
from .constants import EVENT_JOB_COMPLETED, EVENT_JOB_ERROR, EVENT_JOB_STARTED
|
||||
from ..service import build_config_builder, create_data_designer
|
||||
from utils.paths import ensure_dir, recipe_datasets_root
|
||||
|
||||
_ARTIFACT_ROOT = recipe_datasets_root()
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parents[5]
|
||||
_ARTIFACT_ROOT = _PROJECT_ROOT / "studio" / "backend" / "assets" / "datasets"
|
||||
|
||||
|
||||
class _QueueLogHandler(logging.Handler):
|
||||
|
|
@ -88,7 +88,7 @@ def run_job_process(
|
|||
artifact_root=_ARTIFACT_ROOT,
|
||||
)
|
||||
merge_batches = bool(run.get("merge_batches"))
|
||||
ensure_dir(_ARTIFACT_ROOT)
|
||||
_ARTIFACT_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
run_config_raw = run.get("run_config") or {}
|
||||
|
||||
builder = build_config_builder(recipe)
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ from __future__ import annotations
|
|||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -13,8 +12,6 @@ from functools import lru_cache
|
|||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from utils.paths import ensure_dir, oxc_validator_tmp_root
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OXC_VALIDATION_FN_MARKER = "unsloth_oxc_validator"
|
||||
|
|
@ -238,12 +235,6 @@ def _run_oxc_batch(
|
|||
"codes": code_values,
|
||||
}
|
||||
try:
|
||||
tmp_dir = ensure_dir(oxc_validator_tmp_root())
|
||||
env = dict(os.environ)
|
||||
tmp_dir_str = str(tmp_dir)
|
||||
env["TMPDIR"] = tmp_dir_str
|
||||
env["TMP"] = tmp_dir_str
|
||||
env["TEMP"] = tmp_dir_str
|
||||
proc = subprocess.run(
|
||||
["node", str(_OXC_RUNNER_PATH)],
|
||||
cwd=str(_OXC_TOOL_DIR),
|
||||
|
|
@ -251,7 +242,6 @@ def _run_oxc_batch(
|
|||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
env=env,
|
||||
)
|
||||
except (OSError, ValueError) as exc:
|
||||
logger.warning("OXC subprocess launch failed: %s", exc)
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ from utils.hardware import clear_gpu_cache
|
|||
|
||||
from utils.models import is_vision_model, get_base_model_from_lora
|
||||
from utils.models.model_config import detect_audio_type
|
||||
from utils.paths import ensure_dir, outputs_root, resolve_export_dir, resolve_output_dir
|
||||
from core.inference import get_inference_backend
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -129,7 +128,7 @@ class ExportBackend:
|
|||
logger.error(f"Error during memory cleanup: {e}")
|
||||
return False
|
||||
|
||||
def scan_checkpoints(self, outputs_dir: str = str(outputs_root())) -> List[Tuple[str, List[Tuple[str, str]]]]:
|
||||
def scan_checkpoints(self, outputs_dir: str = "./outputs") -> List[Tuple[str, List[Tuple[str, str]]]]:
|
||||
"""
|
||||
Scan outputs folder for training runs and their checkpoints.
|
||||
|
||||
|
|
@ -326,9 +325,8 @@ class ExportBackend:
|
|||
|
||||
# Save locally if requested
|
||||
if save_directory:
|
||||
save_directory = str(resolve_export_dir(save_directory))
|
||||
logger.info(f"Saving merged model locally to: {save_directory}")
|
||||
ensure_dir(Path(save_directory))
|
||||
os.makedirs(save_directory, exist_ok=True)
|
||||
|
||||
self.current_model.save_pretrained_merged(
|
||||
save_directory,
|
||||
|
|
@ -388,9 +386,8 @@ class ExportBackend:
|
|||
try:
|
||||
# Save locally if requested
|
||||
if save_directory:
|
||||
save_directory = str(resolve_export_dir(save_directory))
|
||||
logger.info(f"Saving base model locally to: {save_directory}")
|
||||
ensure_dir(Path(save_directory))
|
||||
os.makedirs(save_directory, exist_ok=True)
|
||||
|
||||
self.current_model.save_pretrained(save_directory)
|
||||
self.current_tokenizer.save_pretrained(save_directory)
|
||||
|
|
@ -478,7 +475,6 @@ class ExportBackend:
|
|||
|
||||
# Save locally if requested
|
||||
if save_directory:
|
||||
save_directory = str(resolve_export_dir(save_directory))
|
||||
# Resolve to absolute path so unsloth's relative-path internals
|
||||
# (check_llama_cpp, use_local_gguf, _download_convert_hf_to_gguf)
|
||||
# all resolve against the repo root cwd, NOT the export directory.
|
||||
|
|
@ -486,7 +482,7 @@ class ExportBackend:
|
|||
logger.info(f"Saving GGUF model locally to: {abs_save_dir}")
|
||||
|
||||
# Create the directory if it doesn't exist
|
||||
ensure_dir(Path(abs_save_dir))
|
||||
os.makedirs(abs_save_dir, exist_ok=True)
|
||||
|
||||
# On WSL, patch out sudo check before llama.cpp build
|
||||
_apply_wsl_sudo_patch()
|
||||
|
|
@ -586,9 +582,8 @@ class ExportBackend:
|
|||
try:
|
||||
# Save locally if requested
|
||||
if save_directory:
|
||||
save_directory = str(resolve_export_dir(save_directory))
|
||||
logger.info(f"Saving LoRA adapter locally to: {save_directory}")
|
||||
ensure_dir(Path(save_directory))
|
||||
os.makedirs(save_directory, exist_ok=True)
|
||||
|
||||
self.current_model.save_pretrained(save_directory)
|
||||
self.current_tokenizer.save_pretrained(save_directory)
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import threading
|
|||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, List, Optional, Tuple
|
||||
from utils.paths import outputs_root
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -379,7 +378,7 @@ class ExportOrchestrator:
|
|||
return success
|
||||
|
||||
def scan_checkpoints(
|
||||
self, outputs_dir: str = str(outputs_root())
|
||||
self, outputs_dir: str = "./outputs"
|
||||
) -> List[Tuple[str, list]]:
|
||||
"""Scan for checkpoints — no ML imports needed, runs locally."""
|
||||
from utils.models.checkpoints import scan_checkpoints
|
||||
|
|
|
|||
|
|
@ -30,21 +30,13 @@ from datasets import Dataset, load_dataset
|
|||
from utils.models import is_vision_model, detect_audio_type
|
||||
from utils.datasets import format_and_template_dataset
|
||||
from utils.datasets import MODEL_TO_TEMPLATE_MAPPER, TEMPLATE_TO_RESPONSES_MAPPER
|
||||
from utils.paths import ensure_dir, resolve_dataset_path, resolve_output_dir, resolve_tensorboard_dir
|
||||
from trl import SFTTrainer, SFTConfig
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
||||
def _build_report_targets(training_args) -> list[str] | str:
|
||||
report_to: list[str] = []
|
||||
if training_args.get("enable_wandb", False):
|
||||
report_to.append("wandb")
|
||||
if training_args.get("enable_tensorboard", False):
|
||||
report_to.append("tensorboard")
|
||||
return report_to or "none"
|
||||
_BACKEND_ROOT = Path(__file__).resolve().parents[2]
|
||||
_ASSETS_DATASETS_ROOT = _BACKEND_ROOT / "assets" / "datasets"
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -213,14 +205,9 @@ class UnslothTrainer:
|
|||
"lr_scheduler_type": lr_scheduler_type,
|
||||
"seed": random_seed,
|
||||
"output_dir": output_dir,
|
||||
"report_to": _build_report_targets(training_args),
|
||||
"report_to": ["wandb"] if training_args.get('enable_wandb', False) else "none",
|
||||
}
|
||||
|
||||
if training_args.get("enable_tensorboard", False):
|
||||
config["logging_dir"] = str(
|
||||
resolve_tensorboard_dir(training_args.get("tensorboard_dir"))
|
||||
)
|
||||
|
||||
# max_steps vs epochs
|
||||
if max_steps_val and max_steps_val > 0:
|
||||
config["max_steps"] = max_steps_val
|
||||
|
|
@ -1542,7 +1529,6 @@ class UnslothTrainer:
|
|||
import numpy as np
|
||||
import soundfile as sf
|
||||
from datasets import Dataset as HFDataset
|
||||
from utils.paths import ensure_dir, tmp_root
|
||||
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
|
||||
|
|
@ -1639,18 +1625,10 @@ class UnslothTrainer:
|
|||
audio_bytes = buf.getvalue()
|
||||
|
||||
# 1. Get word timings from Whisper
|
||||
with tempfile.NamedTemporaryFile(
|
||||
suffix=".wav",
|
||||
delete=False,
|
||||
dir=str(ensure_dir(tmp_root())),
|
||||
) as tmp:
|
||||
with tempfile.NamedTemporaryFile(suffix=".wav", delete=True) as tmp:
|
||||
tmp.write(audio_bytes)
|
||||
tmp.flush()
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
whisper_result = whisper_model.transcribe(tmp_path, word_timestamps=True)
|
||||
finally:
|
||||
Path(tmp_path).unlink(missing_ok=True)
|
||||
whisper_result = whisper_model.transcribe(tmp.name, word_timestamps=True)
|
||||
|
||||
normalized_transcript = text_normalizations(text)
|
||||
words_with_timings = []
|
||||
|
|
@ -1838,7 +1816,7 @@ class UnslothTrainer:
|
|||
file_path = dataset_file
|
||||
else:
|
||||
# Fallback: try relative to assets/datasets
|
||||
file_path = str(resolve_dataset_path(dataset_file))
|
||||
file_path = str(_ASSETS_DATASETS_ROOT / dataset_file)
|
||||
|
||||
file_path_obj = Path(file_path)
|
||||
|
||||
|
|
@ -2091,7 +2069,7 @@ class UnslothTrainer:
|
|||
dataset: Dataset,
|
||||
eval_dataset: Dataset = None,
|
||||
eval_steps: float = 0.00,
|
||||
output_dir: str | None = None,
|
||||
output_dir: str = "./outputs",
|
||||
num_epochs: int = 3,
|
||||
learning_rate: float = 5e-5,
|
||||
batch_size: int = 2,
|
||||
|
|
@ -2108,7 +2086,7 @@ class UnslothTrainer:
|
|||
wandb_project: str = "unsloth-training",
|
||||
wandb_token: str = None,
|
||||
enable_tensorboard: bool = False,
|
||||
tensorboard_dir: str | None = None,
|
||||
tensorboard_dir: str = "runs",
|
||||
**kwargs) -> bool:
|
||||
"""Start training in a separate thread"""
|
||||
|
||||
|
|
@ -2196,8 +2174,8 @@ class UnslothTrainer:
|
|||
wandb.init(project=training_args.get('wandb_project', 'unsloth-training'))
|
||||
|
||||
# Create output directory
|
||||
output_dir = str(resolve_output_dir(training_args.get("output_dir")))
|
||||
ensure_dir(Path(output_dir))
|
||||
output_dir = training_args.get('output_dir', './outputs')
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
# ========== AUDIO TRAINER BRANCH ==========
|
||||
if self._audio_type == 'csm':
|
||||
|
|
@ -2407,15 +2385,11 @@ class UnslothTrainer:
|
|||
"weight_decay": training_args.get('weight_decay', 0.01),
|
||||
"seed": training_args.get('random_seed', 3407),
|
||||
"output_dir": output_dir,
|
||||
"report_to": _build_report_targets(training_args),
|
||||
"report_to": ["wandb"] if training_args.get('enable_wandb', False) else "none",
|
||||
"include_num_input_tokens_seen": True, # Enable token counting
|
||||
"dataset_num_proc": 1 if (self.is_audio or self.is_audio_vlm or self._cuda_audio_used) else safe_num_proc(max(1, os.cpu_count() // 4)),
|
||||
"max_seq_length": training_args.get('max_seq_length', 2048),
|
||||
}
|
||||
if training_args.get("enable_tensorboard", False):
|
||||
config_args["logging_dir"] = str(
|
||||
resolve_tensorboard_dir(training_args.get("tensorboard_dir"))
|
||||
)
|
||||
print(f"[DEBUG] dataset_num_proc={config_args['dataset_num_proc']} (is_audio={self.is_audio}, is_audio_vlm={self.is_audio_vlm}, _cuda_audio_used={self._cuda_audio_used})")
|
||||
|
||||
# On Windows with transformers 5.x, disable DataLoader multiprocessing
|
||||
|
|
|
|||
|
|
@ -121,7 +121,6 @@ def run_training_process(
|
|||
sys.path.insert(0, backend_path)
|
||||
|
||||
from core.training.trainer import UnslothTrainer, TrainingProgress
|
||||
from utils.paths import ensure_dir, resolve_output_dir, resolve_tensorboard_dir
|
||||
|
||||
import transformers
|
||||
logger.info("Subprocess loaded transformers %s", transformers.__version__)
|
||||
|
|
@ -306,14 +305,7 @@ def run_training_process(
|
|||
# Generate output dir
|
||||
output_dir = config.get("output_dir")
|
||||
if not output_dir:
|
||||
output_dir = f"{model_name.replace('/', '_')}_{int(time.time())}"
|
||||
output_dir = str(resolve_output_dir(output_dir))
|
||||
ensure_dir(Path(output_dir))
|
||||
|
||||
tensorboard_dir = config.get("tensorboard_dir")
|
||||
if config.get("enable_tensorboard", False):
|
||||
tensorboard_dir = str(resolve_tensorboard_dir(tensorboard_dir))
|
||||
ensure_dir(Path(tensorboard_dir))
|
||||
output_dir = f"./outputs/{model_name.replace('/', '_')}_{int(time.time())}"
|
||||
|
||||
# Start training (directly — no inner thread, we ARE the subprocess)
|
||||
_send_status(event_queue, "Starting training...")
|
||||
|
|
@ -339,7 +331,7 @@ def run_training_process(
|
|||
wandb_project=config.get("wandb_project", "unsloth-training"),
|
||||
wandb_token=config.get("wandb_token"),
|
||||
enable_tensorboard=config.get("enable_tensorboard", False),
|
||||
tensorboard_dir=tensorboard_dir,
|
||||
tensorboard_dir=config.get("tensorboard_dir", "runs"),
|
||||
eval_dataset=eval_dataset,
|
||||
eval_steps=eval_steps,
|
||||
max_seq_length=config.get("max_seq_length", 2048),
|
||||
|
|
|
|||
|
|
@ -8,13 +8,11 @@ import re
|
|||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from utils.paths import ensure_dir, unstructured_seed_cache_root
|
||||
|
||||
DEFAULT_CHUNK_SIZE = 1200
|
||||
DEFAULT_CHUNK_OVERLAP = 200
|
||||
MAX_CHUNK_SIZE = 20000
|
||||
_MIN_BREAK_RATIO = 0.6
|
||||
_CACHE_DIR = unstructured_seed_cache_root()
|
||||
_CACHE_DIR = Path.home() / ".cache" / "unsloth" / "data-recipe" / "unstructured-seed-cache"
|
||||
|
||||
|
||||
def resolve_chunking(
|
||||
|
|
@ -87,7 +85,7 @@ def materialize_unstructured_seed_dataset(
|
|||
raise ValueError("No text found in unstructured seed source.")
|
||||
|
||||
rows = [{"chunk_text": chunk} for chunk in chunks]
|
||||
ensure_dir(_CACHE_DIR)
|
||||
_CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
import pandas as pd
|
||||
except ImportError as exc: # pragma: no cover
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ from data_designer_unstructured_seed.chunking import (
|
|||
resolve_chunking,
|
||||
)
|
||||
from core.data_recipe.jsonable import to_preview_jsonable
|
||||
from utils.paths import ensure_dir, seed_uploads_root
|
||||
|
||||
from models.data_recipe import (
|
||||
SeedInspectRequest,
|
||||
|
|
@ -32,7 +31,7 @@ DATA_EXTS = (".parquet", ".jsonl", ".json", ".csv")
|
|||
DEFAULT_SPLIT = "train"
|
||||
LOCAL_UPLOAD_EXTS = {".csv", ".json", ".jsonl"}
|
||||
UNSTRUCTURED_UPLOAD_EXTS = {".txt", ".md"}
|
||||
SEED_UPLOAD_DIR = seed_uploads_root()
|
||||
SEED_UPLOAD_DIR = Path.home() / ".cache" / "unsloth" / "data-recipe" / "seed-uploads"
|
||||
|
||||
|
||||
def _serialize_preview_value(value: Any) -> Any:
|
||||
|
|
@ -305,7 +304,7 @@ def inspect_seed_upload(payload: SeedInspectUploadRequest) -> SeedInspectRespons
|
|||
if len(file_bytes) > max_size_bytes:
|
||||
raise HTTPException(status_code=413, detail="file too large (max 50MB)")
|
||||
|
||||
ensure_dir(SEED_UPLOAD_DIR)
|
||||
SEED_UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
||||
stored_name = f"{uuid4().hex}_{filename}"
|
||||
stored_path = SEED_UPLOAD_DIR / stored_name
|
||||
stored_path.write_bytes(file_bytes)
|
||||
|
|
|
|||
|
|
@ -42,12 +42,6 @@ from models.datasets import (
|
|||
LocalDatasetsResponse,
|
||||
UploadDatasetResponse,
|
||||
)
|
||||
from utils.paths import (
|
||||
dataset_uploads_root,
|
||||
ensure_dir,
|
||||
recipe_datasets_root,
|
||||
resolve_dataset_path,
|
||||
)
|
||||
|
||||
|
||||
def _serialize_preview_value(value):
|
||||
|
|
@ -97,8 +91,9 @@ _ARCHIVE_EXTS = ('.tar', '.tar.gz', '.tgz', '.gz', '.zst', '.zip', '.txt')
|
|||
DATA_EXTS = _TABULAR_EXTS + _ARCHIVE_EXTS
|
||||
LOCAL_FILE_EXTS = ('.json', '.jsonl', '.csv', '.parquet')
|
||||
LOCAL_UPLOAD_EXTS = {".csv", ".json", ".jsonl", ".parquet"}
|
||||
LOCAL_DATASETS_ROOT = recipe_datasets_root()
|
||||
DATASET_UPLOAD_DIR = dataset_uploads_root()
|
||||
BACKEND_ROOT = Path(__file__).resolve().parents[1]
|
||||
LOCAL_DATASETS_ROOT = BACKEND_ROOT / "assets" / "datasets"
|
||||
DATASET_UPLOAD_DIR = LOCAL_DATASETS_ROOT / "uploads"
|
||||
|
||||
|
||||
def _safe_read_metadata(path: Path) -> dict | None:
|
||||
|
|
@ -282,7 +277,7 @@ async def upload_dataset(
|
|||
)
|
||||
|
||||
max_size_bytes = 512 * 1024 * 1024
|
||||
ensure_dir(DATASET_UPLOAD_DIR)
|
||||
DATASET_UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
||||
stem = Path(filename).stem
|
||||
stored_name = f"{uuid4().hex}_{stem}{ext}"
|
||||
stored_path = DATASET_UPLOAD_DIR / stored_name
|
||||
|
|
@ -338,7 +333,7 @@ def check_format(
|
|||
|
||||
logger.info(f"Checking format for dataset: {request.dataset_name}")
|
||||
|
||||
dataset_path = resolve_dataset_path(request.dataset_name)
|
||||
dataset_path = Path(request.dataset_name)
|
||||
total_rows = None
|
||||
|
||||
if dataset_path.exists():
|
||||
|
|
|
|||
|
|
@ -576,16 +576,11 @@ def _decode_audio_base64(b64: str) -> np.ndarray:
|
|||
import torchaudio
|
||||
import tempfile
|
||||
import os
|
||||
from utils.paths import ensure_dir, tmp_root
|
||||
|
||||
raw = base64.b64decode(b64)
|
||||
# torchaudio.load needs a file path or file-like object with format hint
|
||||
# Write to a temp file so torchaudio can auto-detect the format
|
||||
with tempfile.NamedTemporaryFile(
|
||||
suffix=".audio",
|
||||
delete=False,
|
||||
dir=str(ensure_dir(tmp_root())),
|
||||
) as tmp:
|
||||
with tempfile.NamedTemporaryFile(suffix=".audio", delete=False) as tmp:
|
||||
tmp.write(raw)
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -31,7 +31,6 @@ try:
|
|||
)
|
||||
from utils.models.model_config import _pick_best_gguf, _extract_quant_label, is_audio_input_type
|
||||
from core.inference import get_inference_backend
|
||||
from utils.paths import outputs_root, exports_root, resolve_output_dir, resolve_export_dir
|
||||
except ImportError:
|
||||
# Fallback: try to import from parent directory
|
||||
parent_backend = backend_path.parent / "backend"
|
||||
|
|
@ -49,7 +48,6 @@ except ImportError:
|
|||
)
|
||||
from utils.models.model_config import _pick_best_gguf, _extract_quant_label, is_audio_input_type
|
||||
from core.inference import get_inference_backend
|
||||
from utils.paths import outputs_root, exports_root, resolve_output_dir, resolve_export_dir
|
||||
|
||||
from models import (
|
||||
CheckpointInfo,
|
||||
|
|
@ -324,8 +322,8 @@ async def get_model_config(
|
|||
|
||||
@router.get("/loras")
|
||||
async def scan_loras(
|
||||
outputs_dir: str = Query(default=str(outputs_root()), description="Directory to scan for LoRA adapters"),
|
||||
exports_dir: str = Query(default=str(exports_root()), description="Directory to scan for exported models"),
|
||||
outputs_dir: str = Query(default="./outputs", description="Directory to scan for LoRA adapters"),
|
||||
exports_dir: str = Query(default="./exports", description="Directory to scan for exported models"),
|
||||
current_subject: str = Depends(get_current_subject),
|
||||
):
|
||||
"""
|
||||
|
|
@ -335,12 +333,10 @@ async def scan_loras(
|
|||
(from exports_dir) in a single list, distinguished by source field.
|
||||
"""
|
||||
try:
|
||||
resolved_outputs_dir = str(resolve_output_dir(outputs_dir))
|
||||
resolved_exports_dir = str(resolve_export_dir(exports_dir))
|
||||
lora_list = []
|
||||
|
||||
# Scan training outputs
|
||||
trained_loras = scan_trained_loras(outputs_dir=resolved_outputs_dir)
|
||||
trained_loras = scan_trained_loras(outputs_dir=outputs_dir)
|
||||
for display_name, adapter_path in trained_loras:
|
||||
base_model = get_base_model_from_lora(adapter_path)
|
||||
lora_list.append(LoRAInfo(
|
||||
|
|
@ -351,7 +347,7 @@ async def scan_loras(
|
|||
))
|
||||
|
||||
# Scan exported models (merged, LoRA, base — skips GGUF)
|
||||
exported = scan_exported_models(exports_dir=resolved_exports_dir)
|
||||
exported = scan_exported_models(exports_dir=exports_dir)
|
||||
for display_name, model_path, export_type, base_model in exported:
|
||||
lora_list.append(LoRAInfo(
|
||||
display_name=display_name,
|
||||
|
|
@ -363,7 +359,7 @@ async def scan_loras(
|
|||
|
||||
return LoRAScanResponse(
|
||||
loras=lora_list,
|
||||
outputs_dir=resolved_outputs_dir
|
||||
outputs_dir=outputs_dir
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
|
|
@ -481,7 +477,7 @@ async def get_gguf_variants(
|
|||
@router.get("/checkpoints", response_model=CheckpointListResponse)
|
||||
async def list_checkpoints(
|
||||
outputs_dir: str = Query(
|
||||
default=str(outputs_root()),
|
||||
default="./outputs",
|
||||
description="Directory to scan for checkpoints",
|
||||
),
|
||||
current_subject: str = Depends(get_current_subject),
|
||||
|
|
@ -492,8 +488,7 @@ async def list_checkpoints(
|
|||
Scans the outputs folder for training runs and their checkpoints.
|
||||
"""
|
||||
try:
|
||||
resolved_outputs_dir = str(resolve_output_dir(outputs_dir))
|
||||
raw_models = scan_checkpoints(outputs_dir=resolved_outputs_dir)
|
||||
raw_models = scan_checkpoints(outputs_dir=outputs_dir)
|
||||
|
||||
models = [
|
||||
ModelCheckpoints(
|
||||
|
|
@ -510,7 +505,7 @@ async def list_checkpoints(
|
|||
]
|
||||
|
||||
return CheckpointListResponse(
|
||||
outputs_dir=resolved_outputs_dir,
|
||||
outputs_dir=outputs_dir,
|
||||
models=models,
|
||||
)
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ if str(backend_path) not in sys.path:
|
|||
try:
|
||||
from core.training import get_training_backend
|
||||
from utils.models.model_config import load_model_defaults
|
||||
from utils.paths import resolve_dataset_path
|
||||
except ImportError:
|
||||
# Fallback: try to import from parent directory
|
||||
parent_backend = backend_path.parent / "backend"
|
||||
|
|
@ -31,7 +30,6 @@ except ImportError:
|
|||
sys.path.insert(0, str(parent_backend))
|
||||
from core.training import get_training_backend
|
||||
from utils.models.model_config import load_model_defaults
|
||||
from utils.paths import resolve_dataset_path
|
||||
|
||||
# Auth
|
||||
from auth.authentication import get_current_subject
|
||||
|
|
@ -119,8 +117,24 @@ async def start_training(
|
|||
if request.local_datasets:
|
||||
validated_datasets = []
|
||||
missing_datasets = []
|
||||
# Get the backend directory (where this file is located)
|
||||
backend_dir = Path(__file__).parent.parent
|
||||
assets_datasets_dir = backend_dir / "assets" / "datasets"
|
||||
|
||||
for dataset_path in request.local_datasets:
|
||||
dataset_file = resolve_dataset_path(dataset_path)
|
||||
dataset_file = Path(dataset_path)
|
||||
|
||||
# If not absolute, try multiple locations
|
||||
if not dataset_file.is_absolute():
|
||||
# First try: relative to current working directory
|
||||
candidate = Path.cwd() / dataset_path
|
||||
if not candidate.exists():
|
||||
# Second try: relative to assets/datasets folder
|
||||
candidate = assets_datasets_dir / dataset_path
|
||||
if not candidate.exists():
|
||||
# Third try: just the filename in assets/datasets
|
||||
candidate = assets_datasets_dir / dataset_file.name
|
||||
dataset_file = candidate
|
||||
|
||||
if not dataset_file.exists():
|
||||
missing_datasets.append(
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import json
|
|||
import logging
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple
|
||||
from utils.paths import outputs_root, resolve_output_dir
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -34,7 +33,7 @@ def _read_checkpoint_loss(checkpoint_path: Path) -> Optional[float]:
|
|||
|
||||
|
||||
def scan_checkpoints(
|
||||
outputs_dir: str = str(outputs_root()),
|
||||
outputs_dir: str = "./outputs",
|
||||
) -> List[Tuple[str, List[Tuple[str, str, Optional[float]]], dict]]:
|
||||
"""
|
||||
Scan outputs folder for training runs and their checkpoints.
|
||||
|
|
@ -46,7 +45,7 @@ def scan_checkpoints(
|
|||
set to the loss of the last (highest-step) intermediate checkpoint.
|
||||
"""
|
||||
models = []
|
||||
outputs_path = resolve_output_dir(outputs_dir)
|
||||
outputs_path = Path(outputs_dir)
|
||||
|
||||
if not outputs_path.exists():
|
||||
logger.warning(f"Outputs directory not found: {outputs_dir}")
|
||||
|
|
|
|||
|
|
@ -7,15 +7,7 @@ Model and LoRA configuration handling
|
|||
from transformers import AutoConfig
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Dict, Any
|
||||
from utils.paths import (
|
||||
normalize_path,
|
||||
is_local_path,
|
||||
is_model_cached,
|
||||
outputs_root,
|
||||
exports_root,
|
||||
resolve_output_dir,
|
||||
resolve_export_dir,
|
||||
)
|
||||
from utils.paths import normalize_path, is_local_path, is_model_cached
|
||||
from utils.utils import without_hf_auth
|
||||
import logging
|
||||
import os
|
||||
|
|
@ -902,7 +894,7 @@ def download_gguf_file(
|
|||
return local_path
|
||||
|
||||
|
||||
def scan_trained_loras(outputs_dir: str = str(outputs_root())) -> List[Tuple[str, str]]:
|
||||
def scan_trained_loras(outputs_dir: str = "./outputs") -> List[Tuple[str, str]]:
|
||||
"""
|
||||
Scan outputs folder for trained LoRA adapters.
|
||||
|
||||
|
|
@ -916,7 +908,7 @@ def scan_trained_loras(outputs_dir: str = str(outputs_root())) -> List[Tuple[str
|
|||
]
|
||||
"""
|
||||
trained_loras = []
|
||||
outputs_path = resolve_output_dir(outputs_dir)
|
||||
outputs_path = Path(outputs_dir)
|
||||
|
||||
if not outputs_path.exists():
|
||||
logger.warning(f"Outputs directory not found: {outputs_dir}")
|
||||
|
|
@ -945,7 +937,7 @@ def scan_trained_loras(outputs_dir: str = str(outputs_root())) -> List[Tuple[str
|
|||
logger.error(f"Error scanning outputs folder: {e}")
|
||||
return []
|
||||
|
||||
def scan_exported_models(exports_dir: str = str(exports_root())) -> List[Tuple[str, str, str, Optional[str]]]:
|
||||
def scan_exported_models(exports_dir: str = "./exports") -> List[Tuple[str, str, str, Optional[str]]]:
|
||||
"""
|
||||
Scan exports folder for exported models (merged, LoRA, GGUF).
|
||||
|
||||
|
|
@ -958,7 +950,7 @@ def scan_exported_models(exports_dir: str = str(exports_root())) -> List[Tuple[s
|
|||
export_type: "lora" | "merged" | "gguf"
|
||||
"""
|
||||
results = []
|
||||
exports_path = resolve_export_dir(exports_dir)
|
||||
exports_path = Path(exports_dir)
|
||||
|
||||
if not exports_path.exists():
|
||||
return results
|
||||
|
|
@ -1046,7 +1038,7 @@ def scan_exported_models(exports_dir: str = str(exports_root())) -> List[Tuple[s
|
|||
# Fallback: read base model from the original training run's
|
||||
# adapter_config.json in ./outputs/{run_name}/
|
||||
if not base_model:
|
||||
outputs_adapter_cfg = resolve_output_dir(run_dir.name) / "adapter_config.json"
|
||||
outputs_adapter_cfg = Path("./outputs") / run_dir.name / "adapter_config.json"
|
||||
try:
|
||||
if outputs_adapter_cfg.exists():
|
||||
cfg = json.loads(outputs_adapter_cfg.read_text())
|
||||
|
|
|
|||
|
|
@ -5,52 +5,10 @@
|
|||
Path utilities for model and dataset handling
|
||||
"""
|
||||
from .path_utils import normalize_path, is_local_path, is_model_cached, get_cache_path
|
||||
from .storage_roots import (
|
||||
studio_root,
|
||||
assets_root,
|
||||
datasets_root,
|
||||
dataset_uploads_root,
|
||||
recipe_datasets_root,
|
||||
outputs_root,
|
||||
exports_root,
|
||||
auth_root,
|
||||
auth_db_path,
|
||||
tmp_root,
|
||||
seed_uploads_root,
|
||||
unstructured_seed_cache_root,
|
||||
oxc_validator_tmp_root,
|
||||
tensorboard_root,
|
||||
ensure_dir,
|
||||
resolve_under_root,
|
||||
resolve_output_dir,
|
||||
resolve_export_dir,
|
||||
resolve_tensorboard_dir,
|
||||
resolve_dataset_path,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
'normalize_path',
|
||||
'is_local_path',
|
||||
'is_model_cached',
|
||||
'get_cache_path',
|
||||
'studio_root',
|
||||
'assets_root',
|
||||
'datasets_root',
|
||||
'dataset_uploads_root',
|
||||
'recipe_datasets_root',
|
||||
'outputs_root',
|
||||
'exports_root',
|
||||
'auth_root',
|
||||
'auth_db_path',
|
||||
'tmp_root',
|
||||
'seed_uploads_root',
|
||||
'unstructured_seed_cache_root',
|
||||
'oxc_validator_tmp_root',
|
||||
'tensorboard_root',
|
||||
'ensure_dir',
|
||||
'resolve_under_root',
|
||||
'resolve_output_dir',
|
||||
'resolve_export_dir',
|
||||
'resolve_tensorboard_dir',
|
||||
'resolve_dataset_path',
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,143 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
def studio_root() -> Path:
|
||||
return Path.home() / ".unsloth" / "studio"
|
||||
|
||||
|
||||
def assets_root() -> Path:
|
||||
return studio_root() / "assets"
|
||||
|
||||
|
||||
def datasets_root() -> Path:
|
||||
return assets_root() / "datasets"
|
||||
|
||||
|
||||
def dataset_uploads_root() -> Path:
|
||||
return datasets_root() / "uploads"
|
||||
|
||||
|
||||
def recipe_datasets_root() -> Path:
|
||||
return datasets_root() / "recipes"
|
||||
|
||||
|
||||
def outputs_root() -> Path:
|
||||
return studio_root() / "outputs"
|
||||
|
||||
|
||||
def exports_root() -> Path:
|
||||
return studio_root() / "exports"
|
||||
|
||||
|
||||
def auth_root() -> Path:
|
||||
return studio_root() / "auth"
|
||||
|
||||
|
||||
def auth_db_path() -> Path:
|
||||
return auth_root() / "auth.db"
|
||||
|
||||
|
||||
def tmp_root() -> Path:
|
||||
return Path(tempfile.gettempdir()) / "unsloth-studio"
|
||||
|
||||
|
||||
def seed_uploads_root() -> Path:
|
||||
return tmp_root() / "seed-uploads"
|
||||
|
||||
|
||||
def unstructured_seed_cache_root() -> Path:
|
||||
return tmp_root() / "unstructured-seed-cache"
|
||||
|
||||
|
||||
def oxc_validator_tmp_root() -> Path:
|
||||
return tmp_root() / "oxc-validator"
|
||||
|
||||
|
||||
def tensorboard_root() -> Path:
|
||||
return studio_root() / "runs"
|
||||
|
||||
|
||||
def ensure_dir(path: Path) -> Path:
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def _clean_relative_path(path_value: str, *, strip_prefixes: tuple[str, ...] = ()) -> Path:
|
||||
path = Path(path_value).expanduser()
|
||||
parts = [part for part in path.parts if part not in ("", ".")]
|
||||
while parts and parts[0] in strip_prefixes:
|
||||
parts = parts[1:]
|
||||
return Path(*parts) if parts else Path()
|
||||
|
||||
|
||||
def resolve_under_root(
|
||||
path_value: str | None,
|
||||
*,
|
||||
root: Path,
|
||||
strip_prefixes: tuple[str, ...] = (),
|
||||
) -> Path:
|
||||
if not path_value or not str(path_value).strip():
|
||||
return root
|
||||
|
||||
path = Path(str(path_value).strip()).expanduser()
|
||||
if path.is_absolute():
|
||||
return path
|
||||
|
||||
cleaned = _clean_relative_path(str(path), strip_prefixes=strip_prefixes)
|
||||
return root / cleaned
|
||||
|
||||
|
||||
def resolve_output_dir(path_value: str | None = None) -> Path:
|
||||
return resolve_under_root(
|
||||
path_value,
|
||||
root=outputs_root(),
|
||||
strip_prefixes=("outputs",),
|
||||
)
|
||||
|
||||
|
||||
def resolve_export_dir(path_value: str | None = None) -> Path:
|
||||
return resolve_under_root(
|
||||
path_value,
|
||||
root=exports_root(),
|
||||
strip_prefixes=("exports",),
|
||||
)
|
||||
|
||||
|
||||
def resolve_tensorboard_dir(path_value: str | None = None) -> Path:
|
||||
return resolve_under_root(
|
||||
path_value,
|
||||
root=tensorboard_root(),
|
||||
strip_prefixes=("runs", "tensorboard"),
|
||||
)
|
||||
|
||||
|
||||
def resolve_dataset_path(path_value: str) -> Path:
|
||||
path = Path(path_value).expanduser()
|
||||
if path.is_absolute():
|
||||
return path
|
||||
|
||||
parts = [part for part in Path(path_value).parts if part not in ("", ".")]
|
||||
if parts[:2] == ["assets", "datasets"]:
|
||||
parts = parts[2:]
|
||||
if parts and parts[0] == "uploads":
|
||||
cleaned = Path(*parts[1:]) if len(parts) > 1 else Path()
|
||||
return dataset_uploads_root() / cleaned
|
||||
if parts and parts[0] == "recipes":
|
||||
cleaned = Path(*parts[1:]) if len(parts) > 1 else Path()
|
||||
return recipe_datasets_root() / cleaned
|
||||
|
||||
cleaned = Path(*parts) if parts else Path()
|
||||
candidates = [
|
||||
dataset_uploads_root() / cleaned,
|
||||
recipe_datasets_root() / cleaned,
|
||||
datasets_root() / cleaned,
|
||||
dataset_uploads_root() / cleaned.name,
|
||||
recipe_datasets_root() / cleaned.name,
|
||||
]
|
||||
for candidate in candidates:
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
return candidates[0]
|
||||
|
|
@ -49,11 +49,9 @@ export async function listModels(): Promise<ListModelsResponse> {
|
|||
return parseJsonOrThrow<ListModelsResponse>(response);
|
||||
}
|
||||
|
||||
export async function listLoras(outputsDir?: string): Promise<ListLorasResponse> {
|
||||
const query = outputsDir
|
||||
? `?${new URLSearchParams({ outputs_dir: outputsDir }).toString()}`
|
||||
: "";
|
||||
const response = await authFetch(`/api/models/loras${query}`);
|
||||
export async function listLoras(outputsDir = "./outputs"): Promise<ListLorasResponse> {
|
||||
const query = new URLSearchParams({ outputs_dir: outputsDir }).toString();
|
||||
const response = await authFetch(`/api/models/loras?${query}`);
|
||||
return parseJsonOrThrow<ListLorasResponse>(response);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -162,8 +162,8 @@ export function ExportPage() {
|
|||
// For other formats, nest under training-run/checkpoint
|
||||
const saveDir =
|
||||
exportMethod === "gguf"
|
||||
? `${baseModelName.split("/").pop() ?? selectedModelIdx ?? "model"}-finetune-gguf`
|
||||
: `${selectedModelIdx ?? "model"}/${checkpoint}`;
|
||||
? `./exports/${(baseModelName.split("/").pop() ?? selectedModelIdx ?? "model")}-finetune-gguf`
|
||||
: `./exports/${selectedModelIdx ?? "model"}/${checkpoint}`;
|
||||
const pushToHub = destination === "hub";
|
||||
const repoId = pushToHub && hfUsername && modelName
|
||||
? `${hfUsername}/${modelName}`
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue