ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(studio): studio storage roots path utilities

Browse source
This commit is contained in:
Shine1i 2026-03-09 19:58:22 +01:00 • committed by Roland Tannous
commit 904e440513
18 changed files with 308 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

6
studio/backend/core/data_recipe/jobs/worker.py
View file

@ -18,9 +18,9 @@ from typing import Any
from ..jsonable import to_jsonable, to_preview_jsonable
from .constants import EVENT_JOB_COMPLETED, EVENT_JOB_ERROR, EVENT_JOB_STARTED
from ..service import build_config_builder, create_data_designer
from utils.paths import ensure_dir, recipe_datasets_root
_PROJECT_ROOT = Path(__file__).resolve().parents[5]
_ARTIFACT_ROOT = _PROJECT_ROOT / "studio" / "backend" / "assets" / "datasets"
_ARTIFACT_ROOT = recipe_datasets_root()
class _QueueLogHandler(logging.Handler):
@ -103,7 +103,7 @@ def run_job_process(
artifact_root=_ARTIFACT_ROOT,
)
merge_batches = bool(run.get("merge_batches"))
_ARTIFACT_ROOT.mkdir(parents=True, exist_ok=True)
ensure_dir(_ARTIFACT_ROOT)
run_config_raw = run.get("run_config") or {}
builder = build_config_builder(recipe)

12
studio/backend/core/data_recipe/local_callable_validators.py
View file

@ -4,8 +4,8 @@
from __future__ import annotations
import json
import os
import structlog
from loggers import get_logger
import subprocess
from copy import deepcopy
from dataclasses import dataclass
@ -13,6 +13,9 @@ from functools import lru_cache
from pathlib import Path
from typing import Any
from loggers import get_logger
from utils.paths import ensure_dir, oxc_validator_tmp_root
logger = get_logger(__name__)
OXC_VALIDATION_FN_MARKER = "unsloth_oxc_validator"
@ -236,6 +239,12 @@ def _run_oxc_batch(
"codes": code_values,
}
try:
tmp_dir = ensure_dir(oxc_validator_tmp_root())
env = dict(os.environ)
tmp_dir_str = str(tmp_dir)
env["TMPDIR"] = tmp_dir_str
env["TMP"] = tmp_dir_str
env["TEMP"] = tmp_dir_str
proc = subprocess.run(
["node", str(_OXC_RUNNER_PATH)],
cwd=str(_OXC_TOOL_DIR),
@ -243,6 +252,7 @@ def _run_oxc_batch(
text=True,
capture_output=True,
check=False,
env=env,
)
except (OSError, ValueError) as exc:
logger.warning("OXC subprocess launch failed: %s", exc)

15
studio/backend/core/export/export.py
View file

@ -22,6 +22,7 @@ from utils.hardware import clear_gpu_cache
from utils.models import is_vision_model, get_base_model_from_lora
from utils.models.model_config import detect_audio_type
from utils.paths import ensure_dir, outputs_root, resolve_export_dir, resolve_output_dir
from core.inference import get_inference_backend
logger = get_logger(__name__)
@ -129,7 +130,7 @@ class ExportBackend:
logger.error(f"Error during memory cleanup: {e}")
return False
def scan_checkpoints(self, outputs_dir: str = "./outputs") -> List[Tuple[str, List[Tuple[str, str]]]]:
def scan_checkpoints(self, outputs_dir: str = str(outputs_root())) -> List[Tuple[str, List[Tuple[str, str]]]]:
"""
Scan outputs folder for training runs and their checkpoints.
@ -326,8 +327,9 @@ class ExportBackend:
# Save locally if requested
if save_directory:
save_directory = str(resolve_export_dir(save_directory))
logger.info(f"Saving merged model locally to: {save_directory}")
os.makedirs(save_directory, exist_ok=True)
ensure_dir(Path(save_directory))
self.current_model.save_pretrained_merged(
save_directory,
@ -387,8 +389,9 @@ class ExportBackend:
try:
# Save locally if requested
if save_directory:
save_directory = str(resolve_export_dir(save_directory))
logger.info(f"Saving base model locally to: {save_directory}")
os.makedirs(save_directory, exist_ok=True)
ensure_dir(Path(save_directory))
self.current_model.save_pretrained(save_directory)
self.current_tokenizer.save_pretrained(save_directory)
@ -476,6 +479,7 @@ class ExportBackend:
# Save locally if requested
if save_directory:
save_directory = str(resolve_export_dir(save_directory))
# Resolve to absolute path so unsloth's relative-path internals
# (check_llama_cpp, use_local_gguf, _download_convert_hf_to_gguf)
# all resolve against the repo root cwd, NOT the export directory.
@ -483,7 +487,7 @@ class ExportBackend:
logger.info(f"Saving GGUF model locally to: {abs_save_dir}")
# Create the directory if it doesn't exist
os.makedirs(abs_save_dir, exist_ok=True)
ensure_dir(Path(abs_save_dir))
# On WSL, patch out sudo check before llama.cpp build
_apply_wsl_sudo_patch()
@ -583,8 +587,9 @@ class ExportBackend:
try:
# Save locally if requested
if save_directory:
save_directory = str(resolve_export_dir(save_directory))
logger.info(f"Saving LoRA adapter locally to: {save_directory}")
os.makedirs(save_directory, exist_ok=True)
ensure_dir(Path(save_directory))
self.current_model.save_pretrained(save_directory)
self.current_tokenizer.save_pretrained(save_directory)

3
studio/backend/core/export/orchestrator.py
View file

@ -22,6 +22,7 @@ import threading
import time
from pathlib import Path
from typing import Any, List, Optional, Tuple
from utils.paths import outputs_root
logger = get_logger(__name__)
@ -379,7 +380,7 @@ class ExportOrchestrator:
return success
def scan_checkpoints(
self, outputs_dir: str = "./outputs"
self, outputs_dir: str = str(outputs_root())
) -> List[Tuple[str, list]]:
"""Scan for checkpoints — no ML imports needed, runs locally."""
from utils.models.checkpoints import scan_checkpoints

48
studio/backend/core/training/trainer.py
View file

@ -31,12 +31,20 @@ from datasets import Dataset, load_dataset
from utils.models import is_vision_model, detect_audio_type
from utils.datasets import format_and_template_dataset
from utils.datasets import MODEL_TO_TEMPLATE_MAPPER, TEMPLATE_TO_RESPONSES_MAPPER
from utils.paths import ensure_dir, resolve_dataset_path, resolve_output_dir, resolve_tensorboard_dir
from trl import SFTTrainer, SFTConfig
logger = get_logger(__name__)
_BACKEND_ROOT = Path(__file__).resolve().parents[2]
_ASSETS_DATASETS_ROOT = _BACKEND_ROOT / "assets" / "datasets"
def _build_report_targets(training_args) -> list[str] | str:
report_to: list[str] = []
if training_args.get("enable_wandb", False):
report_to.append("wandb")
if training_args.get("enable_tensorboard", False):
report_to.append("tensorboard")
return report_to or "none"
@dataclass
@ -270,9 +278,14 @@ class UnslothTrainer:
"lr_scheduler_type": lr_scheduler_type,
"seed": random_seed,
"output_dir": output_dir,
"report_to": ["wandb"] if training_args.get('enable_wandb', False) else "none",
"report_to": _build_report_targets(training_args),
}
if training_args.get("enable_tensorboard", False):
config["logging_dir"] = str(
resolve_tensorboard_dir(training_args.get("tensorboard_dir"))
)
# max_steps vs epochs
if max_steps_val and max_steps_val > 0:
config["max_steps"] = max_steps_val
@ -1594,6 +1607,7 @@ class UnslothTrainer:
import numpy as np
import soundfile as sf
from datasets import Dataset as HFDataset
from utils.paths import ensure_dir, tmp_root
device = "cuda" if torch.cuda.is_available() else "cpu"
@ -1690,10 +1704,18 @@ class UnslothTrainer:
audio_bytes = buf.getvalue()
# 1. Get word timings from Whisper
with tempfile.NamedTemporaryFile(suffix=".wav", delete=True) as tmp:
with tempfile.NamedTemporaryFile(
suffix=".wav",
delete=False,
dir=str(ensure_dir(tmp_root())),
) as tmp:
tmp.write(audio_bytes)
tmp.flush()
whisper_result = whisper_model.transcribe(tmp.name, word_timestamps=True)
tmp_path = tmp.name
try:
whisper_result = whisper_model.transcribe(tmp_path, word_timestamps=True)
finally:
Path(tmp_path).unlink(missing_ok=True)
normalized_transcript = text_normalizations(text)
words_with_timings = []
@ -1881,7 +1903,7 @@ class UnslothTrainer:
file_path = dataset_file
else:
# Fallback: try relative to assets/datasets
file_path = str(_ASSETS_DATASETS_ROOT / dataset_file)
file_path = str(resolve_dataset_path(dataset_file))
file_path_obj = Path(file_path)
@ -2158,7 +2180,7 @@ class UnslothTrainer:
dataset: Dataset,
eval_dataset: Dataset = None,
eval_steps: float = 0.00,
output_dir: str = "./outputs",
output_dir: str | None = None,
num_epochs: int = 3,
learning_rate: float = 5e-5,
batch_size: int = 2,
@ -2175,7 +2197,7 @@ class UnslothTrainer:
wandb_project: str = "unsloth-training",
wandb_token: str = None,
enable_tensorboard: bool = False,
tensorboard_dir: str = "runs",
tensorboard_dir: str | None = None,
**kwargs) -> bool:
"""Start training in a separate thread"""
@ -2263,8 +2285,8 @@ class UnslothTrainer:
wandb.init(project=training_args.get('wandb_project', 'unsloth-training'))
# Create output directory
output_dir = training_args.get('output_dir', './outputs')
os.makedirs(output_dir, exist_ok=True)
output_dir = str(resolve_output_dir(training_args.get("output_dir")))
ensure_dir(Path(output_dir))
# ========== AUDIO TRAINER BRANCH ==========
if self._audio_type == 'csm':
@ -2474,11 +2496,15 @@ class UnslothTrainer:
"weight_decay": training_args.get('weight_decay', 0.01),
"seed": training_args.get('random_seed', 3407),
"output_dir": output_dir,
"report_to": ["wandb"] if training_args.get('enable_wandb', False) else "none",
"report_to": _build_report_targets(training_args),
"include_num_input_tokens_seen": True, # Enable token counting
"dataset_num_proc": 1 if (self.is_audio or self.is_audio_vlm or self._cuda_audio_used) else safe_num_proc(max(1, os.cpu_count() // 4)),
"max_seq_length": training_args.get('max_seq_length', 2048),
}
if training_args.get("enable_tensorboard", False):
config_args["logging_dir"] = str(
resolve_tensorboard_dir(training_args.get("tensorboard_dir"))
)
logger.info(f"[DEBUG] dataset_num_proc={config_args['dataset_num_proc']} (is_audio={self.is_audio}, is_audio_vlm={self.is_audio_vlm}, _cuda_audio_used={self._cuda_audio_used})")
# On Windows with transformers 5.x, disable DataLoader multiprocessing

12
studio/backend/core/training/worker.py
View file

@ -133,6 +133,7 @@ def run_training_process(
sys.path.insert(0, backend_path)
from core.training.trainer import UnslothTrainer, TrainingProgress
from utils.paths import ensure_dir, resolve_output_dir, resolve_tensorboard_dir
import transformers
logger.info("Subprocess loaded transformers %s", transformers.__version__)
@ -363,7 +364,14 @@ def run_training_process(
# Generate output dir
output_dir = config.get("output_dir")
if not output_dir:
output_dir = f"./outputs/{model_name.replace('/', '_')}_{int(time.time())}"
output_dir = f"{model_name.replace('/', '_')}_{int(time.time())}"
output_dir = str(resolve_output_dir(output_dir))
ensure_dir(Path(output_dir))
tensorboard_dir = config.get("tensorboard_dir")
if config.get("enable_tensorboard", False):
tensorboard_dir = str(resolve_tensorboard_dir(tensorboard_dir))
ensure_dir(Path(tensorboard_dir))
# Start training (directly — no inner thread, we ARE the subprocess)
_send_status(event_queue, "Starting training...")
@ -389,7 +397,7 @@ def run_training_process(
wandb_project=config.get("wandb_project", "unsloth-training"),
wandb_token=config.get("wandb_token"),
enable_tensorboard=config.get("enable_tensorboard", False),
tensorboard_dir=config.get("tensorboard_dir", "runs"),
tensorboard_dir=tensorboard_dir,
eval_dataset=eval_dataset,
eval_steps=eval_steps,
max_seq_length=config.get("max_seq_length", 2048),

6
studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py
View file

@ -8,11 +8,13 @@ import re
from pathlib import Path
from typing import Any
from utils.paths import ensure_dir, unstructured_seed_cache_root
DEFAULT_CHUNK_SIZE = 1200
DEFAULT_CHUNK_OVERLAP = 200
MAX_CHUNK_SIZE = 20000
_MIN_BREAK_RATIO = 0.6
_CACHE_DIR = Path.home() / ".cache" / "unsloth" / "data-recipe" / "unstructured-seed-cache"
_CACHE_DIR = unstructured_seed_cache_root()
def resolve_chunking(
@ -85,7 +87,7 @@ def materialize_unstructured_seed_dataset(
raise ValueError("No text found in unstructured seed source.")
rows = [{"chunk_text": chunk} for chunk in chunks]
_CACHE_DIR.mkdir(parents=True, exist_ok=True)
ensure_dir(_CACHE_DIR)
try:
import pandas as pd
except ImportError as exc: # pragma: no cover

5
studio/backend/routes/data_recipe/seed.py
View file

@ -18,6 +18,7 @@ from data_designer_unstructured_seed.chunking import (
resolve_chunking,
)
from core.data_recipe.jsonable import to_preview_jsonable
from utils.paths import ensure_dir, seed_uploads_root
from models.data_recipe import (
SeedInspectRequest,
@ -31,7 +32,7 @@ DATA_EXTS = (".parquet", ".jsonl", ".json", ".csv")
DEFAULT_SPLIT = "train"
LOCAL_UPLOAD_EXTS = {".csv", ".json", ".jsonl"}
UNSTRUCTURED_UPLOAD_EXTS = {".txt", ".md"}
SEED_UPLOAD_DIR = Path.home() / ".cache" / "unsloth" / "data-recipe" / "seed-uploads"
SEED_UPLOAD_DIR = seed_uploads_root()
def _serialize_preview_value(value: Any) -> Any:
@ -304,7 +305,7 @@ def inspect_seed_upload(payload: SeedInspectUploadRequest) -> SeedInspectRespons
if len(file_bytes) > max_size_bytes:
raise HTTPException(status_code=413, detail="file too large (max 50MB)")
SEED_UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
ensure_dir(SEED_UPLOAD_DIR)
stored_name = f"{uuid4().hex}_{filename}"
stored_path = SEED_UPLOAD_DIR / stored_name
stored_path.write_bytes(file_bytes)

15
studio/backend/routes/datasets.py
View file

@ -38,6 +38,12 @@ from models.datasets import (
LocalDatasetsResponse,
UploadDatasetResponse,
)
from utils.paths import (
dataset_uploads_root,
ensure_dir,
recipe_datasets_root,
resolve_dataset_path,
)
def _serialize_preview_value(value):
@ -87,9 +93,8 @@ _ARCHIVE_EXTS = ('.tar', '.tar.gz', '.tgz', '.gz', '.zst', '.zip', '.txt')
DATA_EXTS = _TABULAR_EXTS + _ARCHIVE_EXTS
LOCAL_FILE_EXTS = ('.json', '.jsonl', '.csv', '.parquet')
LOCAL_UPLOAD_EXTS = {".csv", ".json", ".jsonl", ".parquet"}
BACKEND_ROOT = Path(__file__).resolve().parents[1]
LOCAL_DATASETS_ROOT = BACKEND_ROOT / "assets" / "datasets"
DATASET_UPLOAD_DIR = LOCAL_DATASETS_ROOT / "uploads"
LOCAL_DATASETS_ROOT = recipe_datasets_root()
DATASET_UPLOAD_DIR = dataset_uploads_root()
def _safe_read_metadata(path: Path) -> dict | None:
@ -273,7 +278,7 @@ async def upload_dataset(
)
max_size_bytes = 512 * 1024 * 1024
DATASET_UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
ensure_dir(DATASET_UPLOAD_DIR)
stem = Path(filename).stem
stored_name = f"{uuid4().hex}_{stem}{ext}"
stored_path = DATASET_UPLOAD_DIR / stored_name
@ -329,7 +334,7 @@ def check_format(
logger.info(f"Checking format for dataset: {request.dataset_name}")
dataset_path = Path(request.dataset_name)
dataset_path = resolve_dataset_path(request.dataset_name)
total_rows = None
if dataset_path.exists():

7
studio/backend/routes/inference.py
View file

@ -570,11 +570,16 @@ def _decode_audio_base64(b64: str) -> np.ndarray:
import torchaudio
import tempfile
import os
from utils.paths import ensure_dir, tmp_root
raw = base64.b64decode(b64)
# torchaudio.load needs a file path or file-like object with format hint
# Write to a temp file so torchaudio can auto-detect the format
with tempfile.NamedTemporaryFile(suffix=".audio", delete=False) as tmp:
with tempfile.NamedTemporaryFile(
suffix=".audio",
delete=False,
dir=str(ensure_dir(tmp_root())),
) as tmp:
tmp.write(raw)
tmp_path = tmp.name
try:

21
studio/backend/routes/models.py
View file

@ -33,6 +33,7 @@ try:
)
from utils.models.model_config import _pick_best_gguf, _extract_quant_label, is_audio_input_type
from core.inference import get_inference_backend
from utils.paths import outputs_root, exports_root, resolve_output_dir, resolve_export_dir
except ImportError:
# Fallback: try to import from parent directory
parent_backend = backend_path.parent / "backend"
@ -51,6 +52,7 @@ except ImportError:
)
from utils.models.model_config import _pick_best_gguf, _extract_quant_label, is_audio_input_type
from core.inference import get_inference_backend
from utils.paths import outputs_root, exports_root, resolve_output_dir, resolve_export_dir
from models import (
CheckpointInfo,
@ -339,8 +341,8 @@ async def get_model_config(
@router.get("/loras")
async def scan_loras(
outputs_dir: str = Query(default="./outputs", description="Directory to scan for LoRA adapters"),
exports_dir: str = Query(default="./exports", description="Directory to scan for exported models"),
outputs_dir: str = Query(default=str(outputs_root()), description="Directory to scan for LoRA adapters"),
exports_dir: str = Query(default=str(exports_root()), description="Directory to scan for exported models"),
current_subject: str = Depends(get_current_subject),
):
"""
@ -350,10 +352,12 @@ async def scan_loras(
(from exports_dir) in a single list, distinguished by source field.
"""
try:
resolved_outputs_dir = str(resolve_output_dir(outputs_dir))
resolved_exports_dir = str(resolve_export_dir(exports_dir))
lora_list = []
# Scan training outputs
trained_loras = scan_trained_loras(outputs_dir=outputs_dir)
trained_loras = scan_trained_loras(outputs_dir=resolved_outputs_dir)
for display_name, adapter_path in trained_loras:
base_model = get_base_model_from_lora(adapter_path)
lora_list.append(LoRAInfo(
@ -364,7 +368,7 @@ async def scan_loras(
))
# Scan exported models (merged, LoRA, base — skips GGUF)
exported = scan_exported_models(exports_dir=exports_dir)
exported = scan_exported_models(exports_dir=resolved_exports_dir)
for display_name, model_path, export_type, base_model in exported:
lora_list.append(LoRAInfo(
display_name=display_name,
@ -376,7 +380,7 @@ async def scan_loras(
return LoRAScanResponse(
loras=lora_list,
outputs_dir=outputs_dir
outputs_dir=resolved_outputs_dir
)
except Exception as e:
@ -523,7 +527,7 @@ async def get_gguf_variants(
@router.get("/checkpoints", response_model=CheckpointListResponse)
async def list_checkpoints(
outputs_dir: str = Query(
default="./outputs",
default=str(outputs_root()),
description="Directory to scan for checkpoints",
),
current_subject: str = Depends(get_current_subject),
@ -534,7 +538,8 @@ async def list_checkpoints(
Scans the outputs folder for training runs and their checkpoints.
"""
try:
raw_models = scan_checkpoints(outputs_dir=outputs_dir)
resolved_outputs_dir = str(resolve_output_dir(outputs_dir))
raw_models = scan_checkpoints(outputs_dir=resolved_outputs_dir)
models = [
ModelCheckpoints(
@ -551,7 +556,7 @@ async def list_checkpoints(
]
return CheckpointListResponse(
outputs_dir=outputs_dir,
outputs_dir=resolved_outputs_dir,
models=models,
)
except Exception as e:

20
studio/backend/routes/training.py
View file

@ -24,6 +24,7 @@ if str(backend_path) not in sys.path:
try:
from core.training import get_training_backend
from utils.models.model_config import load_model_defaults
from utils.paths import resolve_dataset_path
except ImportError:
# Fallback: try to import from parent directory
parent_backend = backend_path.parent / "backend"
@ -31,6 +32,7 @@ except ImportError:
sys.path.insert(0, str(parent_backend))
from core.training import get_training_backend
from utils.models.model_config import load_model_defaults
from utils.paths import resolve_dataset_path
# Auth
from auth.authentication import get_current_subject
@ -111,24 +113,8 @@ async def start_training(
if request.local_datasets:
validated_datasets = []
missing_datasets = []
# Get the backend directory (where this file is located)
backend_dir = Path(__file__).parent.parent
assets_datasets_dir = backend_dir / "assets" / "datasets"
for dataset_path in request.local_datasets:
dataset_file = Path(dataset_path)
# If not absolute, try multiple locations
if not dataset_file.is_absolute():
# First try: relative to current working directory
candidate = Path.cwd() / dataset_path
if not candidate.exists():
# Second try: relative to assets/datasets folder
candidate = assets_datasets_dir / dataset_path
if not candidate.exists():
# Third try: just the filename in assets/datasets
candidate = assets_datasets_dir / dataset_file.name
dataset_file = candidate
dataset_file = resolve_dataset_path(dataset_path)
if not dataset_file.exists():
missing_datasets.append(

5
studio/backend/utils/models/checkpoints.py
View file

@ -9,6 +9,7 @@ import structlog
from loggers import get_logger
from pathlib import Path
from typing import List, Optional, Tuple
from utils.paths import outputs_root, resolve_output_dir
logger = get_logger(__name__)
@ -34,7 +35,7 @@ def _read_checkpoint_loss(checkpoint_path: Path) -> Optional[float]:
def scan_checkpoints(
outputs_dir: str = "./outputs",
outputs_dir: str = str(outputs_root()),
) -> List[Tuple[str, List[Tuple[str, str, Optional[float]]], dict]]:
"""
Scan outputs folder for training runs and their checkpoints.
@ -46,7 +47,7 @@ def scan_checkpoints(
set to the loss of the last (highest-step) intermediate checkpoint.
"""
models = []
outputs_path = Path(outputs_dir)
outputs_path = resolve_output_dir(outputs_dir)
if not outputs_path.exists():
logger.warning(f"Outputs directory not found: {outputs_dir}")

20
studio/backend/utils/models/model_config.py
View file

@ -7,7 +7,15 @@ Model and LoRA configuration handling
from transformers import AutoConfig
from dataclasses import dataclass
from typing import Optional, Dict, Any
from utils.paths import normalize_path, is_local_path, is_model_cached
from utils.paths import (
normalize_path,
is_local_path,
is_model_cached,
outputs_root,
exports_root,
resolve_output_dir,
resolve_export_dir,
)
from utils.utils import without_hf_auth
import structlog
from loggers import get_logger
@ -983,7 +991,7 @@ def is_embedding_model(model_name: str, hf_token: Optional[str] = None) -> bool:
return False
def scan_trained_loras(outputs_dir: str = "./outputs") -> List[Tuple[str, str]]:
def scan_trained_loras(outputs_dir: str = str(outputs_root())) -> List[Tuple[str, str]]:
"""
Scan outputs folder for trained LoRA adapters.
@ -997,7 +1005,7 @@ def scan_trained_loras(outputs_dir: str = "./outputs") -> List[Tuple[str, str]]:
]
"""
trained_loras = []
outputs_path = Path(outputs_dir)
outputs_path = resolve_output_dir(outputs_dir)
if not outputs_path.exists():
logger.warning(f"Outputs directory not found: {outputs_dir}")
@ -1026,7 +1034,7 @@ def scan_trained_loras(outputs_dir: str = "./outputs") -> List[Tuple[str, str]]:
logger.error(f"Error scanning outputs folder: {e}")
return []
def scan_exported_models(exports_dir: str = "./exports") -> List[Tuple[str, str, str, Optional[str]]]:
def scan_exported_models(exports_dir: str = str(exports_root())) -> List[Tuple[str, str, str, Optional[str]]]:
"""
Scan exports folder for exported models (merged, LoRA, GGUF).
@ -1039,7 +1047,7 @@ def scan_exported_models(exports_dir: str = "./exports") -> List[Tuple[str, str,
export_type: "lora" | "merged" | "gguf"
"""
results = []
exports_path = Path(exports_dir)
exports_path = resolve_export_dir(exports_dir)
if not exports_path.exists():
return results
@ -1127,7 +1135,7 @@ def scan_exported_models(exports_dir: str = "./exports") -> List[Tuple[str, str,
# Fallback: read base model from the original training run's
# adapter_config.json in ./outputs/{run_name}/
if not base_model:
outputs_adapter_cfg = Path("./outputs") / run_dir.name / "adapter_config.json"
outputs_adapter_cfg = resolve_output_dir(run_dir.name) / "adapter_config.json"
try:
if outputs_adapter_cfg.exists():
cfg = json.loads(outputs_adapter_cfg.read_text())

38
studio/backend/utils/paths/__init__.py
View file

@ -5,10 +5,48 @@
Path utilities for model and dataset handling
"""
from .path_utils import normalize_path, is_local_path, is_model_cached, get_cache_path
from .storage_roots import (
studio_root,
assets_root,
datasets_root,
dataset_uploads_root,
recipe_datasets_root,
outputs_root,
exports_root,
tmp_root,
seed_uploads_root,
unstructured_seed_cache_root,
oxc_validator_tmp_root,
tensorboard_root,
ensure_dir,
resolve_under_root,
resolve_output_dir,
resolve_export_dir,
resolve_tensorboard_dir,
resolve_dataset_path,
)
__all__ = [
'normalize_path',
'is_local_path',
'is_model_cached',
'get_cache_path',
'studio_root',
'assets_root',
'datasets_root',
'dataset_uploads_root',
'recipe_datasets_root',
'outputs_root',
'exports_root',
'tmp_root',
'seed_uploads_root',
'unstructured_seed_cache_root',
'oxc_validator_tmp_root',
'tensorboard_root',
'ensure_dir',
'resolve_under_root',
'resolve_output_dir',
'resolve_export_dir',
'resolve_tensorboard_dir',
'resolve_dataset_path',
]

134
studio/backend/utils/paths/storage_roots.py Normal file
View file

@ -0,0 +1,134 @@
from __future__ import annotations
from pathlib import Path
def studio_root() -> Path:
return Path.home() / ".unsloth" / "studio"
def assets_root() -> Path:
return studio_root() / "assets"
def datasets_root() -> Path:
return assets_root() / "datasets"
def dataset_uploads_root() -> Path:
return datasets_root() / "uploads"
def recipe_datasets_root() -> Path:
return datasets_root() / "recipes"
def outputs_root() -> Path:
return studio_root() / "outputs"
def exports_root() -> Path:
return studio_root() / "exports"
def tmp_root() -> Path:
return studio_root() / "tmp"
def seed_uploads_root() -> Path:
return tmp_root() / "seed-uploads"
def unstructured_seed_cache_root() -> Path:
return tmp_root() / "unstructured-seed-cache"
def oxc_validator_tmp_root() -> Path:
return tmp_root() / "oxc-validator"
def tensorboard_root() -> Path:
return studio_root() / "runs"
def ensure_dir(path: Path) -> Path:
path.mkdir(parents=True, exist_ok=True)
return path
def _clean_relative_path(path_value: str, *, strip_prefixes: tuple[str, ...] = ()) -> Path:
path = Path(path_value).expanduser()
parts = [part for part in path.parts if part not in ("", ".")]
while parts and parts[0] in strip_prefixes:
parts = parts[1:]
return Path(*parts) if parts else Path()
def resolve_under_root(
path_value: str | None,
*,
root: Path,
strip_prefixes: tuple[str, ...] = (),
) -> Path:
if not path_value or not str(path_value).strip():
return root
path = Path(str(path_value).strip()).expanduser()
if path.is_absolute():
return path
cleaned = _clean_relative_path(str(path), strip_prefixes=strip_prefixes)
return root / cleaned
def resolve_output_dir(path_value: str | None = None) -> Path:
return resolve_under_root(
path_value,
root=outputs_root(),
strip_prefixes=("outputs",),
)
def resolve_export_dir(path_value: str | None = None) -> Path:
return resolve_under_root(
path_value,
root=exports_root(),
strip_prefixes=("exports",),
)
def resolve_tensorboard_dir(path_value: str | None = None) -> Path:
return resolve_under_root(
path_value,
root=tensorboard_root(),
strip_prefixes=("runs", "tensorboard"),
)
def resolve_dataset_path(path_value: str) -> Path:
path = Path(path_value).expanduser()
if path.is_absolute():
return path
parts = [part for part in Path(path_value).parts if part not in ("", ".")]
if parts[:2] == ["assets", "datasets"]:
parts = parts[2:]
if parts and parts[0] == "uploads":
cleaned = Path(*parts[1:]) if len(parts) > 1 else Path()
return dataset_uploads_root() / cleaned
if parts and parts[0] == "recipes":
cleaned = Path(*parts[1:]) if len(parts) > 1 else Path()
return recipe_datasets_root() / cleaned
cleaned = Path(*parts) if parts else Path()
candidates = [
dataset_uploads_root() / cleaned,
recipe_datasets_root() / cleaned,
datasets_root() / cleaned,
dataset_uploads_root() / cleaned.name,
recipe_datasets_root() / cleaned.name,
]
for candidate in candidates:
if candidate.exists():
return candidate
return candidates[0]

8
studio/frontend/src/features/chat/api/chat-api.ts
View file

@ -49,9 +49,11 @@ export async function listModels(): Promise<ListModelsResponse> {
return parseJsonOrThrow<ListModelsResponse>(response);
}
export async function listLoras(outputsDir = "./outputs"): Promise<ListLorasResponse> {
const query = new URLSearchParams({ outputs_dir: outputsDir }).toString();
const response = await authFetch(`/api/models/loras?${query}`);
export async function listLoras(outputsDir?: string): Promise<ListLorasResponse> {
const query = outputsDir
? `?${new URLSearchParams({ outputs_dir: outputsDir }).toString()}`
: "";
const response = await authFetch(`/api/models/loras${query}`);
return parseJsonOrThrow<ListLorasResponse>(response);
}

4
studio/frontend/src/features/export/export-page.tsx
View file

@ -162,8 +162,8 @@ export function ExportPage() {
// For other formats, nest under training-run/checkpoint
const saveDir =
exportMethod === "gguf"
? `./exports/${(baseModelName.split("/").pop() ?? selectedModelIdx ?? "model")}-finetune-gguf`
: `./exports/${selectedModelIdx ?? "model"}/${checkpoint}`;
? `${baseModelName.split("/").pop() ?? selectedModelIdx ?? "model"}-finetune-gguf`
: `${selectedModelIdx ?? "model"}/${checkpoint}`;
const pushToHub = destination === "hub";
const repoId = pushToHub && hfUsername && modelName
? `${hfUsername}/${modelName}`