6e057ffebe* Studio: training survives a non-writable HF datasets cache A shared HF datasets cache can contain subtrees owned by another user (for example populated by an earlier root-run job). datasets then dies with "[Errno 13] Permission denied: ..._builder.lock" while locking the cached builder and the training run fails. load_dataset in the training worker and trainer now goes through a wrapper that catches the EACCES and rebuilds the dataset in a Studio-owned cache under cache_root()/hf-datasets, logging the fallback. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Scope the HF_DATASETS_CACHE override to the fallback load * Route non-streaming dataset preview loads through the cache-safe wrapper --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
51 lines
1.8 KiB
Python
51 lines
1.8 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-only
|
|
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
|
|
|
"""Permission-safe wrapper around datasets.load_dataset.
|
|
|
|
A shared HF datasets cache can contain subtrees owned by another user (for
|
|
example populated by an earlier root-run job). datasets then raises
|
|
"[Errno 13] Permission denied: ..._builder.lock" while locking the cached
|
|
builder, killing the training run even though the dataset itself is fine.
|
|
Retry such loads in a Studio-owned cache so the run proceeds; the worst case
|
|
is one rebuild of the dataset in the fallback location.
|
|
"""
|
|
|
|
import logging
|
|
import os
|
|
|
|
from utils.paths.storage_roots import cache_root
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def studio_datasets_cache() -> str:
|
|
path = cache_root() / "hf-datasets"
|
|
path.mkdir(parents = True, exist_ok = True)
|
|
return str(path)
|
|
|
|
|
|
def load_dataset_cache_safe(*args, **kwargs):
|
|
"""datasets.load_dataset, retried in a Studio-owned cache on EACCES."""
|
|
from datasets import load_dataset
|
|
try:
|
|
return load_dataset(*args, **kwargs)
|
|
except PermissionError as error:
|
|
fallback = studio_datasets_cache()
|
|
logger.warning(
|
|
"HF datasets cache is not writable (%s); rebuilding in %s",
|
|
error,
|
|
fallback,
|
|
)
|
|
kwargs["cache_dir"] = fallback
|
|
# Nested builders consult the env var while the load runs; restore it
|
|
# after so other datasets keep trying the shared cache first.
|
|
old_env = os.environ.get("HF_DATASETS_CACHE")
|
|
os.environ["HF_DATASETS_CACHE"] = fallback
|
|
try:
|
|
return load_dataset(*args, **kwargs)
|
|
finally:
|
|
if old_env is None:
|
|
os.environ.pop("HF_DATASETS_CACHE", None)
|
|
else:
|
|
os.environ["HF_DATASETS_CACHE"] = old_env
|