diff --git a/studio/backend/utils/llama_cpp_update.py b/studio/backend/utils/llama_cpp_update.py index 174e6ef4dc..83602842af 100644 --- a/studio/backend/utils/llama_cpp_update.py +++ b/studio/backend/utils/llama_cpp_update.py @@ -54,6 +54,8 @@ logger = structlog.get_logger(__name__) DEFAULT_PUBLISHED_REPO = "unslothai/llama.cpp" _INSTALL_TIMEOUT_SECONDS = 1800 # 30 min ceiling for download + build/validate +# install_llama_prebuilt.py EXIT_NO_SPACE: out of disk, retrying will not help. +_EXIT_NO_SPACE = 4 # Background job state. Single in-flight update at a time, guarded by _job_lock. _JOB_IDLE = _flow.JOB_IDLE @@ -496,6 +498,16 @@ def _run_llama_phase( + (" Reload your model to use it." if model_was_active else "") ), } + except _flow.InstallerExit as exc: + # Raw "installer exited 4: " says nothing actionable in the UI. + if exc.returncode == _EXIT_NO_SPACE: + logger.warning("llama update: out of disk space") + raise RuntimeError( + "Not enough disk space to install llama.cpp. Free up space or point " + "UNSLOTH_STUDIO_HOME/TMPDIR at a larger volume, then retry." + ) from exc + logger.warning("llama update: failed", error = str(exc)) + raise except Exception as exc: logger.warning("llama update: failed", error = str(exc)) raise diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index 6ea850139e..676933b67c 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -63,6 +63,7 @@ EXIT_SUCCESS = 0 EXIT_FALLBACK = 2 EXIT_ERROR = 1 EXIT_BUSY = 3 +EXIT_NO_SPACE = 4 # DiskPart-prompt suppression. RunAsInvoker does NOT stop amd-smi's runtime # elevation (its manifest is asInvoker), so this is just harmless belt-and- @@ -3674,7 +3675,8 @@ def hydrate_source_tree( break except Exception as exc: last_exc = exc - if index == len(source_urls) - 1: + # A full disk fails every mirror; stop so a later 404 cannot mask it. + if _environment_fatal_reason(exc) or index == len(source_urls) - 1: raise log(f"source tree download failed from {source_url}: {exc}") if not downloaded: @@ -6000,6 +6002,14 @@ def validate_prebuilt_attempts( ) raise ExistingInstallSatisfied(attempt, tried_fallback) + # Advisory: a few GB free usually fits, and rejecting here would also skip + # the source-build fallback. + if index == 0: + low_disk = _low_disk_warning(install_dir) + if low_disk is not None: + log(low_disk) + _log_disk_space_help() + staging_dir = create_install_staging_dir(install_dir) quantized_path = work_dir / f"stories260K-q4-{index}.gguf" if quantized_path.exists(): @@ -6028,7 +6038,9 @@ def validate_prebuilt_attempts( attempt_error = PrebuiltFallback( f"candidate attempt failed before activation for {attempt.name}: {exc}" ) - if index == len(attempt_list) - 1: + if _environment_fatal_reason(exc) or index == len(attempt_list) - 1: + if attempt_error is exc: + raise raise attempt_error from exc log( "selected CUDA bundle failed before activation; trying next prebuilt fallback " @@ -6149,6 +6161,132 @@ def diffusion_visual_server_backfill_needed( return True +def _causal_chain(exc: BaseException) -> Iterable[BaseException]: + seen: set[int] = set() + current: BaseException | None = exc + while current is not None and id(current) not in seen: + seen.add(id(current)) + yield current + # `raise X from None` sets __suppress_context__: the earlier exception is + # unrelated, so following __context__ anyway would misreport the cause. + if current.__cause__ is not None: + current = current.__cause__ + elif current.__suppress_context__: + current = None + else: + current = current.__context__ + + +# ERROR_HANDLE_DISK_FULL / ERROR_DISK_FULL. CPython's PC/errmap.h maps 112 to +# ENOSPC but has no case for 39, which arrives as EINVAL, so check winerror too. +_WINDOWS_DISK_FULL = (39, 112) +# A quota (NFS/XFS/container) leaves blocks this user cannot have, so the bigger +# source build is just as doomed; named apart from ENOSPC so df does not mislead. +# Guarded: the MSVC CRT has no EDQUOT, so on Windows CPython aliases it to the +# Winsock WSAEDQUOT (10069), which no file write raises. +_DISK_FULL_ERRNOS = {errno.ENOSPC: "no space left on device"} +if hasattr(errno, "EDQUOT"): + _DISK_FULL_ERRNOS[errno.EDQUOT] = "disk quota exceeded" + + +def _winerror_of(exc: OSError) -> Any: + """exc.winerror, defensively. Not getattr(exc, ..., None): urllib's HTTPError + is an OSError that proxies unknown attributes to a wrapped file object and + raises KeyError (not AttributeError) on 3.9, which getattr will not swallow. + A 404 from a mirror must not crash the classifier.""" + try: + return exc.winerror + except Exception: + return None + + +def _out_of_space_reason(exc: BaseException) -> str | None: + """Why `exc` means the install cannot fit, or None if it means something else.""" + if isinstance(exc, OSError): + reason = _DISK_FULL_ERRNOS.get(exc.errno) + if reason is not None: + return reason + if _winerror_of(exc) in _WINDOWS_DISK_FULL: + return "no space left on device" + # shutil.copytree stringifies each per-file OSError and raises Error(errors) + # outside the except block, so errno and the chain are gone and only text + # survives. OSError.__str__ returns early on winerror, so Windows reads + # "[WinError 112]" and never "[Errno 28]": match both, brackets included so + # WinError 112 does not match WinError 1120. + if isinstance(exc, shutil.Error): + text = str(exc) + for code, reason in _DISK_FULL_ERRNOS.items(): + if f"[Errno {code}]" in text: + return reason + if any(f"[WinError {code}]" in text for code in _WINDOWS_DISK_FULL): + return "no space left on device" + return None + + +def _environment_fatal_reason(exc: BaseException) -> str | None: + for cause in _causal_chain(exc): + reason = _out_of_space_reason(cause) + if reason is not None: + return reason + return None + + +def _log_disk_space_help() -> None: + log( + "free up space or point TMPDIR and UNSLOTH_STUDIO_HOME at a larger " + "volume (e.g. /workspace), then re-run" + ) + + +@contextmanager +def scratch_dir(prefix: str) -> Iterator[Path]: + """Temp dir whose cleanup never raises: an rmtree failure on the way out would + replace the in-flight exception and lose EXIT_NO_SPACE. Not + TemporaryDirectory(ignore_cleanup_errors = True), which is 3.10+ (setup.sh + still runs this helper under the host python, and we support 3.9).""" + path = Path(tempfile.mkdtemp(prefix = prefix)) + try: + yield path + finally: + shutil.rmtree(path, ignore_errors = True) + + +def _first_existing_ancestor(path: Path) -> Path: + current = path + while current != current.parent and not current.exists(): + current = current.parent + return current + + +def _low_disk_warning(install_dir: Path, *, advised_gb: float = 5.0) -> str | None: + """Advisory only, never fatal. A prebuilt install peaks well under 1 GB (the + largest published bundle is 0.77 GB, macOS is 0.01 GB), so a fixed threshold + cannot decide whether this host has room -- a real ENOSPC decides that. The + number here is the headroom a source-build fallback would want.""" + advised = int(advised_gb * (1024**3)) + targets = { + "build/download scratch (TMPDIR)": Path(tempfile.gettempdir()), + "llama.cpp install dir": _first_existing_ancestor(install_dir), + } + for label, path in targets.items(): + try: + free = shutil.disk_usage(path).free + except OSError: + continue + if free < advised: + return ( + f"low disk space for llama.cpp: {label} at {path} has " + f"{free / (1024**3):.1f} GB free (~{advised_gb:.0f} GB recommended)" + ) + return None + + +def _fail_no_space(reason: str) -> None: + log(reason) + _log_disk_space_help() + raise SystemExit(EXIT_NO_SPACE) + + def install_prebuilt( install_dir: Path, llama_tag: str, @@ -6217,8 +6355,7 @@ def install_prebuilt( # recorded so the updater re-asserts it (#7213). sync_marker_force_cpu(install_dir, persist_force_cpu) return - with tempfile.TemporaryDirectory(prefix = "unsloth-llama-prebuilt-") as tmp: - work_dir = Path(tmp) + with scratch_dir("unsloth-llama-prebuilt-") as work_dir: probe_path = work_dir / "stories260K.gguf" download_validation_model(probe_path, validation_model_cache_path(install_dir)) release_count = len(release_plans) @@ -6263,6 +6400,8 @@ def install_prebuilt( except ExistingInstallSatisfied: return except PrebuiltFallback as exc: + if _environment_fatal_reason(exc): + raise if release_index == release_count - 1: raise log( @@ -6296,6 +6435,11 @@ def install_prebuilt( log(f"prebuilt busy reason: {exc}") raise SystemExit(EXIT_BUSY) from exc except PrebuiltFallback as exc: + fatal = _environment_fatal_reason(exc) + if fatal: + log(f"prebuilt install failed: {fatal}") + _log_disk_space_help() + raise SystemExit(EXIT_NO_SPACE) from exc log("prebuilt install path failed; falling back to source build") log(f"prebuilt fallback reason: {exc}") report = collect_system_report(host, choice, install_dir) @@ -6466,6 +6610,10 @@ def main() -> int: install_kind = args.install_kind, ) except PrebuiltFallback as exc: + # A full disk is not a bad build: the CPU source rebuild needs more space. + fatal = _environment_fatal_reason(exc) + if fatal: + _fail_no_space(f"install validation failed: {fatal}") print(str(exc), file = sys.stderr) raise SystemExit(EXIT_FALLBACK) from exc return EXIT_SUCCESS @@ -6595,9 +6743,15 @@ if __name__ == "__main__": # Expected when the published repo (e.g. ggml-org/llama.cpp) has no # prebuilt manifest. Exit quietly with EXIT_FALLBACK so the caller # falls back to source build without a noisy "fatal helper error". + fatal = _environment_fatal_reason(exc) + if fatal: + _fail_no_space(f"prebuilt install failed: {fatal}") log(textwrap.shorten(str(exc), width = 400, placeholder = "...")) raise SystemExit(EXIT_FALLBACK) except Exception as exc: + fatal = _environment_fatal_reason(exc) + if fatal: + _fail_no_space(f"prebuilt install failed: {fatal}") message = textwrap.shorten(str(exc), width = 400, placeholder = "...") log(f"fatal helper error: {message}") raise SystemExit(EXIT_ERROR) diff --git a/studio/setup.ps1 b/studio/setup.ps1 index 6a8499b195..bb21ce2da6 100644 --- a/studio/setup.ps1 +++ b/studio/setup.ps1 @@ -3763,6 +3763,18 @@ if ($LocalLlamaCppLinked) { } substep "Close Unsloth or other llama.cpp users and retry" "Yellow" exit 3 + } elseif ($prebuiltExit -eq 4) { + step "llama.cpp" "not enough disk space to install llama.cpp" "Yellow" + Write-LlamaFailureLog -Output $prebuiltOutput + substep "Free up disk or move UNSLOTH_STUDIO_HOME/TEMP to a larger volume, then re-run" "Yellow" + $PreservedLlamaServerFound = $false + foreach ($_cand in @( + (Join-Path $LlamaCppDir "llama-server.exe"), + (Join-Path $LlamaCppDir "build\bin\llama-server.exe"), + (Join-Path $LlamaCppDir "build\bin\Release\llama-server.exe"))) { + if (Test-Path -LiteralPath $_cand) { $PreservedLlamaServerFound = $true; break } + } + if (-not $PreservedLlamaServerFound) { $script:LlamaCppDegraded = $true } } else { step "llama.cpp" "prebuilt install failed (continuing)" "Yellow" Write-LlamaFailureLog -Output $prebuiltOutput diff --git a/studio/setup.sh b/studio/setup.sh index 37d8154e59..5c393b9fbf 100755 --- a/studio/setup.sh +++ b/studio/setup.sh @@ -1224,6 +1224,7 @@ LLAMA_CPP_DIR="$UNSLOTH_HOME/llama.cpp" LLAMA_SERVER_BIN="$LLAMA_CPP_DIR/build/bin/llama-server" _NEED_LLAMA_SOURCE_BUILD=false _LLAMA_CPP_DEGRADED=false +_LLAMA_CPP_NO_SPACE=false _LLAMA_FORCE_COMPILE="${UNSLOTH_LLAMA_FORCE_COMPILE:-0}" _REQUESTED_LLAMA_TAG="${UNSLOTH_LLAMA_TAG:-${_DEFAULT_LLAMA_TAG}}" _HOST_SYSTEM="$(uname -s 2>/dev/null || true)" @@ -1451,6 +1452,13 @@ else fi substep "close Unsloth or other llama.cpp users and retry" exit 3 + elif [ "$_PREBUILT_STATUS" -eq 4 ]; then + step "llama.cpp" "not enough disk space to install llama.cpp" "$C_WARN" + print_llama_error_log "$_PREBUILT_LOG" + rm -f "$_PREBUILT_LOG" + substep "free up disk or move UNSLOTH_STUDIO_HOME/TMPDIR to a larger volume, then re-run" + _LLAMA_CPP_NO_SPACE=true + _has_local_llama_server "$LLAMA_CPP_DIR" || _LLAMA_CPP_DEGRADED=true else step "llama.cpp" "prebuilt install failed (continuing)" "$C_WARN" print_llama_error_log "$_PREBUILT_LOG" @@ -1946,7 +1954,14 @@ else --validate-install "$_BUILD_TMP" ) [ -n "$_SMOKE_KIND" ] && _SMOKE_CMD+=(--install-kind "$_SMOKE_KIND") - if ! run_quiet_no_exit "validate source llama.cpp" "${_SMOKE_CMD[@]}"; then + _SMOKE_RC=0 + run_quiet_no_exit "validate source llama.cpp" "${_SMOKE_CMD[@]}" || _SMOKE_RC=$? + # Exit 4 is a full disk, not a bad build: the CPU rebuild needs even + # more space, so keep what we already have. + if [ "$_SMOKE_RC" -eq 4 ]; then + substep "not enough disk space to validate the $_FB_LABEL build; keeping it" "$C_WARN" + _LLAMA_CPP_NO_SPACE=true + elif [ "$_SMOKE_RC" -ne 0 ]; then substep "$_FB_LABEL source build failed smoke test; retrying CPU build..." "$C_WARN" _TRY_METAL_CPU_FALLBACK=false rm -rf "$_BUILD_TMP/build" @@ -2003,8 +2018,10 @@ fi # end _SKIP_GGUF_BUILD check # An arm64 Linux GPU host source-builds for the GPU above. If that produced no # binary, install the fork's arm64 CPU prebuilt (app--linux-arm64-cpu.tar.gz) # instead of leaving the host without llama.cpp. --cpu-fallback drops the GPU -# attributes so the CPU bundle is selected rather than re-attempting CUDA. +# attributes so the CPU bundle is selected rather than re-attempting CUDA. Skipped +# on a full disk: the retry fails the same way and buries the hint. if [ "$_LLAMA_CPP_DEGRADED" = true ] \ + && [ "$_LLAMA_CPP_NO_SPACE" != true ] \ && [ "$_HOST_SYSTEM" = "Linux" ] \ && { [ "$_HOST_MACHINE" = "aarch64" ] || [ "$_HOST_MACHINE" = "arm64" ]; }; then substep "GPU source build unavailable; trying arm64 CPU prebuilt..." diff --git a/tests/studio/install/test_llama_prebuilt_no_space.py b/tests/studio/install/test_llama_prebuilt_no_space.py new file mode 100644 index 0000000000..90bfc7356d --- /dev/null +++ b/tests/studio/install/test_llama_prebuilt_no_space.py @@ -0,0 +1,391 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 +"""Out-of-disk handling in the llama.cpp prebuilt installer: ENOSPC classification +through exception chains, EXIT_NO_SPACE, and the advisory low-disk warning. Offline.""" + +from __future__ import annotations + +import errno +import importlib.util +import shutil +import sys +import urllib.error +from pathlib import Path + +import pytest + + +PACKAGE_ROOT = Path(__file__).resolve().parents[3] +MODULE_PATH = PACKAGE_ROOT / "studio" / "install_llama_prebuilt.py" +SPEC = importlib.util.spec_from_file_location("studio_install_llama_prebuilt", MODULE_PATH) +assert SPEC is not None and SPEC.loader is not None +INSTALL_LLAMA_PREBUILT = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = INSTALL_LLAMA_PREBUILT +SPEC.loader.exec_module(INSTALL_LLAMA_PREBUILT) + +M = INSTALL_LLAMA_PREBUILT +PrebuiltFallback = M.PrebuiltFallback +AssetChoice = M.AssetChoice +ApprovedReleaseChecksums = M.ApprovedReleaseChecksums + +GB = 1024**3 + + +def linux_host() -> "M.HostInfo": + return M.HostInfo( + system = "Linux", + machine = "x86_64", + is_windows = False, + is_linux = True, + is_macos = False, + is_x86_64 = True, + is_arm64 = False, + nvidia_smi = None, + driver_cuda_version = None, + compute_caps = [], + visible_cuda_devices = None, + has_physical_nvidia = False, + has_usable_nvidia = False, + ) + + +def choice(name: str, tag: str = "release-2") -> "M.AssetChoice": + return AssetChoice( + repo = "unslothai/llama.cpp", + tag = tag, + name = name, + url = f"https://example.com/{name}", + source_label = "published", + install_kind = "linux-cpu", + ) + + +def checksums(release_tag: str, llama_tag: str) -> "M.ApprovedReleaseChecksums": + return ApprovedReleaseChecksums( + repo = "unslothai/llama.cpp", + release_tag = release_tag, + upstream_tag = llama_tag, + source_commit = None, + artifacts = {}, + ) + + +def plan(llama_tag: str, release_tag: str, attempts) -> "M.InstallReleasePlan": + return M.InstallReleasePlan( + requested_tag = "latest", + llama_tag = llama_tag, + release_tag = release_tag, + attempts = attempts, + approved_checksums = checksums(release_tag, llama_tag), + ) + + +def fake_disk_usage(free_bytes: int): + def _usage(path): + return shutil._ntuple_diskusage(100 * GB, 100 * GB - free_bytes, free_bytes) + + return _usage + + +def install_harness(monkeypatch: pytest.MonkeyPatch, plans, *, free_bytes: int) -> list[str]: + """Wire install_prebuilt down to a fake per-candidate validation. Returns the + list of candidate names the run actually reached.""" + monkeypatch.setattr(M, "detect_host", lambda: linux_host()) + monkeypatch.setattr( + M, + "resolve_simple_install_release_plans", + lambda llama_tag, host, published_repo, published_release_tag: ("latest", plans), + ) + monkeypatch.setattr( + M, "download_validation_model", lambda probe_path, cache_path: probe_path.write_bytes(b"p") + ) + monkeypatch.setattr(M.shutil, "disk_usage", fake_disk_usage(free_bytes)) + monkeypatch.setattr(M, "existing_install_matches_plan", lambda *args, **kwargs: False) + monkeypatch.setattr(M, "existing_install_matches_choice", lambda *args, **kwargs: False) + monkeypatch.setattr(M, "activate_install_tree", lambda *args, **kwargs: None) + monkeypatch.setattr(M, "ensure_converter_scripts", lambda *args, **kwargs: None) + monkeypatch.setattr(M, "ensure_diffusion_visual_server", lambda *args, **kwargs: None) + monkeypatch.setattr(M, "collect_system_report", lambda *args, **kwargs: "report") + reached: list[str] = [] + monkeypatch.setattr( + M, "validate_prebuilt_choice", lambda attempt, *a, **k: reached.append(attempt.name) + ) + return reached + + +# ── the low-disk check is advisory, never fatal ── + + +def test_low_disk_warning_reports_the_starved_volume(tmp_path, monkeypatch): + monkeypatch.setattr(M.shutil, "disk_usage", fake_disk_usage(1 * GB)) + reason = M._low_disk_warning(tmp_path / "llama.cpp") + assert reason is not None and "low disk space for llama.cpp" in reason + + +def test_low_disk_warning_silent_when_roomy(tmp_path, monkeypatch): + monkeypatch.setattr(M.shutil, "disk_usage", fake_disk_usage(50 * GB)) + assert M._low_disk_warning(tmp_path / "llama.cpp") is None + + +def test_low_disk_warning_ignores_unstatable_paths(tmp_path, monkeypatch): + def _boom(path): + raise OSError(errno.EACCES, "permission denied") + + monkeypatch.setattr(M.shutil, "disk_usage", _boom) + assert M._low_disk_warning(tmp_path / "llama.cpp") is None + + +def test_low_disk_does_not_block_an_install_that_fits(tmp_path, monkeypatch, capsys): + """A 15 MB CPU bundle installs fine on a host with 3 GB free; the fixed + threshold must warn rather than reject it (and skip the source fallback).""" + install_dir = tmp_path / "llama.cpp" + install_dir.mkdir() + only = plan("b10079", "release-2", [choice("app-b10079-linux-x64-cpu.tar.gz")]) + reached = install_harness(monkeypatch, [only], free_bytes = 3 * GB) + + M.install_prebuilt(install_dir, "latest", "unslothai/llama.cpp", "") + + assert reached == ["app-b10079-linux-x64-cpu.tar.gz"] + captured = capsys.readouterr() + assert "low disk space for llama.cpp" in captured.out + captured.err + + +# ── ENOSPC classification ── + + +def test_classifies_direct_and_chained_enospc(): + assert M._environment_fatal_reason(OSError(errno.ENOSPC, "No space left on device")) + for wrap in ("cause", "context"): + try: + try: + raise OSError(errno.ENOSPC, "No space left on device") + except OSError as inner: + if wrap == "cause": + raise PrebuiltFallback("download failed") from inner + raise PrebuiltFallback("download failed") + except PrebuiltFallback as outer: + assert M._environment_fatal_reason(outer), wrap + + +def test_ignores_unrelated_errors_and_cycles(): + assert M._environment_fatal_reason(OSError(errno.EACCES, "denied")) is None + first, second = PrebuiltFallback("a"), PrebuiltFallback("b") + first.__cause__, second.__cause__ = second, first + assert M._environment_fatal_reason(first) is None + + +def test_suppressed_context_is_not_treated_as_disk_full(): + """`raise ... from None` means the earlier ENOSPC is unrelated.""" + try: + try: + raise OSError(errno.ENOSPC, "No space left on device") + except OSError: + raise PrebuiltFallback("checksum mismatch") from None + except PrebuiltFallback as outer: + assert M._environment_fatal_reason(outer) is None + + +def test_windows_disk_full_winerrors_are_classified(): + """CPython maps ERROR_DISK_FULL (112) to ENOSPC but has no case for + ERROR_HANDLE_DISK_FULL (39), which arrives as EINVAL.""" + for winerror, code in ((112, errno.ENOSPC), (39, errno.EINVAL)): + exc = OSError(code, "The disk is full") + exc.winerror = winerror + assert M._environment_fatal_reason(exc), winerror + + other = OSError(errno.EACCES, "sharing violation") + other.winerror = 32 + assert M._environment_fatal_reason(other) is None + + +def test_http_errors_in_the_chain_do_not_crash_the_classifier(): + """HTTPError is an OSError that proxies unknown attributes to a wrapped file + and raises KeyError, not AttributeError, on 3.9.""" + err = urllib.error.HTTPError("https://example.com/a", 404, "Not Found", {}, None) + assert M._environment_fatal_reason(err) is None + try: + try: + raise err + except urllib.error.HTTPError as inner: + raise PrebuiltFallback("mirror failed") from inner + except PrebuiltFallback as outer: + assert M._environment_fatal_reason(outer) is None + + +@pytest.mark.skipif(not hasattr(errno, "EDQUOT"), reason = "EDQUOT is POSIX only") +def test_quota_exhaustion_counts_as_out_of_space(): + """A quota'd home has free blocks this user cannot have, so the larger source + build is just as doomed. Reported as a quota so df does not mislead.""" + assert M._environment_fatal_reason(OSError(errno.EDQUOT, "Disk quota exceeded")) == ( + "disk quota exceeded" + ) + try: + try: + raise OSError(errno.EDQUOT, "Disk quota exceeded") + except OSError as inner: + raise PrebuiltFallback("bundle download failed") from inner + except PrebuiltFallback as outer: + assert M._environment_fatal_reason(outer) == "disk quota exceeded" + + +def test_a_bare_oserror_never_matches(): + """errno is None on a bare OSError, so it must not collide with a code.""" + assert M._environment_fatal_reason(OSError()) is None + assert M._environment_fatal_reason(shutil.Error("copy failed")) is None + + +def test_flattened_markers_are_not_matched_as_prefixes(): + """Bare "WinError 112" would also match WinError 1120; the brackets pin it.""" + assert ( + M._environment_fatal_reason( + shutil.Error("[('a', 'b', '[WinError 1120] a serial write completed')]") + ) + is None + ) + assert ( + M._environment_fatal_reason( + shutil.Error(f"[('a', 'b', '[Errno {errno.ENOSPC}0] not a real code')]") + ) + is None + ) + + +def test_windows_flattened_disk_full_text_is_classified(): + """copytree stringifies the per-file OSError, and on Windows str(OSError) + prints [WinError 112] and never [Errno 28] (confirmed on a real NTFS volume).""" + flattened = ( + "[('D:\\\\a\\\\src\\\\big.bin', 'T:\\\\dst\\\\big.bin', " + "'[WinError 112] There is not enough space on the disk')]" + ) + assert M._environment_fatal_reason(shutil.Error(flattened)) + assert M._environment_fatal_reason( + shutil.Error("[('a', 'b', '[WinError 39] The disk is full')]") + ) + assert ( + M._environment_fatal_reason(shutil.Error("[('a', 'b', '[WinError 32] sharing violation')]")) + is None + ) + + +def test_validate_install_mode_exits_no_space(tmp_path, monkeypatch): + """setup.sh reacts to a failed staged validation by deleting the finished GPU + build and starting a CPU rebuild, which needs more of the space that ran out.""" + + def boom(*args, **kwargs): + try: + raise OSError(errno.ENOSPC, "No space left on device") + except OSError as inner: + raise PrebuiltFallback("validation model unavailable") from inner + + monkeypatch.setattr(M, "validate_existing_install", boom) + monkeypatch.setattr( + sys, "argv", ["install_llama_prebuilt.py", "--validate-install", str(tmp_path)] + ) + + with pytest.raises(SystemExit) as caught: + M.main() + assert caught.value.code == M.EXIT_NO_SPACE + + +def test_validate_install_mode_still_falls_back_on_ordinary_failure(tmp_path, monkeypatch): + monkeypatch.setattr( + M, + "validate_existing_install", + lambda *a, **k: (_ for _ in ()).throw(PrebuiltFallback("llama-server crashed")), + ) + monkeypatch.setattr( + sys, "argv", ["install_llama_prebuilt.py", "--validate-install", str(tmp_path)] + ) + + with pytest.raises(SystemExit) as caught: + M.main() + assert caught.value.code == M.EXIT_FALLBACK + + +def test_classifies_enospc_hidden_in_a_shutil_error(tmp_path): + """copytree stringifies the per-file OSError, so errno and the chain are gone.""" + src = tmp_path / "src" / "sub" + src.mkdir(parents = True) + (src / "f").write_text("x", encoding = "utf-8") + + def boom(*args, **kwargs): + raise OSError(errno.ENOSPC, "No space left on device") + + with pytest.raises(shutil.Error) as caught: + shutil.copytree(tmp_path / "src", tmp_path / "dst", copy_function = boom) + + assert caught.value.errno is None + assert M._environment_fatal_reason(caught.value) + + +def test_source_tree_enospc_is_not_masked_by_a_later_mirror_error(tmp_path, monkeypatch): + """A full disk fails every mirror, so the first ENOSPC must win over a 404.""" + calls: list[str] = [] + + def fake_download( + url, + path, + *, + expected_sha256 = None, + label = None, + ): + calls.append(url) + if len(calls) == 1: + raise OSError(errno.ENOSPC, "No space left on device") + raise urllib.error.HTTPError(url, 404, "Not Found", {}, None) + + monkeypatch.setattr(M, "download_file_verified", fake_download) + + with pytest.raises(PrebuiltFallback) as caught: + M.hydrate_source_tree( + "deadbeef", + tmp_path / "install", + tmp_path, + source_repo = "unslothai/llama.cpp", + expected_sha256 = None, + exact_source = True, + asset_url = "https://example.com/llama.cpp-source.tar.gz", + ) + + assert len(calls) == 1, f"stopped after the first ENOSPC, tried: {calls}" + assert M._environment_fatal_reason(caught.value) + + +# ── exit codes ── + + +def test_enospc_exits_no_space_without_trying_older_releases(tmp_path, monkeypatch): + install_dir = tmp_path / "llama.cpp" + install_dir.mkdir() + newer = plan("b9002", "release-2", [choice("app-b9002-linux-x64-cpu.tar.gz")]) + older = plan("b9001", "release-1", [choice("app-b9001-linux-x64-cpu.tar.gz", "release-1")]) + reached = install_harness(monkeypatch, [newer, older], free_bytes = 50 * GB) + + def enospc(attempt, *args, **kwargs): + reached.append(attempt.name) + raise OSError(errno.ENOSPC, "No space left on device") + + monkeypatch.setattr(M, "validate_prebuilt_choice", enospc) + + with pytest.raises(SystemExit) as caught: + M.install_prebuilt(install_dir, "latest", "unslothai/llama.cpp", "") + + assert caught.value.code == M.EXIT_NO_SPACE + assert reached == ["app-b9002-linux-x64-cpu.tar.gz"] + + +def test_ordinary_failure_still_exits_fallback(tmp_path, monkeypatch): + install_dir = tmp_path / "llama.cpp" + install_dir.mkdir() + only = plan("b9002", "release-2", [choice("app-b9002-linux-x64-cpu.tar.gz")]) + install_harness(monkeypatch, [only], free_bytes = 50 * GB) + monkeypatch.setattr( + M, + "validate_prebuilt_choice", + lambda *a, **k: (_ for _ in ()).throw(PrebuiltFallback("checksum mismatch")), + ) + + with pytest.raises(SystemExit) as caught: + M.install_prebuilt(install_dir, "latest", "unslothai/llama.cpp", "") + + assert caught.value.code == M.EXIT_FALLBACK