From 23cebfaf9850678a97ae14cbf17e51000b745200 Mon Sep 17 00:00:00 2001 From: Wasim Yousef Said Date: Mon, 11 May 2026 16:24:01 +0200 Subject: [PATCH 01/25] Add Studio web update banner and release version display (#5308) * Add Studio web update and release version display * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Show package version in Studio settings * Break training unload guard barrel cycle --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Lee Jackson <130007945+Imagineer99@users.noreply.github.com> --- build.sh | 31 +- scripts/stamp_studio_release.py | 257 ++++++++++++ studio/backend/main.py | 19 + studio/backend/requirements/studio.txt | 1 + studio/backend/utils/_studio_release_build.py | 11 + studio/backend/utils/studio_version.py | 92 +++++ studio/backend/utils/update_status.py | 374 ++++++++++++++++++ studio/frontend/src/app/provider.tsx | 17 +- .../src/components/web/update-banner.tsx | 136 +++++++ .../components/update-studio-instructions.tsx | 250 +++++++++--- .../src/features/settings/tabs/about-tab.tsx | 136 ++++++- .../hooks/use-training-unload-guard.ts | 10 +- .../frontend/src/features/training/index.ts | 6 +- .../src/hooks/use-web-update-check.ts | 158 ++++++++ 14 files changed, 1426 insertions(+), 72 deletions(-) create mode 100755 scripts/stamp_studio_release.py create mode 100644 studio/backend/utils/_studio_release_build.py create mode 100644 studio/backend/utils/studio_version.py create mode 100644 studio/backend/utils/update_status.py create mode 100644 studio/frontend/src/components/web/update-banner.tsx create mode 100644 studio/frontend/src/hooks/use-web-update-check.ts diff --git a/build.sh b/build.sh index cf8aa02910..1558dca240 100644 --- a/build.sh +++ b/build.sh @@ -2,6 +2,10 @@ set -euo pipefail +# PyPI/Studio release publishing must use `./build.sh publish` (or an +# equivalent stamp -> build -> verify-dist -> upload flow) so packaged Studio +# artifacts include the display-only Studio release version. + # 1. Build frontend (Vite outputs to dist/) cd studio/frontend @@ -70,10 +74,33 @@ cd ../.. # 2. Clean old artifacts rm -rf build dist *.egg-info -# 3. Build wheel +# 3. Stamp display-only Studio release metadata for packaged builds. +_STUDIO_BUILD_INFO="studio/backend/utils/_studio_release_build.py" +_STUDIO_BUILD_INFO_BACKUP="$(mktemp)" +cp "$_STUDIO_BUILD_INFO" "$_STUDIO_BUILD_INFO_BACKUP" +_restore_studio_build_info() { + cp "$_STUDIO_BUILD_INFO_BACKUP" "$_STUDIO_BUILD_INFO" 2>/dev/null || true + rm -f "$_STUDIO_BUILD_INFO_BACKUP" +} +trap _restore_studio_build_info EXIT + +if [ "${1:-}" = "publish" ]; then + STUDIO_STAMPED_VERSION="$(python scripts/stamp_studio_release.py --require-release)" +else + STUDIO_STAMPED_VERSION="$(python scripts/stamp_studio_release.py)" +fi + +# 4. Build wheel/sdist python -m build -# 4. Optionally publish +if [ "${1:-}" = "publish" ]; then + python scripts/stamp_studio_release.py --verify-dist dist --expected "$STUDIO_STAMPED_VERSION" +fi + +_restore_studio_build_info +trap - EXIT + +# 5. Optionally publish if [ "${1:-}" = "publish" ]; then python -m twine upload dist/* fi diff --git a/scripts/stamp_studio_release.py b/scripts/stamp_studio_release.py new file mode 100755 index 0000000000..5547b95137 --- /dev/null +++ b/scripts/stamp_studio_release.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Stamp and verify display-only Studio release metadata for builds.""" + +from __future__ import annotations + +import argparse +import os +import re +import subprocess +import sys +import tarfile +import zipfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +BUILD_INFO_PATH = ( + REPO_ROOT / "studio" / "backend" / "utils" / "_studio_release_build.py" +) +BUILD_INFO_SUFFIX = "studio/backend/utils/_studio_release_build.py" +VERSION_RE = re.compile(r"^v\d+\.\d+\.\d+(?:-[0-9A-Za-z.][0-9A-Za-z.-]*)?$") +GIT_DESCRIBE_SUFFIX_RE = re.compile(r"-\d+-g[0-9A-Fa-f]+(?:-dirty)?$") +MAX_VERSION_LENGTH = 64 +PLACEHOLDER = """# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +\"\"\"Build-stamped Studio release metadata. + +Release builds may rewrite this module in the build workspace before creating +Python artifacts. Keep the committed value neutral so source checkouts do not +accidentally report a stale release tag. +\"\"\" + +STUDIO_RELEASE_VERSION = None +""" + + +def is_valid_version(value: object) -> bool: + if not isinstance(value, str): + return False + version = value.strip() + if not version or len(version) > MAX_VERSION_LENGTH: + return False + if version.endswith("-dirty") or GIT_DESCRIBE_SUFFIX_RE.search(version): + return False + return VERSION_RE.fullmatch(version) is not None + + +def _exact_git_tag() -> str | None: + try: + result = subprocess.run( + [ + "git", + "describe", + "--tags", + "--exact-match", + "--match", + "v[0-9]*", + "HEAD", + ], + cwd = REPO_ROOT, + check = False, + stdout = subprocess.PIPE, + stderr = subprocess.DEVNULL, + text = True, + timeout = 2.0, + ) + except (OSError, subprocess.TimeoutExpired): + return None + if result.returncode != 0: + return None + tag = result.stdout.strip() + return tag if is_valid_version(tag) else None + + +def _git_worktree_is_dirty() -> bool: + try: + result = subprocess.run( + ["git", "status", "--porcelain"], + cwd = REPO_ROOT, + check = False, + stdout = subprocess.PIPE, + stderr = subprocess.DEVNULL, + text = True, + timeout = 2.0, + ) + except (OSError, subprocess.TimeoutExpired): + return True + if result.returncode != 0: + return True + return bool(result.stdout.strip()) + + +def _github_tag() -> str | None: + if os.environ.get("GITHUB_REF_TYPE") != "tag": + return None + github_ref = os.environ.get("GITHUB_REF_NAME", "").strip() + return github_ref or None + + +def resolve_version() -> tuple[str | None, str]: + env_version = os.environ.get("UNSLOTH_STUDIO_RELEASE_VERSION", "").strip() + if env_version: + return (env_version, "UNSLOTH_STUDIO_RELEASE_VERSION") + + github_ref = _github_tag() + if github_ref: + return (github_ref, "GITHUB_REF_NAME") + + git_tag = _exact_git_tag() + if git_tag: + return (git_tag, "exact git tag") + + return (None, "none") + + +def build_info_source(version: str | None) -> str: + literal = repr(version) if version is not None else "None" + return f'''# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Build-stamped Studio release metadata.""" + +STUDIO_RELEASE_VERSION = {literal} +''' + + +def _env_version_conflicts(version: str) -> list[tuple[str, str]]: + conflicts: list[tuple[str, str]] = [] + github_ref = _github_tag() + if github_ref and is_valid_version(github_ref) and github_ref != version: + conflicts.append(("GITHUB_REF_NAME", github_ref)) + + git_tag = _exact_git_tag() + if git_tag and git_tag != version: + conflicts.append(("exact git tag", git_tag)) + + return conflicts + + +def stamp(require_release: bool) -> int: + version, source = resolve_version() + if version is not None and not is_valid_version(version): + print( + f"Invalid Studio release version from {source}: {version!r}", + file = sys.stderr, + ) + return 2 + + if version is not None and source == "UNSLOTH_STUDIO_RELEASE_VERSION": + conflicts = _env_version_conflicts(version) + if conflicts: + details = ", ".join(f"{name}={value!r}" for name, value in conflicts) + print( + "UNSLOTH_STUDIO_RELEASE_VERSION does not match available " + f"release tag metadata: {details}", + file = sys.stderr, + ) + return 2 + + if require_release and source == "exact git tag" and _git_worktree_is_dirty(): + print( + "Refusing to publish from a dirty exact-tag checkout. Set " + "UNSLOTH_STUDIO_RELEASE_VERSION explicitly from release automation " + "or publish from a clean tag checkout.", + file = sys.stderr, + ) + return 2 + + if version is None: + if require_release: + print( + "No Studio release version available. Set " + "UNSLOTH_STUDIO_RELEASE_VERSION, build from a GitHub tag, " + "or run from an exact local Studio release tag.", + file = sys.stderr, + ) + return 2 + BUILD_INFO_PATH.write_text(PLACEHOLDER, encoding = "utf-8") + print("dev") + return 0 + + BUILD_INFO_PATH.write_text(build_info_source(version), encoding = "utf-8") + print(f"Stamping Studio release version {version} from {source}", file = sys.stderr) + print(version) + return 0 + + +def _read_wheel_member(path: Path) -> str | None: + with zipfile.ZipFile(path) as archive: + for name in archive.namelist(): + if name.endswith(BUILD_INFO_SUFFIX): + return archive.read(name).decode("utf-8") + return None + + +def _read_sdist_member(path: Path) -> str | None: + with tarfile.open(path) as archive: + for member in archive.getmembers(): + if member.name.endswith(BUILD_INFO_SUFFIX): + extracted = archive.extractfile(member) + if extracted is None: + return None + return extracted.read().decode("utf-8") + return None + + +def verify_dist(expected: str, dist_dir: Path) -> int: + if not is_valid_version(expected): + print(f"Invalid expected Studio release version: {expected!r}", file = sys.stderr) + return 2 + + artifacts = list(dist_dir.glob("*.whl")) + list(dist_dir.glob("*.tar.gz")) + if not artifacts: + print(f"No wheel or sdist artifacts found in {dist_dir}", file = sys.stderr) + return 2 + + expected_line = f"STUDIO_RELEASE_VERSION = {expected!r}" + failures: list[str] = [] + for artifact in artifacts: + if artifact.suffix == ".whl": + content = _read_wheel_member(artifact) + else: + content = _read_sdist_member(artifact) + if content is None: + failures.append(f"{artifact.name}: missing {BUILD_INFO_SUFFIX}") + elif expected_line not in content: + failures.append(f"{artifact.name}: Studio release version mismatch") + + if failures: + for failure in failures: + print(failure, file = sys.stderr) + return 2 + + print(f"Verified Studio release version {expected} in {len(artifacts)} artifact(s)") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description = __doc__) + parser.add_argument("--require-release", action = "store_true") + parser.add_argument("--verify-dist", type = Path) + parser.add_argument("--expected") + args = parser.parse_args() + + if args.verify_dist is not None: + if not args.expected: + parser.error("--verify-dist requires --expected") + return verify_dist(args.expected, args.verify_dist) + + return stamp(require_release = args.require_release) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/studio/backend/main.py b/studio/backend/main.py index 633b112dc8..650a488212 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -134,6 +134,11 @@ import utils.hardware.hardware as _hw_module from utils.cache_cleanup import clear_unsloth_compiled_cache from utils.native_path_leases import native_path_leases_supported +from utils.update_status import ( + get_studio_install_source_status, + get_studio_update_status, +) +from utils.studio_version import get_studio_version def get_unsloth_version() -> str: @@ -155,6 +160,7 @@ def get_unsloth_version() -> str: UNSLOTH_VERSION = get_unsloth_version() +STUDIO_VERSION = get_studio_version() @asynccontextmanager @@ -296,6 +302,7 @@ async def health_check(): "timestamp": datetime.now().isoformat(), "service": "Unsloth UI Backend", "version": UNSLOTH_VERSION, + "studio_version": STUDIO_VERSION, "device_type": device_type, "chat_only": _hw_module.CHAT_ONLY, "desktop_protocol_version": 1, @@ -308,6 +315,18 @@ async def health_check(): } +@app.get("/api/studio/install-source") +def studio_install_source(_current_subject: str = Depends(get_current_subject)): + """Return source-aware install metadata without remote update checks.""" + return get_studio_install_source_status(UNSLOTH_VERSION) + + +@app.get("/api/studio/update-status") +def studio_update_status(_current_subject: str = Depends(get_current_subject)): + """Return source-aware manual update status for browser-served Studio.""" + return get_studio_update_status(UNSLOTH_VERSION) + + @app.post("/api/shutdown") async def shutdown_server( request: Request, diff --git a/studio/backend/requirements/studio.txt b/studio/backend/requirements/studio.txt index 186ba82fe0..1bf751c368 100644 --- a/studio/backend/requirements/studio.txt +++ b/studio/backend/requirements/studio.txt @@ -3,6 +3,7 @@ typer fastapi uvicorn pydantic +packaging matplotlib pandas nest_asyncio diff --git a/studio/backend/utils/_studio_release_build.py b/studio/backend/utils/_studio_release_build.py new file mode 100644 index 0000000000..267197a202 --- /dev/null +++ b/studio/backend/utils/_studio_release_build.py @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Build-stamped Studio release metadata. + +Release builds may rewrite this module in the build workspace before creating +Python artifacts. Keep the committed value neutral so source checkouts do not +accidentally report a stale release tag. +""" + +STUDIO_RELEASE_VERSION = None diff --git a/studio/backend/utils/studio_version.py b/studio/backend/utils/studio_version.py new file mode 100644 index 0000000000..70059f8a3c --- /dev/null +++ b/studio/backend/utils/studio_version.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Network-free Studio release version resolution for display-only UI.""" + +from __future__ import annotations + +import re +import subprocess +from pathlib import Path + +from utils import _studio_release_build + +_DEV_VERSION = "dev" +_GIT_TIMEOUT_SECONDS = 1.0 +_STUDIO_TAG_RE = re.compile(r"^v\d+\.\d+\.\d+(?:-[0-9A-Za-z.][0-9A-Za-z.-]*)?$") +_GIT_DESCRIBE_SUFFIX_RE = re.compile(r"-\d+-g[0-9A-Fa-f]+(?:-dirty)?$") +_MAX_VERSION_LENGTH = 64 + + +def is_valid_studio_release_version(value: object) -> bool: + """Return True for Studio release tags such as ``v0.1.39-beta``.""" + if not isinstance(value, str): + return False + version = value.strip() + if not version or len(version) > _MAX_VERSION_LENGTH: + return False + if version.endswith("-dirty") or _GIT_DESCRIBE_SUFFIX_RE.search(version): + return False + return _STUDIO_TAG_RE.fullmatch(version) is not None + + +def _repo_root() -> Path: + return Path(__file__).resolve().parents[3] + + +def _path_is_in_site_packages(path: Path) -> bool: + return any(part in {"site-packages", "dist-packages"} for part in path.parts) + + +def _is_source_checkout(repo_root: Path) -> bool: + return (repo_root / ".git").exists() and not _path_is_in_site_packages( + Path(__file__).resolve() + ) + + +def _exact_git_studio_tag(repo_root: Path) -> str | None: + try: + result = subprocess.run( + [ + "git", + "describe", + "--tags", + "--exact-match", + "--match", + "v[0-9]*", + "HEAD", + ], + cwd = repo_root, + check = False, + stdout = subprocess.PIPE, + stderr = subprocess.DEVNULL, + text = True, + timeout = _GIT_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.TimeoutExpired): + return None + + if result.returncode != 0: + return None + + tag = result.stdout.strip() + return tag if is_valid_studio_release_version(tag) else None + + +def get_studio_version(repo_root: Path | None = None) -> str: + """Return the installed Studio release tag for display, or ``dev``. + + This value is intentionally separate from the PyPI ``unsloth`` package + version used by update checks. It never performs network requests. + """ + resolved_repo_root = repo_root or _repo_root() + + if _is_source_checkout(resolved_repo_root): + git_tag = _exact_git_studio_tag(resolved_repo_root) + return git_tag if git_tag is not None else _DEV_VERSION + + stamped_version = _studio_release_build.STUDIO_RELEASE_VERSION + if is_valid_studio_release_version(stamped_version): + return stamped_version.strip() + + return _DEV_VERSION diff --git a/studio/backend/utils/update_status.py b/studio/backend/utils/update_status.py new file mode 100644 index 0000000000..9142203a69 --- /dev/null +++ b/studio/backend/utils/update_status.py @@ -0,0 +1,374 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Web update status helpers for browser-served Unsloth Studio. + +This module is intentionally side-effect light: no network work happens at +import time or from /api/health. The PyPI check is lazy, cached, and only used +for normal PyPI-managed installs. +""" + +from __future__ import annotations + +import json +import os +import threading +import time +import urllib.request +from dataclasses import dataclass +from datetime import datetime, timezone +from importlib.metadata import PackageNotFoundError, distribution +from pathlib import Path +from typing import Any + +from packaging.version import InvalidVersion, Version + +PACKAGE_NAME = "unsloth" +PYPI_JSON_URL = "https://pypi.org/pypi/unsloth/json" +PYPI_TIMEOUT_SECONDS = 3 +PYPI_RESPONSE_MAX_BYTES = 5 * 1024 * 1024 +PYPI_SUCCESS_TTL_SECONDS = 12 * 60 * 60 +PYPI_FAILURE_TTL_SECONDS = 60 * 60 +RELEASE_NOTES_URL = "https://unsloth.ai/docs/new/changelog" +DISABLE_ENV_VAR = "UNSLOTH_DISABLE_UPDATE_CHECK" + +LOCAL_INSTALL_SOURCES = {"editable", "local_path", "vcs", "local_repo"} + + +@dataclass(frozen = True) +class LatestVersionResult: + latest_version: str | None + checked_at: str + reason: str | None = None + error: str | None = None + + +@dataclass +class _LatestVersionCacheEntry: + result: LatestVersionResult + expires_at: float + + +_cache_condition = threading.Condition() +_latest_version_cache: _LatestVersionCacheEntry | None = None +_latest_version_fetching = False + + +def reset_update_status_cache() -> None: + """Clear the in-process PyPI cache. Intended for tests.""" + global _latest_version_cache, _latest_version_fetching + with _cache_condition: + _latest_version_cache = None + _latest_version_fetching = False + _cache_condition.notify_all() + + +def detect_install_source() -> str: + """Return a coarse install source without exposing local paths. + + Sources are intentionally conservative. PEP 610 local/vcs metadata wins. + Legacy source installs are treated as local only when package files resolve + outside site-packages/dist-packages and under a Git checkout. + """ + try: + dist = distribution(PACKAGE_NAME) + except PackageNotFoundError: + return ( + "local_repo" + if _path_has_git_parent(_repo_root_from_this_file()) + else "unknown" + ) + + try: + direct_url = dist.read_text("direct_url.json") + except Exception: + return "unknown" + if direct_url: + return _source_from_direct_url(direct_url) + + for package_path in _distribution_package_paths(dist): + if not _path_is_under_python_package_dir(package_path) and _path_has_git_parent( + package_path + ): + return "local_repo" + + return "pypi" + + +def get_studio_install_source_status(current_version: str) -> dict[str, Any]: + """Return install-source metadata without remote update checks.""" + install_source = detect_install_source() + reason = None + if install_source in LOCAL_INSTALL_SOURCES: + reason = "local_source" + elif install_source == "unknown": + reason = "unknown_source" + + return _status_response( + current_version = current_version, + latest_version = None, + install_source = install_source, + reason = reason, + ) + + +def get_studio_update_status(current_version: str) -> dict[str, Any]: + """Return public, read-only update status for the web UI.""" + install_source = detect_install_source() + + if os.environ.get(DISABLE_ENV_VAR) == "1": + return _status_response( + current_version = current_version, + latest_version = None, + install_source = install_source, + reason = "disabled", + ) + + if install_source in LOCAL_INSTALL_SOURCES: + return _status_response( + current_version = current_version, + latest_version = None, + install_source = install_source, + reason = "local_source", + ) + + if install_source != "pypi": + return _status_response( + current_version = current_version, + latest_version = None, + install_source = install_source, + reason = "unknown_source", + ) + + current = _parse_current_version(current_version) + if current is None: + return _status_response( + current_version = current_version, + latest_version = None, + install_source = install_source, + reason = "invalid_current_version" + if current_version != "dev" + else "dev_build", + ) + latest_result = get_latest_pypi_version() + if latest_result.latest_version is None: + return _status_response( + current_version = current_version, + latest_version = None, + install_source = install_source, + reason = latest_result.reason or "offline", + error = latest_result.error, + checked_at = latest_result.checked_at, + ) + + try: + latest = Version(latest_result.latest_version) + except InvalidVersion: + return _status_response( + current_version = current_version, + latest_version = latest_result.latest_version, + install_source = install_source, + reason = "invalid_latest_version", + error = "PyPI returned an invalid version.", + checked_at = latest_result.checked_at, + ) + + if latest > current: + return _status_response( + current_version = current_version, + latest_version = latest_result.latest_version, + install_source = install_source, + update_available = True, + can_show_web_notification = True, + checked_at = latest_result.checked_at, + ) + + return _status_response( + current_version = current_version, + latest_version = latest_result.latest_version, + install_source = install_source, + reason = "current_not_older", + checked_at = latest_result.checked_at, + ) + + +def get_latest_pypi_version() -> LatestVersionResult: + """Return the latest PyPI version using a small in-process TTL cache.""" + global _latest_version_cache, _latest_version_fetching + + while True: + now = time.monotonic() + with _cache_condition: + if _latest_version_cache and _latest_version_cache.expires_at > now: + return _latest_version_cache.result + if not _latest_version_fetching: + _latest_version_fetching = True + break + _cache_condition.wait(timeout = PYPI_TIMEOUT_SECONDS + 1) + + try: + result = _fetch_latest_pypi_version() + except Exception: + result = LatestVersionResult( + latest_version = None, + checked_at = _utc_now_iso(), + reason = "offline", + error = "Could not check PyPI update metadata.", + ) + + ttl = ( + PYPI_SUCCESS_TTL_SECONDS if result.latest_version else PYPI_FAILURE_TTL_SECONDS + ) + with _cache_condition: + _latest_version_cache = _LatestVersionCacheEntry( + result = result, + expires_at = time.monotonic() + ttl, + ) + _latest_version_fetching = False + _cache_condition.notify_all() + return result + + +def _fetch_latest_pypi_version() -> LatestVersionResult: + checked_at = _utc_now_iso() + request = urllib.request.Request( + PYPI_JSON_URL, + headers = {"User-Agent": "unsloth-studio-update-check"}, + ) + + try: + with urllib.request.urlopen(request, timeout = PYPI_TIMEOUT_SECONDS) as response: + body = response.read(PYPI_RESPONSE_MAX_BYTES + 1) + if len(body) > PYPI_RESPONSE_MAX_BYTES: + return LatestVersionResult( + latest_version = None, + checked_at = checked_at, + reason = "malformed_response", + error = "PyPI returned oversized update metadata.", + ) + payload = json.loads(body.decode("utf-8")) + except json.JSONDecodeError: + return LatestVersionResult( + latest_version = None, + checked_at = checked_at, + reason = "malformed_response", + error = "PyPI returned malformed update metadata.", + ) + except OSError: + return LatestVersionResult( + latest_version = None, + checked_at = checked_at, + reason = "offline", + error = "Could not reach PyPI for update metadata.", + ) + + latest = ( + payload.get("info", {}).get("version") if isinstance(payload, dict) else None + ) + if not isinstance(latest, str) or not latest.strip(): + return LatestVersionResult( + latest_version = None, + checked_at = checked_at, + reason = "malformed_response", + error = "PyPI update metadata did not include a version.", + ) + + return LatestVersionResult(latest_version = latest.strip(), checked_at = checked_at) + + +def _status_response( + *, + current_version: str, + latest_version: str | None, + install_source: str, + reason: str | None = None, + error: str | None = None, + update_available: bool = False, + can_show_web_notification: bool = False, + checked_at: str | None = None, +) -> dict[str, Any]: + return { + "current_version": current_version, + "latest_version": latest_version, + "update_available": update_available, + "install_source": install_source, + "can_show_web_notification": can_show_web_notification, + "release_notes_url": RELEASE_NOTES_URL, + "checked_at": checked_at or _utc_now_iso(), + "reason": reason, + "error": error, + } + + +def _source_from_direct_url(direct_url: str) -> str: + try: + payload = json.loads(direct_url) + except json.JSONDecodeError: + return "unknown" + + if not isinstance(payload, dict): + return "unknown" + + dir_info = payload.get("dir_info") + if isinstance(dir_info, dict) and dir_info.get("editable") is True: + return "editable" + + if isinstance(payload.get("vcs_info"), dict): + return "vcs" + + url = payload.get("url") + if isinstance(url, str) and url.startswith("file:"): + return "local_path" + + return "unknown" + + +def _distribution_package_paths(dist: Any) -> list[Path]: + paths: list[Path] = [] + files = getattr(dist, "files", None) or [] + for file in files: + text = str(file) + if not text.startswith(("unsloth/", "unsloth_cli/", "studio/")): + continue + try: + paths.append(Path(dist.locate_file(file)).resolve()) + except OSError: + continue + return paths + + +def _path_is_under_python_package_dir(path: Path) -> bool: + return any(part in {"site-packages", "dist-packages"} for part in path.parts) + + +def _path_has_git_parent(path: Path) -> bool: + for candidate in (path, *path.parents): + if (candidate / ".git").exists(): + return True + return False + + +def _repo_root_from_this_file() -> Path: + # update_status.py -> utils -> backend -> studio -> repo root + try: + return Path(__file__).resolve().parents[3] + except IndexError: + return Path(__file__).resolve().parent + + +def _parse_current_version(current_version: str) -> Version | None: + if current_version == "dev": + return None + try: + return Version(current_version) + except InvalidVersion: + return None + + +def _utc_now_iso() -> str: + return ( + datetime.now(timezone.utc) + .replace(microsecond = 0) + .isoformat() + .replace("+00:00", "Z") + ) diff --git a/studio/frontend/src/app/provider.tsx b/studio/frontend/src/app/provider.tsx index 62e78b809a..fe3173aa41 100644 --- a/studio/frontend/src/app/provider.tsx +++ b/studio/frontend/src/app/provider.tsx @@ -9,6 +9,7 @@ import { shouldUseCustomWindowTitlebar, } from "@/components/tauri/window-titlebar"; import { Toaster } from "@/components/ui/sonner"; +import { WebUpdateBanner } from "@/components/web/update-banner"; import { getTauriAuthFailure, tauriAutoAuth } from "@/features/auth"; import { NativeIntentDrain } from "@/features/native-intents/native-intent-drain"; import { useTauriBackend, type BackendStatus } from "@/hooks/use-tauri-backend"; @@ -154,6 +155,13 @@ const HIDDEN_TITLEBAR_SIDEBAR_ROUTES = new Set([ "/signup", ]); +const WEB_UPDATE_HIDDEN_ROUTES = new Set([ + "/onboarding", + "/login", + "/change-password", + "/signup", +]); + function TauriWrapper({ children }: { children: ReactNode }) { const pathname = useRouterState({ select: (s) => s.location.pathname }); const { @@ -234,7 +242,14 @@ function TauriWrapper({ children }: { children: ReactNode }) { return () => { disposed = true; }; }, [status, desktopAuthRetry]); - if (!isTauri) return <>{children}; + if (!isTauri) { + return ( + <> + {children} + + + ); + } const showApp = status === "running" && desktopAuthReady; const startupStatus = status === "running" ? "starting" : status; diff --git a/studio/frontend/src/components/web/update-banner.tsx b/studio/frontend/src/components/web/update-banner.tsx new file mode 100644 index 0000000000..685a8aa3dc --- /dev/null +++ b/studio/frontend/src/components/web/update-banner.tsx @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { Button } from "@/components/ui/button"; +import { useWebUpdateCheck } from "@/hooks/use-web-update-check"; +import { isTauri } from "@/lib/api-base"; +import { copyToClipboard } from "@/lib/copy-to-clipboard"; +import { AnimatePresence, motion } from "motion/react"; +import { type ReactElement, useEffect, useRef, useState } from "react"; + +const STUDIO_UPDATE_CMD = "unsloth studio update"; +const RELEASE_NOTES_URL = "https://unsloth.ai/docs/new/changelog"; +const EASE_OUT_QUART: [number, number, number, number] = [0.165, 0.84, 0.44, 1]; + +interface WebUpdateBannerProps { + enabled?: boolean; +} + +export function WebUpdateBanner({ + enabled = true, +}: WebUpdateBannerProps): ReactElement | null { + const { status, dismiss } = useWebUpdateCheck({ enabled }); + const [copiedVersion, setCopiedVersion] = useState(null); + const dismissTimerRef = useRef | null>(null); + + useEffect(() => { + return () => { + if (dismissTimerRef.current) { + clearTimeout(dismissTimerRef.current); + } + }; + }, []); + + if (isTauri) { + return null; + } + + async function handleCopyCommand() { + if (!(await copyToClipboard(STUDIO_UPDATE_CMD))) { + return; + } + setCopiedVersion(status?.latestVersion ?? null); + if (dismissTimerRef.current) { + clearTimeout(dismissTimerRef.current); + } + dismissTimerRef.current = setTimeout(() => dismiss(), 900); + } + + return ( + + {status ? ( + +
+ + +
+ +
+

+ Package update available: {status.latestVersion} +

+

+ Installed package: {status.currentVersion}. To update Studio, + run this in your terminal, then restart Studio. +

+
+
+ +
+ + + +
+
+ + ) : null} + + ); +} diff --git a/studio/frontend/src/features/settings/components/update-studio-instructions.tsx b/studio/frontend/src/features/settings/components/update-studio-instructions.tsx index 4d0b87981b..e4cdccd2d7 100644 --- a/studio/frontend/src/features/settings/components/update-studio-instructions.tsx +++ b/studio/frontend/src/features/settings/components/update-studio-instructions.tsx @@ -1,8 +1,8 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -import { cn } from "@/lib/utils"; import { copyToClipboard } from "@/lib/copy-to-clipboard"; +import { cn } from "@/lib/utils"; import { Copy01Icon, Tick02Icon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { AnimatePresence, motion, useReducedMotion } from "motion/react"; @@ -14,11 +14,42 @@ const STUDIO_UPDATE_FALLBACK_UNIX_CMD = "curl -fsSL https://unsloth.ai/install.sh | sh"; const STUDIO_UPDATE_FALLBACK_WINDOWS_CMD = "irm https://unsloth.ai/install.ps1 | iex"; +const STUDIO_LOCAL_PULL_CMD = "git pull --ff-only"; +const STUDIO_LOCAL_UPDATE_CMD = "unsloth studio update --local"; +const STUDIO_LOCAL_FALLBACK_UNIX_CMD = "./install.sh --local"; +const STUDIO_LOCAL_FALLBACK_WINDOWS_CMD = ".\\install.ps1 --local"; export type UpdateShell = "windows" | "unix"; +export type UpdateInstallSource = + | "pypi" + | "editable" + | "local_path" + | "vcs" + | "local_repo" + | "unknown"; +type UpdateInstallSourceState = UpdateInstallSource | "loading"; function getStudioUpdateInstructionLine(shell: UpdateShell): string { - return shell === "windows" ? "Open PowerShell and run:" : "Open Terminal and run:"; + return shell === "windows" + ? "Open PowerShell and run:" + : "Open Terminal and run:"; +} + +function isLocalInstallSource( + installSource?: UpdateInstallSourceState | null, +): boolean { + return Boolean( + installSource && + installSource !== "pypi" && + installSource !== "unknown" && + installSource !== "loading", + ); +} + +function isUnknownInstallSource( + installSource?: UpdateInstallSourceState | null, +): boolean { + return installSource === "unknown"; } function CopyableCommand({ @@ -54,7 +85,7 @@ function CopyableCommand({
{copied ? ( - + ) : ( )} @@ -77,24 +111,38 @@ function CopyableCommand({ ); } +// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: keep source-specific update guidance in one component so the command matrix stays visible. export function UpdateStudioInstructions({ className, defaultShell, + installSource, showTitle = true, }: { className?: string; defaultShell: UpdateShell; + installSource?: UpdateInstallSourceState | null; showTitle?: boolean; }): ReactElement { const [shell, setShell] = useState(defaultShell); const prefersReducedMotion = useReducedMotion(); const windows = shell === "windows"; + const localInstallSource = isLocalInstallSource(installSource); + const checkoutInstallSource = + installSource === "editable" || installSource === "local_repo"; + const packagedSourceInstall = + installSource === "vcs" || installSource === "local_path"; + const loadingInstallSource = installSource === "loading"; + const unknownInstallSource = isUnknownInstallSource(installSource); const fadeTransition = prefersReducedMotion ? { duration: 0 } : { duration: 0.16, ease: [0.165, 0.84, 0.44, 1] as const }; - const fadeInitial = prefersReducedMotion ? { opacity: 1 } : { opacity: 0, y: 2 }; + const fadeInitial = prefersReducedMotion + ? { opacity: 1 } + : { opacity: 0, y: 2 }; const fadeAnimate = { opacity: 1, y: 0 }; - const fadeExit = prefersReducedMotion ? { opacity: 1 } : { opacity: 0, y: -2 }; + const fadeExit = prefersReducedMotion + ? { opacity: 1 } + : { opacity: 0, y: -2 }; useEffect(() => { setShell(defaultShell); @@ -133,9 +181,9 @@ export function UpdateStudioInstructions({ onClick={() => setShell("unix")} className={cn( "px-0.5 py-0.5 font-medium transition-colors", - !windows - ? "text-foreground" - : "text-muted-foreground hover:text-emerald-600", + windows + ? "text-muted-foreground hover:text-emerald-600" + : "text-foreground", )} aria-pressed={!windows} > @@ -143,43 +191,157 @@ export function UpdateStudioInstructions({
- - - {getStudioUpdateInstructionLine(shell)} - - - -

- If that fails or unsloth studio update is unavailable, run: -

- - + {loadingInstallSource ? ( +

+ Checking how Studio was installed… +

+ ) : localInstallSource ? ( + <> +

+ Source or local install detected. To avoid replacing it with PyPI, + update from the checkout or source you originally installed from. +

+ {checkoutInstallSource ? ( + <> +

+ Pull latest changes from your Unsloth repo checkout, then update + Studio locally: +

+ + +

+ If the Studio update command is unavailable, run the local + installer from that checkout: +

+ + + + + + + ) : null} + {packagedSourceInstall ? ( + <> +

+ This looks like a source or VCS package install. Reinstall from + the original local path or Git URL you used. +

+

+ If you still have the Unsloth repo checkout, run the local + installer from that checkout: +

+ + + + + + + ) : null} +

+ Restart Studio after updating for changes to take effect. +

+ + ) : unknownInstallSource ? ( + <> +

+ Studio could not detect how it was installed. Check how you + installed Studio first, then choose the matching update path. +

+

+ For curl or PyPI installs, run: +

- - -

- Restart Studio after updating for changes to take effect. -

+

+ For local checkout installs, update from that checkout instead and + use the local update command: +

+ +

+ Restart Studio after updating for changes to take effect. +

+ + ) : ( + <> + + + {getStudioUpdateInstructionLine(shell)} + + + +

+ If that fails or unsloth studio update is unavailable, run: +

+ + + + + +

+ Restart Studio after updating for changes to take effect. +

+ + )} ); } diff --git a/studio/frontend/src/features/settings/tabs/about-tab.tsx b/studio/frontend/src/features/settings/tabs/about-tab.tsx index 9629900237..68d82a804a 100644 --- a/studio/frontend/src/features/settings/tabs/about-tab.tsx +++ b/studio/frontend/src/features/settings/tabs/about-tab.tsx @@ -1,12 +1,12 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -import { Button } from "@/components/ui/button"; import { ShutdownDialog } from "@/components/shutdown-dialog"; -import { UpdateStudioInstructions } from "../components/update-studio-instructions"; +import { Button } from "@/components/ui/button"; import { usePlatformStore } from "@/config/env"; -import { apiUrl } from "@/lib/api-base"; -import { removeTrainingUnloadGuard } from "@/features/training/hooks/use-training-unload-guard"; +import { getAuthToken } from "@/features/auth"; +import { removeTrainingUnloadGuard } from "@/features/training"; +import { apiUrl, isTauri } from "@/lib/api-base"; import { ArrowUpRight01Icon, Book03Icon, @@ -18,28 +18,108 @@ import { HugeiconsIcon } from "@hugeicons/react"; import { useEffect, useState } from "react"; import { SettingsRow } from "../components/settings-row"; import { SettingsSection } from "../components/settings-section"; +import { + type UpdateInstallSource, + UpdateStudioInstructions, +} from "../components/update-studio-instructions"; + +type ApiObject = Record; + +const INSTALL_SOURCE_KEY = "install_source"; + +const UPDATE_INSTALL_SOURCES = new Set([ + "pypi", + "editable", + "local_path", + "vcs", + "local_repo", + "unknown", +]); + +function isUpdateInstallSource(value: unknown): value is UpdateInstallSource { + return ( + typeof value === "string" && + UPDATE_INSTALL_SOURCES.has(value as UpdateInstallSource) + ); +} + +async function fetchStudioVersions(): Promise<{ + packageVersion: string | null; + studioVersion: string | null; +}> { + try { + const res = await fetch(apiUrl("/api/health")); + if (!res.ok) { + return { packageVersion: null, studioVersion: null }; + } + const data = (await res.json()) as ApiObject; + const packageVersion = data.version; + const studioVersion = data.studio_version; + return { + packageVersion: + typeof packageVersion === "string" ? packageVersion : null, + studioVersion: typeof studioVersion === "string" ? studioVersion : null, + }; + } catch { + return { packageVersion: null, studioVersion: null }; + } +} + +async function fetchInstallSource(): Promise { + if (isTauri) { + return "unknown"; + } + + const token = getAuthToken(); + if (!token) { + return "unknown"; + } + + try { + const headers = new Headers(); + headers.set("Authorization", `Bearer ${token}`); + const res = await fetch(apiUrl("/api/studio/install-source"), { headers }); + if (!res.ok) { + return "unknown"; + } + const data = (await res.json()) as ApiObject; + const installSource = data[INSTALL_SOURCE_KEY]; + return isUpdateInstallSource(installSource) ? installSource : "unknown"; + } catch { + return "unknown"; + } +} export function AboutTab() { const deviceType = usePlatformStore((s) => s.deviceType); const defaultShell = deviceType === "windows" ? "windows" : "unix"; const [shutdownOpen, setShutdownOpen] = useState(false); - const [version, setVersion] = useState("dev"); + const [packageVersion, setPackageVersion] = useState("dev"); + const [studioVersion, setStudioVersion] = useState("dev"); + const [installSource, setInstallSource] = useState< + UpdateInstallSource | "loading" + >("loading"); useEffect(() => { let canceled = false; - (async () => { - try { - const res = await fetch(apiUrl("/api/health")); - if (!res.ok) return; - const data = (await res.json()) as { version?: string }; - if (!canceled && data.version) { - setVersion(data.version); - } - } catch { - // fall back to dev label + fetchStudioVersions().then((nextVersions) => { + if (canceled) { + return; } - })(); + if (nextVersions.packageVersion) { + setPackageVersion(nextVersions.packageVersion); + } + if (nextVersions.studioVersion) { + setStudioVersion(nextVersions.studioVersion); + } + }); + + fetchInstallSource().then((nextInstallSource) => { + if (!canceled) { + setInstallSource(nextInstallSource); + } + }); return () => { canceled = true; @@ -56,14 +136,25 @@ export function AboutTab() { - - {version} + + + {studioVersion} + + + + + {packageVersion} +
- +
@@ -99,7 +190,10 @@ export function AboutTab() { rel="noopener noreferrer" className="inline-flex items-center gap-1 text-xs font-medium text-muted-foreground hover:text-foreground" > - + Report an issue @@ -108,7 +202,7 @@ export function AboutTab() { diff --git a/studio/frontend/src/features/training/hooks/use-training-unload-guard.ts b/studio/frontend/src/features/training/hooks/use-training-unload-guard.ts index 78c1e3d5c3..1808cf5888 100644 --- a/studio/frontend/src/features/training/hooks/use-training-unload-guard.ts +++ b/studio/frontend/src/features/training/hooks/use-training-unload-guard.ts @@ -2,7 +2,7 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import { useEffect } from "react"; -import { useTrainingRuntimeStore } from "@/features/training"; +import { useTrainingRuntimeStore } from "../stores/training-runtime-store"; let currentHandler: ((e: BeforeUnloadEvent) => void) | null = null; @@ -13,14 +13,18 @@ let currentHandler: ((e: BeforeUnloadEvent) => void) | null = null; export function useTrainingUnloadGuard() { useEffect(() => { const handler = (e: BeforeUnloadEvent) => { - if (!useTrainingRuntimeStore.getState().isTrainingRunning) return; + if (!useTrainingRuntimeStore.getState().isTrainingRunning) { + return; + } e.preventDefault(); e.returnValue = ""; }; currentHandler = handler; window.addEventListener("beforeunload", handler); return () => { - if (currentHandler === handler) currentHandler = null; + if (currentHandler === handler) { + currentHandler = null; + } window.removeEventListener("beforeunload", handler); }; }, []); diff --git a/studio/frontend/src/features/training/index.ts b/studio/frontend/src/features/training/index.ts index 571cfb6ff5..83d8edba75 100644 --- a/studio/frontend/src/features/training/index.ts +++ b/studio/frontend/src/features/training/index.ts @@ -16,7 +16,11 @@ export { useDatasetPreviewDialogStore } from "./stores/dataset-preview-dialog-st export { uploadTrainingDataset } from "./api/datasets-api"; export { listLocalModels } from "./api/models-api"; export type { LocalModelInfo } from "./api/models-api"; -export type { TrainingPhase, TrainingViewData, TrainingSeriesPoint } from "./types/runtime"; +export type { + TrainingPhase, + TrainingViewData, + TrainingSeriesPoint, +} from "./types/runtime"; export type { TrainingRunSummary, TrainingRunListResponse, diff --git a/studio/frontend/src/hooks/use-web-update-check.ts b/studio/frontend/src/hooks/use-web-update-check.ts new file mode 100644 index 0000000000..1ad0fe544c --- /dev/null +++ b/studio/frontend/src/hooks/use-web-update-check.ts @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { getAuthToken } from "@/features/auth"; +import { apiUrl, isTauri } from "@/lib/api-base"; +import { useCallback, useEffect, useState } from "react"; + +const WEB_UPDATE_CHECK_DELAY_MS = 5000; +const DISMISS_PREFIX = "unsloth_web_update_dismissed"; +const CAN_SHOW_KEY = "can_show_web_notification"; +const UPDATE_AVAILABLE_KEY = "update_available"; +const INSTALL_SOURCE_KEY = "install_source"; +const LATEST_VERSION_KEY = "latest_version"; +const CURRENT_VERSION_KEY = "current_version"; +const CHECKED_AT_KEY = "checked_at"; + +type ApiObject = Record; + +export type WebUpdateInstallSource = + | "pypi" + | "editable" + | "local_path" + | "vcs" + | "local_repo" + | "unknown"; + +export interface WebUpdateStatus { + currentVersion: string; + latestVersion: string; + installSource: "pypi"; + checkedAt: string; +} + +interface UseWebUpdateCheckOptions { + enabled?: boolean; + delayMs?: number; +} + +function stringField(value: ApiObject, key: string): string | null { + const field = value[key]; + return typeof field === "string" ? field : null; +} + +function toDisplayableUpdateStatus(value: unknown): WebUpdateStatus | null { + if (!value || typeof value !== "object") { + return null; + } + + const status = value as ApiObject; + const latestVersion = stringField(status, LATEST_VERSION_KEY); + const currentVersion = stringField(status, CURRENT_VERSION_KEY); + const checkedAt = stringField(status, CHECKED_AT_KEY); + if ( + status[CAN_SHOW_KEY] !== true || + status[UPDATE_AVAILABLE_KEY] !== true || + status[INSTALL_SOURCE_KEY] !== "pypi" || + !latestVersion || + !currentVersion || + !checkedAt + ) { + return null; + } + + return { + currentVersion, + latestVersion, + installSource: "pypi", + checkedAt, + }; +} + +function dismissalKey(status: WebUpdateStatus): string { + return `${DISMISS_PREFIX}:${status.installSource}:${status.latestVersion}`; +} + +function isDismissed(status: WebUpdateStatus): boolean { + if (typeof window === "undefined") { + return true; + } + try { + return window.localStorage.getItem(dismissalKey(status)) !== null; + } catch { + return false; + } +} + +function markDismissed(status: WebUpdateStatus): void { + if (typeof window === "undefined") { + return; + } + try { + window.localStorage.setItem(dismissalKey(status), String(Date.now())); + } catch { + // Ignore storage failures; the banner can still be dismissed in-memory. + } +} + +async function fetchDisplayableUpdateStatus(): Promise { + const token = getAuthToken(); + if (!token) { + return null; + } + + const headers = new Headers(); + headers.set("Authorization", `Bearer ${token}`); + const res = await fetch(apiUrl("/api/studio/update-status"), { headers }); + if (!res.ok) { + return null; + } + + return toDisplayableUpdateStatus(await res.json()); +} + +export function useWebUpdateCheck({ + enabled = true, + delayMs = WEB_UPDATE_CHECK_DELAY_MS, +}: UseWebUpdateCheckOptions = {}) { + const [status, setStatus] = useState(null); + + useEffect(() => { + if (isTauri || !enabled || !getAuthToken()) { + const clearTimer = window.setTimeout(() => setStatus(null), 0); + return () => window.clearTimeout(clearTimer); + } + + let canceled = false; + const timer = window.setTimeout(() => { + fetchDisplayableUpdateStatus() + .then((nextStatus) => { + if (canceled) { + return; + } + setStatus(nextStatus && !isDismissed(nextStatus) ? nextStatus : null); + }) + .catch(() => { + if (!canceled) { + setStatus(null); + } + }); + }, delayMs); + + return () => { + canceled = true; + window.clearTimeout(timer); + }; + }, [delayMs, enabled]); + + const dismiss = useCallback(() => { + setStatus((current) => { + if (current) { + markDismissed(current); + } + return null; + }); + }, []); + + return { status: enabled && !isTauri ? status : null, dismiss }; +} From 1794a544b543d9d3d827b124044cc49f4bf9f9e7 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 11 May 2026 18:57:20 -0700 Subject: [PATCH 02/25] ci: retry transient github.com 5xx on unsloth-zoo git fetches in CI (#5389) Windows Studio API CI run 25676130116 / job 75374388468 failed at "Install Studio (--local, --no-torch)" because github.com itself returned HTTP 500 mid-clone: remote: Internal Server Error fatal: unable to access 'https://github.com/unslothai/unsloth-zoo/': The requested URL returned error: 500 exit code: 128 The runner did nothing wrong. github.com served the same repo fine seconds before and after, and adjacent commits on main were green. Without a retry, every transient upstream blip turns one job red. Scope the retry layer to CI workflows only, leaving install.sh and install.ps1 unchanged so end-user installs keep their existing behavior (a transient github.com hiccup will still surface verbatim on a user's machine, where they can re-run interactively). .github/workflows/{mlx-ci,version-compat-ci,consolidated-tests-ci}.yml - inline 3-attempt retry loop around the four direct `git clone` / `pip install git+...unsloth-zoo` invocations, emitting GitHub Actions ::warning::/::error:: annotations so transient hits surface in the job summary Only kicks in for upstream failures (5xx, exit 128, network errors) and so does not mask genuine install errors -- a malformed pip spec, a missing dependency, a real type error in the zoo's setup.py all still fail on the first attempt. --- .github/workflows/consolidated-tests-ci.yml | 40 +++++++++++++++++---- .github/workflows/mlx-ci.yml | 15 +++++++- .github/workflows/version-compat-ci.yml | 18 ++++++++-- 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/.github/workflows/consolidated-tests-ci.yml b/.github/workflows/consolidated-tests-ci.yml index 4ad3d9f16a..0f6f89d354 100644 --- a/.github/workflows/consolidated-tests-ci.yml +++ b/.github/workflows/consolidated-tests-ci.yml @@ -234,9 +234,23 @@ jobs: # tests/conftest.py spoof which handles that. run: | set -euxo pipefail - git clone --depth=1 --branch="$UNSLOTH_ZOO_REF" \ - https://github.com/unslothai/unsloth-zoo \ - "$RUNNER_TEMP/unsloth-zoo" + # github.com occasionally 500s on the git fetch; retry so a + # single upstream blip does not fail CI. + for attempt in 1 2 3; do + rm -rf "$RUNNER_TEMP/unsloth-zoo" + if git clone --depth=1 --branch="$UNSLOTH_ZOO_REF" \ + https://github.com/unslothai/unsloth-zoo \ + "$RUNNER_TEMP/unsloth-zoo"; then + break + fi + if [ "$attempt" -eq 3 ]; then + echo "::error::git clone unsloth-zoo failed after 3 attempts" + exit 1 + fi + delay=$((5 * attempt)) + echo "::warning::clone failed (attempt $attempt/3), retrying in ${delay}s..." + sleep "$delay" + done pip install -e "$RUNNER_TEMP/unsloth-zoo" --no-deps pip show unsloth_zoo @@ -2040,9 +2054,23 @@ jobs: # main-branch fixes flow into the smoke without a release). run: | set -euxo pipefail - git clone --depth=1 --branch="$UNSLOTH_ZOO_REF" \ - https://github.com/unslothai/unsloth-zoo \ - "$RUNNER_TEMP/unsloth-zoo" + # github.com occasionally 500s on the git fetch; retry so a + # single upstream blip does not fail CI. + for attempt in 1 2 3; do + rm -rf "$RUNNER_TEMP/unsloth-zoo" + if git clone --depth=1 --branch="$UNSLOTH_ZOO_REF" \ + https://github.com/unslothai/unsloth-zoo \ + "$RUNNER_TEMP/unsloth-zoo"; then + break + fi + if [ "$attempt" -eq 3 ]; then + echo "::error::git clone unsloth-zoo failed after 3 attempts" + exit 1 + fi + delay=$((5 * attempt)) + echo "::warning::clone failed (attempt $attempt/3), retrying in ${delay}s..." + sleep "$delay" + done pip install -e "$RUNNER_TEMP/unsloth-zoo" --no-deps pip show unsloth_zoo diff --git a/.github/workflows/mlx-ci.yml b/.github/workflows/mlx-ci.yml index 61e0566903..52038dd332 100644 --- a/.github/workflows/mlx-ci.yml +++ b/.github/workflows/mlx-ci.yml @@ -153,7 +153,20 @@ jobs: 'httpx==0.28.1' pip install --index-url https://download.pytorch.org/whl/cpu \ 'torch==2.10.0' - pip install "unsloth_zoo @ git+https://github.com/unslothai/unsloth-zoo" + # github.com occasionally 500s on the git fetch; retry the + # zoo install so a single upstream blip does not fail CI. + for attempt in 1 2 3; do + if pip install "unsloth_zoo @ git+https://github.com/unslothai/unsloth-zoo"; then + break + fi + if [ "$attempt" -eq 3 ]; then + echo "::error::pip install unsloth_zoo failed after 3 attempts" + exit 1 + fi + delay=$((5 * attempt)) + echo "::warning::unsloth_zoo install failed (attempt $attempt/3), retrying in ${delay}s..." + sleep "$delay" + done pip install -e . --no-deps # Real Apple Silicon sanity: confirm _IS_MLX activates on real diff --git a/.github/workflows/version-compat-ci.yml b/.github/workflows/version-compat-ci.yml index ff3218bba0..b14a759916 100644 --- a/.github/workflows/version-compat-ci.yml +++ b/.github/workflows/version-compat-ci.yml @@ -203,8 +203,22 @@ jobs: with: { path: unsloth } - name: Clone unsloth-zoo @ main run: | - git clone --depth=1 https://github.com/unslothai/unsloth-zoo \ - "$RUNNER_TEMP/unsloth-zoo" + # github.com occasionally 500s on the git fetch; retry so a + # single upstream blip does not fail CI. + for attempt in 1 2 3; do + rm -rf "$RUNNER_TEMP/unsloth-zoo" + if git clone --depth=1 https://github.com/unslothai/unsloth-zoo \ + "$RUNNER_TEMP/unsloth-zoo"; then + break + fi + if [ "$attempt" -eq 3 ]; then + echo "::error::git clone unsloth-zoo failed after 3 attempts" + exit 1 + fi + delay=$((5 * attempt)) + echo "::warning::clone failed (attempt $attempt/3), retrying in ${delay}s..." + sleep "$delay" + done - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.12' From ac765d2efbcd7410e95ab8b2feade2aaae95b806 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 11 May 2026 20:36:52 -0700 Subject: [PATCH 03/25] studio/ci: pre-install lockfile supply-chain audit (npm + cargo) (#5392) * studio/ci: pre-install lockfile supply-chain audit (npm + cargo) The Mini Shai-Hulud wave that hit @tanstack/* on 2026-05-11 19:20-19:26 UTC (GHSA-g7cv-rxg3-hmpx) pushed 84 malicious versions across 42 packages. Each compromised tarball carried an `optionalDependencies` entry pointing at a GitHub-hosted prepare script that exfiltrated GitHub / npm / AWS / Vault / SSH credentials on `npm install` / `npm ci`. Our current lockfile pins ALL @tanstack/* at pre-malicious versions so we were not exposed, but the only defense layer between "dependabot opens a security-update PR during a malicious window" and "a compromised package's postinstall runs on the CI runner" is the advisory-DB latency. `npm audit` and OSV-Scanner are reactive: there is a window between malicious publication and GHSA landing. Add a pre-install lockfile audit that fires on the injection pattern itself, BEFORE `npm ci` gets a chance to execute lifecycle scripts: scripts/lockfile_supply_chain_audit.py npm side (studio/frontend/package-lock.json, lockfileVersion 2/3): 1. every `resolved` URL must point to registry.npmjs.org; direct GitHub / git+ / file: refs are the Shai-Hulud vector 2. every non-bundled entry must carry an `integrity` SHA 3. raw-text scan for known IOC strings (router_init.js, tanstack_runner.js, router_runtime.js, @tanstack/setup, the specific TanStack worm commit hash, getsession.org exfiltration host, "A Mini Shai-Hulud has Appeared" marker) 4. nested `node_modules/.../node_modules/` fold-ins are transparent -- they ride on the parent tarball's integrity cargo side (studio/src-tauri/Cargo.lock): 5. every `source` must be the crates.io registry 6. registry crates must have a `checksum` 7. one allowlist entry: fix-path-env from tauri-apps/fix-path-env-rs at pinned SHA c4c45d5. Any other non-registry source -- or a bump of that pinned SHA -- re-fires the audit until reviewed + appended Wire into four workflows: .github/workflows/security-audit.yml -- new step inside the advisory-audit job, immediately before `npm audit` so the structural pass and the advisory-DB pass appear together in the GitHub step summary. .github/workflows/studio-frontend-ci.yml, .github/workflows/wheel-smoke.yml, .github/workflows/studio-tauri-smoke.yml -- new step immediately BEFORE `npm ci`. If a future malicious bump lands in our lockfile, the audit refuses and `npm ci` never runs, so no `prepare` / `postinstall` from a compromised tarball can execute on the runner. Note on --ignore-scripts: every npm ci in our CI is followed directly by `npm run build` or `tauri build`, both of which depend on package install scripts (esbuild's native-binary postinstall, etc.). Blanket --ignore-scripts breaks the build, so the pre-install structural audit is the practical mitigation. The audit reads lockfiles only; it never executes anything from them. Verified: - Clean state: 0 findings on the current tree (npm + cargo). - Fault injection: synthetic `@tanstack/setup` IOC + non-registry `resolved` URL both fire with exit code 1. - YAML parses cleanly for all four modified workflows. Refs: - https://tanstack.com/blog/npm-supply-chain-compromise-postmortem - https://github.com/TanStack/router/issues/7383 - https://github.com/TanStack/router/security/advisories/GHSA-g7cv-rxg3-hmpx - https://www.aikido.dev/blog/mini-shai-hulud-is-back-tanstack-compromised - https://www.stepsecurity.io/blog/mini-shai-hulud-is-back-a-self-spreading-supply-chain-attack-hits-the-npm-ecosystem * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/security-audit.yml | 21 + .github/workflows/studio-frontend-ci.yml | 8 + .github/workflows/studio-tauri-smoke.yml | 3 + .github/workflows/wheel-smoke.yml | 3 + scripts/lockfile_supply_chain_audit.py | 486 +++++++++++++++++++++++ 5 files changed, 521 insertions(+) create mode 100755 scripts/lockfile_supply_chain_audit.py diff --git a/.github/workflows/security-audit.yml b/.github/workflows/security-audit.yml index 0fc8073e75..df0af95dfa 100644 --- a/.github/workflows/security-audit.yml +++ b/.github/workflows/security-audit.yml @@ -244,6 +244,27 @@ jobs: echo '```' } >> "$GITHUB_STEP_SUMMARY" + # ───────────────────────────────────────────────────────────── + # Pre-install lockfile supply-chain audit (npm + cargo). + # Catches structural anomalies (non-registry resolved URLs, + # missing integrity hashes, known IOC strings) BEFORE `npm + # audit` or OSV-Scanner consult the advisory DB. The advisory + # path is reactive -- there is a window between a malicious + # publication and the GHSA landing. This step fires on the + # injection pattern itself so it catches the same class of + # attack the moment the lockfile shape becomes wrong. + # ───────────────────────────────────────────────────────────── + - name: Lockfile supply-chain audit (pre-install scan) + run: | + python3 scripts/lockfile_supply_chain_audit.py + { + echo "## Lockfile supply-chain audit" + echo + echo "Scanned: studio/frontend/package-lock.json + studio/src-tauri/Cargo.lock" + echo + echo "No structural anomalies or known IOC strings." + } >> "$GITHUB_STEP_SUMMARY" + # ───────────────────────────────────────────────────────────── # npm: Studio frontend # ───────────────────────────────────────────────────────────── diff --git a/.github/workflows/studio-frontend-ci.yml b/.github/workflows/studio-frontend-ci.yml index eb00e297a7..bde62c87f6 100644 --- a/.github/workflows/studio-frontend-ci.yml +++ b/.github/workflows/studio-frontend-ci.yml @@ -58,6 +58,14 @@ jobs: cache: 'npm' cache-dependency-path: studio/frontend/package-lock.json + # Run the structural lockfile scan BEFORE npm ci. A compromised + # tarball runs its `prepare` / `postinstall` during `npm ci`, + # so any catch has to fire upstream of that. The scanner is + # pure-Python read-only; safe to call ahead of every install. + - name: Lockfile supply-chain audit (pre-install scan) + working-directory: ${{ github.workspace }} + run: python3 scripts/lockfile_supply_chain_audit.py + - name: Lockfile must agree with package.json (npm ci is strict) run: npm ci --no-fund --no-audit diff --git a/.github/workflows/studio-tauri-smoke.yml b/.github/workflows/studio-tauri-smoke.yml index d517a5f454..23b57d7e09 100644 --- a/.github/workflows/studio-tauri-smoke.yml +++ b/.github/workflows/studio-tauri-smoke.yml @@ -69,6 +69,9 @@ jobs: echo "$out" [ "$out" = "tauri-cli 2.10.1" ] || { echo "::error::expected tauri-cli 2.10.1, got $out"; exit 1; } + - name: Lockfile supply-chain audit (pre-install scan) + run: python3 scripts/lockfile_supply_chain_audit.py + - name: Frontend build (npm ci, vite) working-directory: studio/frontend run: | diff --git a/.github/workflows/wheel-smoke.yml b/.github/workflows/wheel-smoke.yml index 983070ae13..dad8670393 100644 --- a/.github/workflows/wheel-smoke.yml +++ b/.github/workflows/wheel-smoke.yml @@ -53,6 +53,9 @@ jobs: with: python-version: '3.12' + - name: Lockfile supply-chain audit (pre-install scan) + run: python3 scripts/lockfile_supply_chain_audit.py + - name: Build frontend run: | cd studio/frontend diff --git a/scripts/lockfile_supply_chain_audit.py b/scripts/lockfile_supply_chain_audit.py new file mode 100755 index 0000000000..e52183214e --- /dev/null +++ b/scripts/lockfile_supply_chain_audit.py @@ -0,0 +1,486 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Lockfile supply-chain audit for the Studio frontend and Tauri shell. + +Runs BEFORE `npm ci` / `cargo fetch` in CI. Refuses to proceed when a +lockfile contains patterns that indicate the kind of supply-chain +injection seen in the npm Shai-Hulud waves and the cargo +crates.io brand-squat attempts. + +What it checks +============== + +studio/frontend/package-lock.json (lockfileVersion 2 or 3): + + 1. `resolved` URL origin. Every entry must resolve through + `https://registry.npmjs.org/`. Direct GitHub-hosted dependencies + (`git+ssh://`, `git+https://`, `github:owner/repo#sha`, + `file:`, `http://`) are refused -- npm's TanStack incident used + exactly this vector to land an unaudited GitHub commit hash as + an optional dependency. + + 2. `integrity` field presence. Every non-workspace entry must carry + an `integrity` SHA. A missing integrity means the registry can + swap the tarball after lockfile generation and CI will not + notice. + + 3. Known IOC strings. A hardcoded set of indicator-of-compromise + substrings is grepped across the entire lockfile body (file + names, dependency keys, URLs). The list is updated as new + campaigns surface. Catching one means the local install was + about to pull a publicly-known malicious release. + +studio/src-tauri/Cargo.lock: + + 4. `source` field origin. Every entry with a `source` must point at + `registry+https://github.com/rust-lang/crates.io-index`. Direct + git sources (`git+https://...`) and `path+...` for cross-crate + paths warrant manual review and are flagged. + + 5. Known cargo IOC strings. Same idea as (3), separate list. + +Exit codes +========== + + 0 no findings, or an opt-out env var (UNSLOTH_LOCKFILE_AUDIT_SKIP=1) + is set + 1 one or more findings; stderr lists them with file path and line + number where derivable + 2 internal error (missing dependency, malformed JSON, etc.) + +Operational stance +================== + +This scanner only PARSES the lockfiles -- it never executes anything +in them, never resolves anything against the network. Safe to run +ahead of every `npm ci`. The IOC list is short by design; this +complements (not replaces) `npm audit`, OSV-Scanner, and the +advisory-DB pipeline in `.github/workflows/security-audit.yml`. The +shape of the catch is "we refuse to proceed because the lockfile +itself is shaped wrong", which fires before any third-party install +script gets a chance to run on the runner. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +# ───────────────────────────────────────────────────────────────────── +# Known IOC strings (case-sensitive substring match). +# ───────────────────────────────────────────────────────────────────── +# +# Keep these short and FACTUAL. Each entry is tied to a public advisory +# and is the literal string an attacker would have to embed for the +# attack to work. Adding speculative or generic patterns here would +# generate false positives on dependency upgrades. +NPM_IOC_STRINGS: tuple[str, ...] = ( + # Shai-Hulud TanStack wave -- May 11, 2026 (GHSA-g7cv-rxg3-hmpx). + "router_init.js", + "tanstack_runner.js", + "router_runtime.js", + "@tanstack/setup", + "github:tanstack/router#79ac49eedf774dd4b0cfa308722bc463cfe5885c", + # Exfiltration endpoints observed across both Shai-Hulud waves. + "filev2.getsession.org", + "getsession.org/file/", + # Campaign markers; the worm tarballs print this to stdout on run. + "A Mini Shai-Hulud has Appeared", +) + +CARGO_IOC_STRINGS: tuple[str, ...] = ( + # Reserved for future cargo-side incidents. Empty by default -- + # `source` origin check below catches the structural pattern. +) + + +# ───────────────────────────────────────────────────────────────────── +# Allowed lockfile origins. +# ───────────────────────────────────────────────────────────────────── +NPM_REGISTRY_PREFIX = "https://registry.npmjs.org/" + +# Tarballs are also fetched from this mirror on some GH Actions cached +# runs (npm rewrites the resolved URL on cache hit). Allow either. +NPM_REGISTRY_PREFIXES_ALLOWED: tuple[str, ...] = (NPM_REGISTRY_PREFIX,) + +CARGO_REGISTRY_SOURCE = "registry+https://github.com/rust-lang/crates.io-index" + + +# ───────────────────────────────────────────────────────────────────── +# Cargo non-registry source allowlist. +# ───────────────────────────────────────────────────────────────────── +# +# Each entry is `(crate_name, exact_source_string)`. The crate must +# match by name AND the source must match the full pinned-SHA string +# verbatim. Bumping the commit SHA forces a re-review here: the +# scanner fires until the new SHA is appended. +# +# Studio's Tauri shell pulls `fix-path-env` directly from +# tauri-apps/fix-path-env-rs because the crate is not published to +# crates.io. The pinned commit (c4c45d5) was reviewed at the time it +# landed; future bumps need explicit approval. +CARGO_SOURCE_ALLOWLIST: tuple[tuple[str, str], ...] = ( + ( + "fix-path-env", + "git+https://github.com/tauri-apps/fix-path-env-rs#" + "c4c45d503ea115a839aae718d02f79e7c7f0f673", + ), +) + + +# ───────────────────────────────────────────────────────────────────── +# Finding container. +# ───────────────────────────────────────────────────────────────────── + + +class Finding: + __slots__ = ("path", "package", "kind", "detail") + + def __init__(self, path: str, package: str, kind: str, detail: str) -> None: + self.path = path + self.package = package + self.kind = kind + self.detail = detail + + def __str__(self) -> str: + return ( + f" [{self.kind}] {self.path}\n" + f" package: {self.package}\n" + f" detail: {self.detail}" + ) + + +# ───────────────────────────────────────────────────────────────────── +# package-lock.json audit. +# ───────────────────────────────────────────────────────────────────── + + +def audit_npm_lockfile(path: Path) -> list[Finding]: + findings: list[Finding] = [] + if not path.exists(): + return findings + + raw = path.read_text(encoding = "utf-8") + try: + lock = json.loads(raw) + except json.JSONDecodeError as exc: + findings.append( + Finding( + path = str(path), + package = "", + kind = "malformed-lockfile", + detail = f"could not parse as JSON: {exc}", + ) + ) + return findings + + lockfile_version = lock.get("lockfileVersion") + if lockfile_version not in (2, 3): + findings.append( + Finding( + path = str(path), + package = "", + kind = "unsupported-lockfile-version", + detail = (f"only lockfileVersion 2 or 3 audited; got {lockfile_version}"), + ) + ) + + packages = lock.get("packages") or {} + for key, entry in packages.items(): + # The empty key "" is the project root; workspace entries use + # keys like "node_modules/foo" or "studio/frontend/sub-pkg". + # Skip the project root (it has no `resolved`). + if key == "": + continue + if entry.get("link"): + # Workspace symlink; no tarball to resolve. + continue + + resolved = entry.get("resolved") + # Entries living inside another package's `node_modules/` + # tree are bundled fold-ins -- the parent's tarball ships + # their source verbatim and the parent's `integrity` covers + # the whole subtree. npm represents them in lockfileVersion 3 + # as nested entries with no `resolved` and no `integrity` of + # their own. Treat them as transparent to this audit. + nested = key.count("/node_modules/") >= 1 + + # 1. resolved-URL origin. + if resolved is None: + if nested or entry.get("bundled"): + # Bundled / fold-in entry; covered by parent integrity. + pass + elif entry.get("version"): + # Top-level entry without a resolved URL is suspicious. + findings.append( + Finding( + path = str(path), + package = key, + kind = "missing-resolved-url", + detail = ( + f"version={entry['version']!r} but no `resolved` " + "field; lockfile is incomplete" + ), + ) + ) + else: + if not any(resolved.startswith(p) for p in NPM_REGISTRY_PREFIXES_ALLOWED): + findings.append( + Finding( + path = str(path), + package = key, + kind = "non-registry-resolved-url", + detail = ( + f"resolved={resolved!r}; only " + f"{NPM_REGISTRY_PREFIX} is permitted. Direct " + "GitHub / git / file references are the " + "Shai-Hulud injection vector." + ), + ) + ) + + # 2. integrity-hash presence. + if resolved is not None and not entry.get("integrity"): + findings.append( + Finding( + path = str(path), + package = key, + kind = "missing-integrity-hash", + detail = ( + "no `integrity` field; npm cannot verify the " + "tarball SHA against the registry-published hash" + ), + ) + ) + + # 3. Known IOC strings: scan the raw file body so we hit fields the + # structural pass above doesn't enumerate (scripts, optional + # dependencies, etc.). Cheap and complete. + for ioc in NPM_IOC_STRINGS: + if ioc in raw: + # Best-effort line number lookup. + line_no = _first_line_containing(raw, ioc) + findings.append( + Finding( + path = f"{path}:{line_no}" if line_no else str(path), + package = "", + kind = "known-ioc-string", + detail = ( + f"matched known IOC substring {ioc!r}; this is " + "a public indicator of a recent supply-chain " + "compromise. Refuse to install." + ), + ) + ) + + return findings + + +def _first_line_containing(text: str, needle: str) -> int | None: + for i, line in enumerate(text.splitlines(), start = 1): + if needle in line: + return i + return None + + +# ───────────────────────────────────────────────────────────────────── +# Cargo.lock audit. +# ───────────────────────────────────────────────────────────────────── + + +# Cargo.lock is TOML; parse with stdlib tomllib (Python 3.11+). The +# studio's Tauri shell already requires a modern toolchain so this is +# always available where CI runs. +_PACKAGE_HEADER = re.compile(r"^\[\[package\]\]\s*$") + + +def audit_cargo_lockfile(path: Path) -> list[Finding]: + findings: list[Finding] = [] + if not path.exists(): + return findings + + raw = path.read_text(encoding = "utf-8") + try: + import tomllib # type: ignore[import-not-found] + except ImportError: + # Python <3.11; fall back to a tomli shim if importable. + try: + import tomli as tomllib # type: ignore[no-redef] + except ImportError: + findings.append( + Finding( + path = str(path), + package = "", + kind = "missing-toml-parser", + detail = ( + "Python 3.11+ tomllib or tomli is required to " + "parse Cargo.lock; install tomli or upgrade " + "Python before re-running this audit" + ), + ) + ) + return findings + + try: + lock = tomllib.loads(raw) + except Exception as exc: + findings.append( + Finding( + path = str(path), + package = "", + kind = "malformed-lockfile", + detail = f"could not parse as TOML: {exc}", + ) + ) + return findings + + for entry in lock.get("package", []): + name = entry.get("name") or "" + version = entry.get("version") or "" + source = entry.get("source") + # Workspace-local crates have no `source` field; skip them. + if source is None: + continue + if source != CARGO_REGISTRY_SOURCE: + if (name, source) in CARGO_SOURCE_ALLOWLIST: + # Pre-approved non-registry source pinned by SHA. + pass + else: + findings.append( + Finding( + path = str(path), + package = f"{name}@{version}", + kind = "non-registry-cargo-source", + detail = ( + f"source={source!r}; only " + f"{CARGO_REGISTRY_SOURCE!r} is permitted " + "by default, and no allowlist entry covers " + "this crate. If the source is legitimate, " + "add `(name, source)` to " + "CARGO_SOURCE_ALLOWLIST after reviewing the " + "pinned commit." + ), + ) + ) + if not entry.get("checksum") and source == CARGO_REGISTRY_SOURCE: + findings.append( + Finding( + path = str(path), + package = f"{name}@{version}", + kind = "missing-cargo-checksum", + detail = ( + "registry crate without checksum; cargo cannot " + "verify the downloaded source against the " + "registry-published SHA" + ), + ) + ) + + for ioc in CARGO_IOC_STRINGS: + if ioc in raw: + line_no = _first_line_containing(raw, ioc) + findings.append( + Finding( + path = f"{path}:{line_no}" if line_no else str(path), + package = "", + kind = "known-ioc-string", + detail = f"matched known IOC substring {ioc!r}", + ) + ) + + return findings + + +# ───────────────────────────────────────────────────────────────────── +# CLI. +# ───────────────────────────────────────────────────────────────────── + + +DEFAULT_NPM_LOCKFILES = ("studio/frontend/package-lock.json",) +DEFAULT_CARGO_LOCKFILES = ("studio/src-tauri/Cargo.lock",) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description = "Pre-install lockfile supply-chain audit.", + ) + parser.add_argument( + "--root", + default = str(REPO_ROOT), + help = "Repo root (default: parent of this script).", + ) + parser.add_argument( + "--npm-lockfile", + action = "append", + default = None, + help = ( + "Path to a package-lock.json (repeatable). " + "Default: studio/frontend/package-lock.json." + ), + ) + parser.add_argument( + "--cargo-lockfile", + action = "append", + default = None, + help = ( + "Path to a Cargo.lock (repeatable). " + "Default: studio/src-tauri/Cargo.lock." + ), + ) + args = parser.parse_args(argv) + + if os.environ.get("UNSLOTH_LOCKFILE_AUDIT_SKIP") == "1": + print( + "[lockfile-audit] UNSLOTH_LOCKFILE_AUDIT_SKIP=1; " + "audit skipped (expected only for local triage)", + flush = True, + ) + return 0 + + root = Path(args.root).resolve() + npm_paths = [root / p for p in (args.npm_lockfile or DEFAULT_NPM_LOCKFILES)] + cargo_paths = [root / p for p in (args.cargo_lockfile or DEFAULT_CARGO_LOCKFILES)] + + all_findings: list[Finding] = [] + for p in npm_paths: + print(f"[lockfile-audit] npm: {p}", flush = True) + all_findings.extend(audit_npm_lockfile(p)) + for p in cargo_paths: + print(f"[lockfile-audit] cargo: {p}", flush = True) + all_findings.extend(audit_cargo_lockfile(p)) + + if not all_findings: + print( + f"[lockfile-audit] OK: 0 findings across " + f"{len(npm_paths)} npm + {len(cargo_paths)} cargo lockfile(s)", + flush = True, + ) + return 0 + + print( + f"\n[lockfile-audit] FAIL: {len(all_findings)} finding(s):\n", + file = sys.stderr, + ) + for f in all_findings: + print(str(f), file = sys.stderr) + print(file = sys.stderr) + print( + "[lockfile-audit] Refusing to proceed. Each finding above is " + "either a structural lockfile anomaly or a public indicator-of-" + "compromise. Investigate before running `npm ci` or `cargo fetch`.", + file = sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) From e27cc0ab08001c311ffecb7fb02b04a0e5963d11 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 11 May 2026 20:37:05 -0700 Subject: [PATCH 04/25] studio/ci: npm tarball content scanner (no-install, hostile-input safe) (#5393) * studio/ci: npm tarball content scanner (no-install, hostile-input safe) Counterpart to scripts/scan_packages.py for the npm side. Pip-side scanner reads requirements files, downloads PyPI archives via `pip download --no-deps`, and pattern-scans them for malicious shapes. This change adds the equivalent for npm tarballs. Why === PR #5392 (lockfile_supply_chain_audit.py) catches injection-pattern attacks where the malicious metadata lives IN the lockfile -- e.g. the TanStack Shai-Hulud worm that injected an `optionalDependencies` entry pointing at a GitHub commit. It does not catch the broader class of "legit-registry tarball with malicious content but normal lockfile metadata": attacker steals a maintainer's npm publish token, publishes a malicious version to registry.npmjs.org with a valid integrity hash, and the lockfile entry looks normal -- the malicious code lives inside the tarball's dist/index.js or its own postinstall script. Today that gap is covered reactively by `npm audit` + OSV-Scanner once the GHSA lands; there is a real window before that. This scanner closes the window by inspecting tarball CONTENT. What it checks ============== For each entry in studio/frontend/package-lock.json: 1. Download the tarball directly from registry.npmjs.org. Refuse any non-allowlisted URL. Stream-bounded at 64 MiB. 2. Verify SHA-512 integrity against the lockfile entry BEFORE opening the tarball. 3. Safely extract into a sandboxed temp dir behind guards: - reject symlinks / hardlinks (LNKTYPE, SYMTYPE) - reject absolute paths and `..` traversal - reject character / block / FIFO devices - per-file size cap 8 MiB, cumulative cap 128 MiB, member count cap 50000 - stream open (mode='r|gz') so we abort mid-extract - extracted files set to non-executable mode (0o644) 4. Pattern-scan the extracted text content for: - lifecycle (preinstall/install/postinstall/prepare) scripts in any package.json that fetch + pipe-to-shell external content -- the install-time RCE vector - optionalDependencies pointing at github: / git+ / git: (TanStack worm injection shape) - C2 / exfiltration hosts: getsession.org, 169.254.169.254 (IMDS), 169.254.170.2 (ECS), metadata.google.internal, vault.svc.cluster.local, k8s ServiceAccount token paths, ACTIONS_ID_TOKEN_REQUEST_URL/TOKEN, npm publish-token enumeration endpoint - credential paths a frontend lib should never read: ~/.npmrc, ~/.aws/credentials, ~/.ssh/id_*, /.kube/config, /.docker/config.json - JS regex: Function/eval against base64-decoded payload, process.env.GITHUB_TOKEN / NPM_TOKEN / AWS_* access in package source - obfuscation: large base64-ish blob (>=2 KiB) fed into Function or eval (router_init.js dropper shape) - literal IOC substrings from public advisories Safety ====== Threat model: every tarball is hostile. The scanner: - never runs `npm install`, never executes anything from a downloaded tarball, never calls subprocess on extracted content - downloads only from registry.npmjs.org (defence-in-depth check at parse time AND inside download_tarball) - stdlib-only (no third-party deps -- adding one would itself be a supply-chain liability) - tempdir wiped via atexit on every termination path - exit codes: 0 clean, 1 HIGH/CRITICAL finding, 2 internal error Wiring ====== New job `npm-scan-packages` in security-audit.yml, parallel to `pip-scan-packages`. Triggers same as the existing audits (PR on manifest changes, push to main/pip, daily 04:13 UTC, dispatch). Initially `continue-on-error: true` so the baseline can settle -- matches the existing convention for the other audit steps. Drop that flag once the baseline is clean for a week. Verified locally ================ - AST parse OK. - Real-network 3-package smoke: 0 findings. - Real-network 25-package smoke (Babel + assistant-ui surface): 0 findings, no hard errors. - 9 fault-injection scenarios all pass: 1. zip-slip path traversal refused 2. symlink member refused 3. oversized member refused (size cap) 4. too-many-members refused (count cap) 5. router_init.js IOC + obfuscated-blob shape both detected in synthetic malicious tarball 6. lifecycle fetch-exec in scripts.preinstall detected as CRITICAL 7. AWS IMDS reference (169.254.169.254) detected 8. SRI integrity-parser accepts syntactically-valid SRI 9. download_tarball refuses non-allowlisted hostname Refs ==== - https://tanstack.com/blog/npm-supply-chain-compromise-postmortem - https://github.com/TanStack/router/issues/7383 - https://github.com/TanStack/router/security/advisories/GHSA-g7cv-rxg3-hmpx - https://www.aikido.dev/blog/mini-shai-hulud-is-back-tanstack-compromised - https://www.stepsecurity.io/blog/mini-shai-hulud-is-back-a-self-spreading-supply-chain-attack-hits-the-npm-ecosystem * scan_npm_packages: kill false positives + handle real native binaries First CI run on PR #5393 (run 25710423126 / job 75489317395) hit two false-positive classes plus one cap-too-tight class: False positives (7 findings): @langchain/core 1.1.44 ssrf.{cjs,js}: a SSRF *protection* module that ships a literal blocklist `const CLOUD_METADATA_IPS = [...]` of IMDS hosts as data the library REFUSES to dial. Our scanner saw the IPs as substrings and flagged 6 of them. object-treeify 1.1.33 package.json: a manual `docker` dev script that mounts `~/.npmrc` and `~/.aws` for local containerised builds. npm never runs `scripts.docker` automatically; it is only invoked when a developer runs `npm run docker`. Our bare substring scan flagged the `/.npmrc` reference anyway. Cap-too-tight class (10+ findings): next/swc, rolldown bindings, biome CLI, lightningcss, mermaid sourcemap, typescript.js. The 8 MiB per-file cap was calibrated for JS source and rejected legitimate precompiled native binaries (next-swc .node is 137 MB) and CLI executables (biome is 25-33 MB). Fixes ===== cred-surface-host detection split into two tiers: ALWAYS_BAD substrings have no legitimate use anywhere and still bare-match: `registry.npmjs.org/-/npm/v1/tokens`, `ACTIONS_ID_TOKEN_REQUEST_URL/TOKEN`. NEEDS_CONTEXT substrings (IMDS IPs, GCE metadata host, k8s ServiceAccount path, Vault endpoint) require co-occurrence with EITHER a fetch verb (fetch/axios/http.get/etc) within 200 chars OR an `http(s)?://HOST` URL prefix OR a `host:`/`hostname:` config field. A defensive blocklist literal does not match any of those rules; an actual outbound call always does. cred-surface-path detection moved out of the bare-text scan into `scan_package_json` and scoped to the 4 NPM lifecycle hooks (preinstall / install / postinstall / prepare). A `/.npmrc` reference in a `docker` dev script is silent; a `cat ~/.npmrc | curl ...` in a `postinstall` fires HIGH. Per-file size cap split by content type, sniffed via 16-byte magic header read (ELF / Mach-O / PE / WASM / archive formats), plus suffix list (.node/.wasm/.so/.dll/.dylib/.exe), plus regex for versioned shared libs (libfoo.so.8.17.3), plus a null-byte ratio fallback for extensionless binaries that headers do not catch. Text files: 16 MiB cap (still tight; typescript.js at 9.1 MB is the legitimate ceiling). Binary files: 256 MiB cap (next-swc .node is 137 MB; sharp libvips is ~18 MB; rolldown bindings are 18-26 MB each). Cumulative: 512 MiB per tarball. Tarball: 256 MiB compressed. Binary files are also skipped in the content scanner -- regex over compiled machine code is noise. The IOC substring fallback in `scan_extracted_tree` now uses the same magic-sniff to decide whether to grep. HTTP timeout bumped 30s -> 60s for large tarballs. Verified ======== - AST parse OK. - 11 fault-injection tests pass: * zip-slip, symlink, oversized-declared-size, count-cap * router_init.js IOC detected * IMDS-in-URL still detected (new contextual rule) * langchain SSRF blocklist no longer false-positive * object-treeify docker script no longer false-positive * lifecycle-script `cat ~/.npmrc | curl ...` detected * synthetic ELF (extensionless executable) extracts and is correctly skipped from text scan * versioned `.so.8.17.3` shared lib extracts cleanly - Real-network end-to-end on the full lockfile: 968 packages, 0 findings, 0 hard errors, 76 seconds. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/security-audit.yml | 80 ++ scripts/scan_npm_packages.py | 1201 ++++++++++++++++++++++++++ 2 files changed, 1281 insertions(+) create mode 100755 scripts/scan_npm_packages.py diff --git a/.github/workflows/security-audit.yml b/.github/workflows/security-audit.yml index df0af95dfa..7ab1021ec9 100644 --- a/.github/workflows/security-audit.yml +++ b/.github/workflows/security-audit.yml @@ -57,6 +57,7 @@ on: - 'studio/src-tauri/Cargo.lock' - 'pyproject.toml' - 'scripts/scan_packages.py' + - 'scripts/scan_npm_packages.py' - '.github/workflows/security-audit.yml' push: branches: [main, pip] @@ -815,3 +816,82 @@ jobs: logs-scan-packages-${{ matrix.shard.id }}.txt audit-reqs/ retention-days: 30 + + # ───────────────────────────────────────────────────────────────────── + # npm: pre-install tarball content scan. + # ───────────────────────────────────────────────────────────────────── + npm-scan-packages: + # Counterpart to pip-scan-packages for the npm side. Reads + # studio/frontend/package-lock.json, downloads each resolved + # tarball DIRECTLY from registry.npmjs.org (never via `npm + # install` -- no lifecycle scripts ever run), verifies the + # lockfile integrity hash, unpacks each tarball into a sandboxed + # temp dir behind size / count / path-escape / symlink guards, + # and pattern-scans the extracted file contents for the + # signatures common to npm supply-chain attacks: + # + # - lifecycle (preinstall / install / postinstall / prepare) + # scripts in any package.json that fetch + execute external + # code, + # - C2 / exfiltration hosts (getsession.org, AWS IMDS, + # Kubernetes ServiceAccount token paths, GitHub Actions OIDC, + # HashiCorp Vault endpoints), + # - credential-stealing references (.npmrc, .aws/credentials, + # GITHUB_TOKEN / NPM_TOKEN in JS sources), + # - known IOC filenames (router_init.js, tanstack_runner.js, + # router_runtime.js), + # - obfuscation shapes (Function/eval against base64 blobs). + # + # Threat model: every tarball is hostile. Safety guarantees are + # documented at scripts/scan_npm_packages.py top-of-file. The + # script is stdlib-only so adding it does not increase the + # transitive supply-chain surface. + name: npm scan-packages (Studio frontend tarballs) + runs-on: ubuntu-latest + timeout-minutes: 30 + needs: [] + steps: + - name: Harden runner (egress audit) + uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1 + with: + egress-policy: audit + disable-sudo: true + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + + - name: Sanity-check scan_npm_packages.py + run: | + test -f scripts/scan_npm_packages.py + python3 -c "import ast; ast.parse(open('scripts/scan_npm_packages.py').read())" + + - name: Scan npm tarballs (declared + transitive, no install) + # The script exits 1 on HIGH/CRITICAL findings; we capture the + # full log and surface it in the step summary either way. It + # never runs `npm install`, never executes anything from a + # downloaded tarball, and only fetches from registry.npmjs.org. + # Initially non-blocking so the baseline can settle; drop + # continue-on-error once the baseline is clean for a week. + continue-on-error: true + run: | + set -o pipefail + LOG=logs-scan-npm.txt + python3 scripts/scan_npm_packages.py 2>&1 | tee "$LOG" + { + echo "## scan_npm_packages" + echo + echo '### Findings (tail)' + echo '```' + tail -300 "$LOG" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: scan-npm-packages-log + path: logs-scan-npm.txt + retention-days: 30 diff --git a/scripts/scan_npm_packages.py b/scripts/scan_npm_packages.py new file mode 100755 index 0000000000..97b5ffa9a4 --- /dev/null +++ b/scripts/scan_npm_packages.py @@ -0,0 +1,1201 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +# +# .github/workflows/security-audit.yml's npm-scan-packages job depends +# on this file existing at scripts/scan_npm_packages.py. + +"""scan_npm_packages.py -- npm-side content scanner. + +Counterpart to scripts/scan_packages.py for the pip ecosystem. Reads +studio/frontend/package-lock.json, downloads each resolved tarball +DIRECTLY from registry.npmjs.org (never via `npm install` -- no +lifecycle scripts ever run), verifies the lockfile integrity hash, +unpacks each tarball into a sandboxed temp dir behind size / count / +path-escape / symlink guards, and pattern-scans the extracted file +contents for the signatures common to npm supply-chain attacks: + + - Lifecycle (preinstall / install / postinstall / prepare) scripts + in any package.json that fetch + execute external code. + - C2 / exfiltration hosts (getsession.org, AWS IMDS endpoints, + Kubernetes ServiceAccount token paths, GitHub Actions OIDC, + HashiCorp Vault endpoints). + - Credential-stealing references (~/.npmrc, ~/.aws/credentials, + GITHUB_TOKEN / NPM_TOKEN in JS sources). + - Known IOC filenames from public advisories + (router_init.js, tanstack_runner.js, router_runtime.js). + - Obfuscation shapes (large single JS in package root with a low + whitespace ratio + Function/eval against a base64-decoded blob). + +Safety stance +============= + +This script ingests attacker-controlled archives. Every parse path +assumes the worst: + + 1. Downloads ONLY from `registry.npmjs.org`. Any tarball URL with a + different hostname is refused without fetching. + 2. Tarball download is size-capped (HARD_MAX_TARBALL_BYTES default + 64 MiB). HEAD-style probe via the Content-Length response header + plus a chunked read that aborts on overflow. + 3. SHA-512 integrity verified against the lockfile entry BEFORE the + tarball is even opened. A mismatch aborts that package -- the + scanner does not "fall back" to the registry-published hash. + 4. tar extraction goes through `safe_extract`: + - rejects symbolic links (`SYMTYPE`, `LNKTYPE`) + - rejects absolute paths, `..` traversal, paths outside the + extract root after resolution + - rejects character / block / FIFO devices + - per-file uncompressed size cap (HARD_MAX_FILE_BYTES, default + 8 MiB) AND cumulative cap (HARD_MAX_TOTAL_BYTES, default + 128 MiB) AND member-count cap (HARD_MAX_MEMBERS, default + 50_000) + - tar reads happen via `tarfile.open(mode='r|gz')` streaming + so an oversized file is detected before write + 5. NOTHING from the extracted tree is ever executed. Files are read + as raw bytes, decoded with `errors='replace'`, and grepped. We + never call `node`, `eval`, `compile`, `subprocess.run`, + `os.system`, or anything that would touch the tarball's + declared scripts. + 6. Tempdir is created with `tempfile.mkdtemp(prefix='npm-scan-')`, + fully resolved with .resolve(), and registered with atexit to be + wiped on every termination path. + 7. Stdlib only. No third-party deps -- adding one would itself be a + supply-chain liability. + +Exit codes +========== + + 0 no findings of severity HIGH or higher + 1 one or more HIGH/CRITICAL findings (or pre-scan structural + anomalies -- non-registry resolved URL, missing integrity) + 2 internal error (lockfile missing, integrity mismatch on + download, malformed tarball, etc.) + +The script is meant to be run in CI on every PR that touches +package-lock.json and on a nightly schedule. +""" + +from __future__ import annotations + +import argparse +import atexit +import base64 as _b64 # imported only so the IOC string-scan can detect it +import hashlib +import io +import json +import os +import re +import shutil +import sys +import tarfile +import tempfile +import urllib.parse +import urllib.request +from dataclasses import dataclass, field +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] + +# ───────────────────────────────────────────────────────────────────── +# Hard caps (deliberately conservative; npm tarballs in this repo are +# all well under these limits, so a packaging spike is noticeable). +# ───────────────────────────────────────────────────────────────────── +# Caps calibrated against the real Studio frontend transitive closure: +# - typescript.js is 9.1 MB (TS compiler bundled into one file) +# - mermaid 11.x dist/mermaid.js.map is ~12 MB (sourcemap) +# - lightningcss-linux-x64-{gnu,musl}.node is 10 MB +# - rolldown bindings (.node) are 18-26 MB per platform +# - @next/swc-*.node is ~137 MB (rust-compiled SWC engine) +# - next.js cumulative bundle is ~134 MB (turbopack compiled) +# +# Native binaries (.node, .wasm, .so, .dll, .dylib) are GENUINELY +# huge and not amenable to text pattern scanning -- we extract them +# only to verify the tarball integrity over the full archive, then +# skip them in scan_extracted_tree. They get a much higher per-file +# cap. Text files (JS/TS/JSON/etc) keep the tight cap because the +# pattern scanner runs over them and a 9.1 MB typescript.js is the +# legitimate ceiling. +HARD_MAX_TARBALL_BYTES = 256 * 1024 * 1024 # 256 MiB compressed +HARD_MAX_TEXT_FILE_BYTES = 16 * 1024 * 1024 # 16 MiB per text file +HARD_MAX_BINARY_FILE_BYTES = 256 * 1024 * 1024 # 256 MiB per .node etc +HARD_MAX_TOTAL_BYTES = 512 * 1024 * 1024 # 512 MiB cumulative +HARD_MAX_MEMBERS = 50_000 # entries per tarball +HARD_HTTP_TIMEOUT_S = 60 # per request + +# Native-binary / compiled-asset suffixes that bypass the text cap. +# This is the SUFFIX shortlist; the content-magic check below covers +# extensionless executables (biome) and versioned shared libraries +# (libvips-cpp.so.8.17.3) that the suffix list misses. +_BINARY_SUFFIXES = ( + ".node", + ".wasm", + ".so", + ".dll", + ".dylib", + ".exe", + ".a", + ".lib", + ".o", + ".obj", + ".bin", + ".dat", + ".woff", + ".woff2", + ".ttf", + ".otf", + ".eot", + ".png", + ".jpg", + ".jpeg", + ".gif", + ".webp", + ".ico", + ".mp3", + ".mp4", + ".webm", + ".zip", + ".tar", + ".gz", + ".tgz", + ".xz", + ".bz2", +) + +# Versioned shared libraries: libfoo.so.1.2.3 / libfoo.dylib.1.2. +_VERSIONED_LIB = re.compile( + r"\.(?:so|dylib)(?:\.\d+)+$", + re.IGNORECASE, +) + +# Magic numbers at offset 0 that identify common executable formats. +# We sniff the first ~16 bytes of every member to catch extensionless +# binaries (eg `package/biome`, `package/bin/foo`). +_BINARY_MAGICS = ( + b"\x7fELF", # ELF (Linux executable / .so) + b"MZ", # PE / .exe / .dll (DOS header prefix) + b"\xfe\xed\xfa\xce", # Mach-O 32 BE + b"\xfe\xed\xfa\xcf", # Mach-O 64 BE + b"\xce\xfa\xed\xfe", # Mach-O 32 LE + b"\xcf\xfa\xed\xfe", # Mach-O 64 LE + b"\xca\xfe\xba\xbe", # Mach-O fat / Java class (also starts with this) + b"\x00asm", # WASM + b"PK\x03\x04", # ZIP / JAR / nupkg / xpi + b"PK\x05\x06", # ZIP (empty) + b"\x1f\x8b", # gzip + b"BZh", # bzip2 + b"\xfd7zXZ", # xz + b"7z\xbc\xaf\x27\x1c", # 7zip + b"\x89PNG", # PNG + b"\xff\xd8\xff", # JPEG + b"GIF8", # GIF + b"RIFF", # WAV / WEBP / AVI container + b"\x00\x00\x01\x00", # ICO + b"OggS", # Ogg + b"\x1aE\xdf\xa3", # Matroska / WebM +) + + +def _looks_binary(name: str, header: bytes) -> bool: + """True if `name` or first bytes suggest a non-text file.""" + lower = name.lower() + if lower.endswith(_BINARY_SUFFIXES): + return True + if _VERSIONED_LIB.search(lower): + return True + for magic in _BINARY_MAGICS: + if header.startswith(magic): + return True + # Null-byte density: real text files almost never carry NULs. + if header and (header.count(b"\x00") / len(header)) > 0.02: + return True + return False + + +ALLOWED_DOWNLOAD_HOST = "registry.npmjs.org" + +# ───────────────────────────────────────────────────────────────────── +# Severities + finding shape (mirrors scripts/scan_packages.py). +# ───────────────────────────────────────────────────────────────────── +CRITICAL = "CRITICAL" +HIGH = "HIGH" +MEDIUM = "MEDIUM" +INFO = "INFO" +_SEVERITY_RANK = {CRITICAL: 0, HIGH: 1, MEDIUM: 2, INFO: 3} + + +@dataclass +class Finding: + severity: str + package: str # name@version + filename: str # relative path inside the tarball + pattern: str # what matched + evidence: str = "" # short surrounding snippet + detail: str = "" # human-readable description + + def __str__(self) -> str: + head = f" [{self.severity}] {self.package} :: {self.filename}" + body = f" pattern: {self.pattern}" + if self.detail: + body += f"\n detail: {self.detail}" + if self.evidence: + ev = self.evidence + if len(ev) > 240: + ev = ev[:240] + "..." + body += f"\n evidence: {ev!r}" + return f"{head}\n{body}" + + +@dataclass +class PackageEntry: + name: str + version: str + resolved: str + integrity: str | None + lockfile_key: str + + @property + def display(self) -> str: + return f"{self.name}@{self.version}" + + +# ───────────────────────────────────────────────────────────────────── +# IOC patterns. Two flavours: +# - HOSTS / TOKEN_PATHS: high-confidence substrings; near-zero FP rate +# - JS_PATTERNS / SCRIPT_PATTERNS: regex; tuned to recent campaigns +# Keep this list short and factual. Speculative patterns spam the +# false-positive ledger and dull the signal. +# ───────────────────────────────────────────────────────────────────── + + +# Substring (case-sensitive) -> (severity, detail). +KNOWN_IOC_STRINGS: dict[str, tuple[str, str]] = { + # Shai-Hulud TanStack wave (2026-05-11, GHSA-g7cv-rxg3-hmpx). + "router_init.js": (HIGH, "filename associated with TanStack worm"), + "tanstack_runner.js": (HIGH, "filename associated with TanStack worm"), + "router_runtime.js": (HIGH, "filename associated with TanStack worm"), + "A Mini Shai-Hulud has Appeared": ( + CRITICAL, + "TanStack worm campaign stdout marker", + ), + "github:tanstack/router#79ac49eedf774dd4b0cfa308722bc463cfe5885c": ( + CRITICAL, + "TanStack worm dropper pinned commit", + ), + # Exfil hosts observed across both Shai-Hulud waves. + "filev2.getsession.org": (CRITICAL, "exfiltration C2 host"), + "getsession.org/file/": (CRITICAL, "exfiltration C2 endpoint"), +} + +# Cloud / k8s / CI credential surfaces. A bare substring match here +# false-positives on DEFENSIVE code -- e.g. langchain ships an SSRF +# protection module with a literal blocklist of IMDS IPs. We split +# these into two tiers: +# +# ALWAYS_BAD: substrings with no legitimate use anywhere in a +# dependency. A bare match is enough. +# +# NEEDS_CONTEXT: hosts/paths that DO appear legitimately in +# defensive code. We only fire when they co-occur with a fetch +# verb or appear inside an http URL -- that is the structural +# difference between "blocked address constant" and "exfil +# target". +# +# The dispatch lives in `_scan_cred_surface` below. + +CRED_HOST_ALWAYS_BAD: tuple[tuple[str, str], ...] = ( + ("registry.npmjs.org/-/npm/v1/tokens", "npm publish-token enumeration endpoint"), + ("ACTIONS_ID_TOKEN_REQUEST_URL", "GitHub Actions OIDC token-exchange endpoint env"), + ("ACTIONS_ID_TOKEN_REQUEST_TOKEN", "GitHub Actions OIDC token-exchange token env"), +) + +# Hosts that need fetch-verb or URL-scheme context to be malicious. +CRED_HOST_NEEDS_CONTEXT: tuple[tuple[str, str], ...] = ( + ("169.254.169.254", "AWS / GCP / Azure instance metadata service (IMDS)"), + ("169.254.170.2", "ECS task metadata service"), + ("metadata.google.internal", "GCE metadata service"), + ("vault.svc.cluster.local", "in-cluster HashiCorp Vault endpoint"), + ( + "/var/run/secrets/kubernetes.io/serviceaccount", + "Kubernetes ServiceAccount token path", + ), +) + +# Credentials a frontend package should NEVER need to read. Bare +# substring match is too noisy (object-treeify ships a `docker` dev +# script that mounts ~/.npmrc -- legitimate dev tooling, never run +# at install time). We instead surface these only when they appear +# inside a LIFECYCLE script (preinstall / install / postinstall / +# prepare), which is the only path that runs automatically on +# `npm ci`. See `scan_package_json` below. +CRED_PATH_SUBSTRINGS: tuple[tuple[str, str], ...] = ( + ("/.npmrc", "npm credentials file"), + ("/.aws/credentials", "AWS shared credentials file"), + ("/.ssh/id_rsa", "SSH private key"), + ("/.ssh/id_ed25519", "SSH private key"), + ("/.docker/config.json", "Docker registry credentials"), + ("/.kube/config", "Kubernetes kubeconfig"), +) + +# Fetch verbs whose presence near a metadata host upgrades a bare +# substring hit into an actionable finding. +_FETCH_VERBS_PAT = ( + r"(?:fetch|axios|XMLHttpRequest|got\b|undici|" + r"http\.get|https\.get|http\.request|https\.request|" + r"new\s+URL|url\.parse|net\.connect|" + r"\.request\s*\(|\.get\s*\(\s*['\"]\s*https?://)" +) + +# JS regex patterns (compile lazily). +_JS_FETCH_EVAL = re.compile( + r"""(?xs) + (?: + Function\s*\(\s*['"`] # new Function("...") + | eval\s*\(\s*['"`] + | \(\s*0\s*,\s*eval\s*\)\s*\( + ) + .{0,200} + (?:atob\s*\(|Buffer\s*\.from\s*\([^)]+,\s*['"]base64) + """, +) + +# `process.env.GITHUB_TOKEN` / `NPM_TOKEN` / `AWS_*` access in +# top-level / install-time code is suspicious. We also catch +# `os.environ["GITHUB_TOKEN"]` for the rare Python-in-npm postinstall. +_JS_ENV_TOKEN = re.compile( + r"""(process\.env\.|os\.environ\[?['"])(?: + GITHUB_TOKEN | GH_TOKEN | NPM_TOKEN | NODE_AUTH_TOKEN + | AWS_ACCESS_KEY_ID | AWS_SECRET_ACCESS_KEY | AWS_SESSION_TOKEN + | GOOGLE_APPLICATION_CREDENTIALS + | DOCKER_AUTH_CONFIG | VAULT_TOKEN + )['"]?\]?""", + re.VERBOSE, +) + +# Suspicious lifecycle-script payloads. Anything in a package.json +# `scripts` field that wgets/curls an external resource and executes +# it. We do NOT block ALL curl/wget in scripts (some legit packages +# fetch test fixtures into devDependencies), but we DO block the +# fetch+exec chain. +_LIFECYCLE_FETCH_EXEC = re.compile( + r"""(?xs) + (?:curl|wget|fetch|http\.get|axios\.get)\s+ # fetch verb + .{0,200} + (?:\|\s*(?:sh|bash|node|python|eval)\b # pipe to interpreter + | \&\&\s*(?:sh|bash|node|python|eval)\b # &&-chain to interpreter + | -o\s+\S+\s*&&\s*(?:sh|bash|node|python) # download then run + | --post-file\s+ + | \$\(.*\) # command-sub of fetched content + ) + """, +) + +# Obfuscation: large JS file that is mostly one line of base64-ish +# blob with a Function() / eval() bookend. Tuned against the +# router_init.js shape (2.3 MB obfuscated single-blob). +_OBFUSC_BLOB = re.compile( + r"""(?xs) + (?:Function|eval)\s*\(\s*['"`]? + [A-Za-z0-9+/=_-]{2048,} # >=2 KiB of b64-ish + """, +) + + +# ───────────────────────────────────────────────────────────────────── +# Lockfile parsing. +# ───────────────────────────────────────────────────────────────────── + + +def parse_lockfile(path: Path) -> tuple[list[PackageEntry], list[Finding]]: + """Return (entries, structural_findings). + + Structural findings here are HIGH-severity refusals that should + short-circuit the scan -- a lockfile with non-registry resolved + URLs is itself a finding (covered by scripts/lockfile_supply_chain + _audit.py in detail; we surface a summary here so this scanner is + standalone-runnable). + """ + entries: list[PackageEntry] = [] + findings: list[Finding] = [] + + try: + lock = json.loads(path.read_text(encoding = "utf-8")) + except (OSError, json.JSONDecodeError) as exc: + findings.append( + Finding( + severity = CRITICAL, + package = "", + filename = str(path), + pattern = "lockfile-unreadable", + detail = f"could not parse: {exc}", + ) + ) + return entries, findings + + if lock.get("lockfileVersion") not in (2, 3): + findings.append( + Finding( + severity = HIGH, + package = "", + filename = str(path), + pattern = "unsupported-lockfile-version", + detail = ( + f"only lockfileVersion 2 or 3 supported; got " + f"{lock.get('lockfileVersion')!r}" + ), + ) + ) + return entries, findings + + for key, entry in (lock.get("packages") or {}).items(): + if key == "" or entry.get("link"): + continue + # Nested fold-ins (deps inside another package's node_modules/) + # are covered by the parent tarball's integrity. Skip. + if key.count("/node_modules/") >= 1: + continue + resolved = entry.get("resolved") + if not resolved: + continue + # Strict registry origin check. lockfile_supply_chain_audit + # already catches this; double-defend here so this scanner + # cannot be tricked into fetching from an attacker-chosen URL. + parsed = urllib.parse.urlparse(resolved) + if parsed.scheme != "https" or parsed.hostname != ALLOWED_DOWNLOAD_HOST: + findings.append( + Finding( + severity = CRITICAL, + package = key, + filename = str(path), + pattern = "non-registry-resolved-url", + detail = ( + f"resolved={resolved!r}; only " + f"https://{ALLOWED_DOWNLOAD_HOST}/ is " + "permitted. Refusing to download." + ), + ) + ) + continue + integrity = entry.get("integrity") + if not integrity: + findings.append( + Finding( + severity = HIGH, + package = key, + filename = str(path), + pattern = "missing-integrity-hash", + detail = "no `integrity` field; cannot verify download", + ) + ) + continue + # node_modules/@scope/name -> @scope/name; node_modules/name -> name + nm = "node_modules/" + name = key[len(nm) :] if key.startswith(nm) else key + version = entry.get("version") or "" + entries.append( + PackageEntry( + name = name, + version = version, + resolved = resolved, + integrity = integrity, + lockfile_key = key, + ) + ) + return entries, findings + + +# ───────────────────────────────────────────────────────────────────── +# Tarball download (registry-only, size-capped, integrity-verified). +# ───────────────────────────────────────────────────────────────────── + + +def _decode_integrity(integrity: str) -> tuple[str, bytes] | None: + """Parse SRI integrity 'sha512-' -> (algo, digest_bytes).""" + if "-" not in integrity: + return None + algo, b64 = integrity.split("-", 1) + algo = algo.strip().lower() + if algo not in ("sha256", "sha384", "sha512"): + return None + try: + digest = _b64.b64decode(b64, validate = True) + except Exception: + return None + return algo, digest + + +def download_tarball( + entry: PackageEntry, + dest: Path, + *, + timeout: float = HARD_HTTP_TIMEOUT_S, + max_bytes: int = HARD_MAX_TARBALL_BYTES, +) -> tuple[Path, str | None]: + """Stream-download entry.resolved to dest. Verify SRI integrity. + + Returns (downloaded_path, error_or_none). On any error the + returned path may not exist. Network access is restricted to + https://{ALLOWED_DOWNLOAD_HOST}/ -- the caller passes a Request + we already validated. + """ + # Re-assert hostname; the entry was validated at parse time but a + # defence-in-depth check here means a future refactor cannot + # accidentally bypass it. + parsed = urllib.parse.urlparse(entry.resolved) + if parsed.scheme != "https" or parsed.hostname != ALLOWED_DOWNLOAD_HOST: + return dest, (f"refused download from non-allowlisted URL {entry.resolved!r}") + + decoded = _decode_integrity(entry.integrity or "") + if decoded is None: + return dest, f"unparseable integrity field {entry.integrity!r}" + algo, expected_digest = decoded + h = hashlib.new(algo) + + req = urllib.request.Request( + entry.resolved, + headers = { + "User-Agent": "unsloth-scan-npm-packages/1.0 (+supply-chain audit)", + "Accept": "application/octet-stream", + }, + method = "GET", + ) + try: + with urllib.request.urlopen(req, timeout = timeout) as r: + # Advertised length, if any. + cl = r.headers.get("Content-Length") + if cl is not None: + try: + cl_int = int(cl) + if cl_int > max_bytes: + return dest, (f"Content-Length {cl_int} > cap {max_bytes}") + except ValueError: + pass + written = 0 + with open(dest, "wb") as out: + while True: + chunk = r.read(64 * 1024) + if not chunk: + break + written += len(chunk) + if written > max_bytes: + return dest, ( + f"download exceeded cap {max_bytes} bytes " + f"after {written} bytes" + ) + h.update(chunk) + out.write(chunk) + except Exception as exc: + return dest, f"download failed: {exc}" + + actual = h.digest() + if actual != expected_digest: + return dest, ( + f"integrity mismatch: expected {algo}={_b64.b64encode(expected_digest).decode()!r}, " + f"got {algo}={_b64.b64encode(actual).decode()!r}" + ) + return dest, None + + +# ───────────────────────────────────────────────────────────────────── +# Safe tar extraction. Every Tarfile member is policed before write. +# ───────────────────────────────────────────────────────────────────── + + +def _is_within(root: Path, candidate: Path) -> bool: + try: + return candidate.resolve().is_relative_to(root.resolve()) + except (AttributeError, ValueError): + # Python <3.9 fallback (we target 3.10+ but be defensive). + try: + candidate.resolve().relative_to(root.resolve()) + return True + except Exception: + return False + + +def safe_extract( + tarball_path: Path, + extract_root: Path, + *, + max_total_bytes: int = HARD_MAX_TOTAL_BYTES, + max_members: int = HARD_MAX_MEMBERS, +) -> str | None: + """Extract tarball_path under extract_root with policed members. + + Returns None on success, or a string describing the refusal. + Streams via `r|gz` so we can abort mid-extraction without having + materialised the rest of the archive. + """ + extract_root.mkdir(parents = True, exist_ok = True) + total = 0 + count = 0 + try: + # Open in streaming mode so we never seek backwards in the + # input. `r|gz` rejects malformed gzip frames immediately. + with tarfile.open(tarball_path, mode = "r|gz") as tf: + for member in tf: + count += 1 + if count > max_members: + return f"member count {count} exceeded cap {max_members}" + name = member.name + # Reject obvious path-escape. + if name.startswith("/") or ".." in Path(name).parts: + return f"refused unsafe member name {name!r}" + # Reject device files, FIFOs, sockets, symlinks, hardlinks. + if member.issym() or member.islnk(): + return f"refused link member {name!r} (sym/lnk)" + if member.isdev() or member.isfifo(): + return f"refused special member {name!r}" + # Cumulative cap is checked against DECLARED size up + # front to short-circuit obvious bombs without reading + # the body. + declared = max(member.size, 0) + if declared > HARD_MAX_BINARY_FILE_BYTES: + return ( + f"member {name!r} declared size {declared} > " + f"absolute cap {HARD_MAX_BINARY_FILE_BYTES}" + ) + if total + declared > max_total_bytes: + return ( + f"cumulative bytes {total + declared} > cap " + f"{max_total_bytes} at {name!r}" + ) + # Strip leading "package/" -- the npm convention. We do + # NOT trust npm to be right, so we explicitly resolve + # the destination and refuse anything that escapes. + dest = extract_root / name + if not _is_within(extract_root, dest): + return f"refused escape: {name!r} resolved outside root" + if member.isdir(): + dest.mkdir(parents = True, exist_ok = True) + continue + if not member.isfile(): + # Anything we didn't classify above is unknown. + return f"refused unknown member type for {name!r}" + dest.parent.mkdir(parents = True, exist_ok = True) + src = tf.extractfile(member) + if src is None: + continue + # Sniff first 16 bytes to classify text vs binary. + # Text-cap members get the tight 16 MiB limit; binary + # members (executables, .node, .wasm, native libs) + # get the generous binary cap. We bound BOTH cases. + header = src.read(16) + is_binary = _looks_binary(name, header) + file_cap = ( + HARD_MAX_BINARY_FILE_BYTES + if is_binary + else HARD_MAX_TEXT_FILE_BYTES + ) + if declared > file_cap: + return ( + f"member {name!r} declared size {declared} > " + f"cap {file_cap} ({'binary' if is_binary else 'text'})" + ) + # Read remainder, bounded. + remainder_cap = file_cap - len(header) + rest = src.read(remainder_cap + 1) + data = header + rest + if len(data) > file_cap: + return ( + f"member {name!r} body exceeded declared size cap " + f"({'binary' if is_binary else 'text'})" + ) + total += len(data) + # Write with restrictive mode (rw-r--r--) so even if + # someone runs the extract dir nothing is executable. + with open(dest, "wb") as out: + out.write(data) + os.chmod(dest, 0o644) + except tarfile.TarError as exc: + return f"tar parse error: {exc}" + except Exception as exc: + return f"unexpected extract error: {exc!r}" + return None + + +# ───────────────────────────────────────────────────────────────────── +# Content scanning. +# ───────────────────────────────────────────────────────────────────── + + +def _evidence(text: str, pat: re.Pattern, max_chars: int = 200) -> str: + m = pat.search(text) + if not m: + return "" + start = max(0, m.start() - 30) + end = min(len(text), m.end() + 30) + snippet = text[start:end].replace("\n", " ") + if len(snippet) > max_chars: + snippet = snippet[:max_chars] + "..." + return snippet + + +LIFECYCLE_HOOKS = ("preinstall", "install", "postinstall", "prepare") + + +def scan_package_json( + pkg: PackageEntry, + rel: str, + text: str, +) -> list[Finding]: + findings: list[Finding] = [] + try: + meta = json.loads(text) + except Exception: + return findings + if not isinstance(meta, dict): + return findings + scripts = meta.get("scripts") or {} + if not isinstance(scripts, dict): + return findings + for hook in LIFECYCLE_HOOKS: + body = scripts.get(hook) + if not isinstance(body, str): + continue + if _LIFECYCLE_FETCH_EXEC.search(body): + findings.append( + Finding( + severity = CRITICAL, + package = pkg.display, + filename = rel, + pattern = f"lifecycle-fetch-exec ({hook})", + evidence = body, + detail = ( + f"`scripts.{hook}` fetches an external " + "resource and pipes/chains it to an " + "interpreter; this is the install-time RCE " + "vector. Refusing to install." + ), + ) + ) + # Credential file paths inside a lifecycle script are + # exfiltration prep -- npm runs these scripts automatically + # on `npm ci`. Manual `scripts.*` entries (like a `docker` + # dev script) are out of scope: npm does not run them. + for path_substr, why in CRED_PATH_SUBSTRINGS: + if path_substr in body: + findings.append( + Finding( + severity = HIGH, + package = pkg.display, + filename = rel, + pattern = f"cred-path-in-lifecycle ({hook})", + evidence = body, + detail = ( + f"`scripts.{hook}` references {why} " + f"({path_substr!r}); install-time access " + "to local credential files is the " + "exfiltration prep step" + ), + ) + ) + if _JS_ENV_TOKEN.search(body): + findings.append( + Finding( + severity = HIGH, + package = pkg.display, + filename = rel, + pattern = f"cred-env-in-lifecycle ({hook})", + evidence = _evidence(body, _JS_ENV_TOKEN), + detail = ( + f"`scripts.{hook}` references a credential " + "env var (GITHUB_TOKEN / NPM_TOKEN / AWS_* " + "/ etc); install-time access to runner " + "secrets is the exfiltration prep step" + ), + ) + ) + # Optional deps pointing at github: are the TanStack-style + # injection vector. + opt = meta.get("optionalDependencies") or {} + if isinstance(opt, dict): + for k, v in opt.items(): + if isinstance(v, str) and ( + v.startswith("github:") + or v.startswith("git+") + or v.startswith("git://") + ): + findings.append( + Finding( + severity = HIGH, + package = pkg.display, + filename = rel, + pattern = "optional-dep-non-registry", + evidence = f"{k}={v}", + detail = ( + "package.json `optionalDependencies` " + "points at a non-registry source; this " + "is the Shai-Hulud worm injection shape." + ), + ) + ) + return findings + + +def _host_in_outbound_context(text: str, host: str) -> bool: + """True if `host` appears in a way consistent with an outbound call. + + A bare `"169.254.169.254"` array literal (defensive blocklist) is + safe; a `fetch("http://169.254.169.254/...")` is not. The signal + is co-occurrence with either an HTTP URL scheme or a fetch verb + within a short window. + + A defensive blocklist looks like: + const CLOUD_METADATA_IPS = ["169.254.169.254", "169.254.170.2"]; + An exfil call looks like: + fetch("http://169.254.169.254/latest/meta-data/...") + http.request({ host: "169.254.169.254", path: "/..." }) + """ + # Esc for use in a regex (IPs contain dots). + host_re = re.escape(host) + # 1. URL form: http://host or https://host or //host/ or //host" + url_form = re.compile( + rf"(?:https?:)?//{host_re}(?:[:/\"'?#]|$)", + ) + if url_form.search(text): + return True + # 2. host appears within 200 chars of a fetch verb (either side). + fetch_context = re.compile( + rf"(?:{_FETCH_VERBS_PAT})[^\n]{{0,200}}{host_re}" + rf"|{host_re}[^\n]{{0,200}}(?:{_FETCH_VERBS_PAT})", + re.IGNORECASE, + ) + if fetch_context.search(text): + return True + # 3. `host:` / `hostname:` config field referencing the IP. + cfg_form = re.compile( + rf"(?:host|hostname)\s*:\s*['\"`]{host_re}['\"`]", + re.IGNORECASE, + ) + if cfg_form.search(text): + return True + return False + + +def scan_text_blob( + pkg: PackageEntry, + rel: str, + text: str, +) -> list[Finding]: + findings: list[Finding] = [] + + # IOC substrings (literal, case-sensitive). + for needle, (sev, why) in KNOWN_IOC_STRINGS.items(): + if needle in text: + findings.append( + Finding( + severity = sev, + package = pkg.display, + filename = rel, + pattern = "known-ioc-string", + evidence = needle, + detail = f"{why}: {needle!r}", + ) + ) + + # Credential surfaces. Tier 1: hosts with no legitimate use, + # bare substring is enough. + for needle, why in CRED_HOST_ALWAYS_BAD: + if needle in text: + findings.append( + Finding( + severity = HIGH, + package = pkg.display, + filename = rel, + pattern = "cred-surface-host (always-bad)", + evidence = needle, + detail = ( + f"references {why} ({needle!r}); no legitimate " + "frontend use of this surface" + ), + ) + ) + + # Credential surfaces. Tier 2: hosts that do appear in defensive + # code; require co-occurrence with a fetch verb or URL prefix. + for needle, why in CRED_HOST_NEEDS_CONTEXT: + if needle in text and _host_in_outbound_context(text, needle): + findings.append( + Finding( + severity = HIGH, + package = pkg.display, + filename = rel, + pattern = "cred-surface-host (outbound)", + evidence = needle, + detail = ( + f"references {why} ({needle!r}) in an outbound " + "call / URL / host config; a defensive blocklist " + "literal would not match this rule" + ), + ) + ) + + # Credential PATHS are deliberately not scanned here; they have + # too high a false-positive rate at file scope (defensive code, + # docker mounts, AWS SDK docs strings). `scan_package_json` + # catches the malicious case -- credential paths inside a + # lifecycle script run automatically on `npm ci`. + + # JS-specific regex. + if _JS_FETCH_EVAL.search(text): + findings.append( + Finding( + severity = HIGH, + package = pkg.display, + filename = rel, + pattern = "js-fetch-eval", + evidence = _evidence(text, _JS_FETCH_EVAL), + detail = ( + "Function/eval against base64-decoded payload " + "(obfuscated dropper shape)" + ), + ) + ) + if _JS_ENV_TOKEN.search(text): + findings.append( + Finding( + severity = MEDIUM, + package = pkg.display, + filename = rel, + pattern = "js-env-token", + evidence = _evidence(text, _JS_ENV_TOKEN), + detail = ("references credential env vars in package source"), + ) + ) + if _OBFUSC_BLOB.search(text): + findings.append( + Finding( + severity = HIGH, + package = pkg.display, + filename = rel, + pattern = "obfuscated-blob", + evidence = _evidence(text, _OBFUSC_BLOB), + detail = ( + "large base64-ish blob fed to Function/eval; " + "matches the TanStack worm dropper shape" + ), + ) + ) + + return findings + + +# Filename suffix decides which scanners run. We deliberately treat +# *.cjs/*.mjs/*.ts the same as *.js -- attackers use whichever +# extension the consumer's bundler / loader resolves. +_TEXT_SUFFIXES = ( + ".js", + ".mjs", + ".cjs", + ".ts", + ".tsx", + ".json", + ".html", + ".htm", + ".sh", + ".bash", + ".zsh", + ".py", + ".rb", + ".yml", + ".yaml", +) + + +def scan_extracted_tree( + pkg: PackageEntry, + root: Path, +) -> list[Finding]: + findings: list[Finding] = [] + for path in sorted(root.rglob("*")): + if not path.is_file(): + continue + rel = path.relative_to(root).as_posix() + lower = rel.lower() + if not lower.endswith(_TEXT_SUFFIXES): + # Skip native binaries entirely -- regex over compiled + # machine code is just noise (false positives in WASM + # opcodes, .node BSS segments, image pixel data). Use + # content-magic detection so extensionless executables + # (eg `package/biome`) and versioned shared libraries + # are also skipped. + try: + if path.stat().st_size > HARD_MAX_TEXT_FILE_BYTES: + continue + with open(path, "rb") as fh: + header = fh.read(16) + if _looks_binary(rel, header): + continue + data = header + path.read_bytes()[len(header) :] + except OSError: + continue + text = data.decode("utf-8", errors = "replace") + for needle, (sev, why) in KNOWN_IOC_STRINGS.items(): + if needle in text: + findings.append( + Finding( + severity = sev, + package = pkg.display, + filename = rel, + pattern = "known-ioc-string", + evidence = needle, + detail = f"{why}: {needle!r}", + ) + ) + continue + try: + data = path.read_bytes() + except OSError: + continue + text = data.decode("utf-8", errors = "replace") + if rel.endswith("package.json"): + findings.extend(scan_package_json(pkg, rel, text)) + findings.extend(scan_text_blob(pkg, rel, text)) + return findings + + +# ───────────────────────────────────────────────────────────────────── +# Orchestrator. +# ───────────────────────────────────────────────────────────────────── + + +def scan_one( + pkg: PackageEntry, + workspace: Path, +) -> tuple[list[Finding], str | None]: + """Download + extract + scan a single package. Cleans up its dir. + + Returns (findings, error). `error` is non-None only on hard + failures (download error, integrity mismatch, malformed tarball); + on a clean run with findings the error is None and the caller + decides exit code based on severity. + """ + pkg_dir = workspace / f"{pkg.name.replace('/', '_')}-{pkg.version}" + pkg_dir.mkdir(parents = True, exist_ok = True) + tarball = pkg_dir / "pkg.tgz" + extract = pkg_dir / "x" + try: + _, err = download_tarball(pkg, tarball) + if err: + return [], err + err = safe_extract(tarball, extract) + if err: + return [], err + return scan_extracted_tree(pkg, extract), None + finally: + # Always wipe per-package data to keep the workspace bounded. + try: + shutil.rmtree(pkg_dir, ignore_errors = True) + except Exception: + pass + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description = "Pre-install npm tarball content scanner.", + ) + parser.add_argument( + "--lockfile", + default = str(REPO_ROOT / "studio" / "frontend" / "package-lock.json"), + help = "Path to package-lock.json (default: studio/frontend).", + ) + parser.add_argument( + "--max-packages", + type = int, + default = 0, + help = ( + "Cap on number of packages to scan (0 = no cap). Useful " + "for local triage; CI runs with 0." + ), + ) + parser.add_argument( + "--fail-on", + choices = ("info", "medium", "high", "critical"), + default = "high", + help = ( + "Lowest severity that fails the run (default: high). " + "Medium and below print but exit 0." + ), + ) + args = parser.parse_args(argv) + + lockfile = Path(args.lockfile).resolve() + if not lockfile.exists(): + print(f"[scan-npm] lockfile not found: {lockfile}", file = sys.stderr) + return 2 + + entries, struct_findings = parse_lockfile(lockfile) + if struct_findings: + print( + f"[scan-npm] {len(struct_findings)} structural finding(s) " + "from lockfile pass; subsequent download scan skipped for " + "those entries.", + flush = True, + ) + + if args.max_packages > 0: + entries = entries[: args.max_packages] + + workspace = Path(tempfile.mkdtemp(prefix = "npm-scan-")).resolve() + atexit.register(lambda: shutil.rmtree(workspace, ignore_errors = True)) + print( + f"[scan-npm] workspace: {workspace}\n" + f"[scan-npm] scanning {len(entries)} package(s) from {lockfile}", + flush = True, + ) + + all_findings: list[Finding] = list(struct_findings) + hard_errors: list[tuple[str, str]] = [] + + for i, pkg in enumerate(entries, start = 1): + print( + f"[scan-npm] [{i}/{len(entries)}] {pkg.display}", + flush = True, + ) + findings, err = scan_one(pkg, workspace) + if err: + hard_errors.append((pkg.display, err)) + print(f"[scan-npm] ERROR {pkg.display}: {err}", flush = True) + continue + all_findings.extend(findings) + for f in findings: + print(str(f), flush = True) + + # Sort by severity then package. + all_findings.sort(key = lambda f: (_SEVERITY_RANK[f.severity], f.package)) + + print( + f"\n[scan-npm] summary: {len(entries)} package(s), " + f"{len(all_findings)} finding(s), " + f"{len(hard_errors)} hard error(s)", + flush = True, + ) + + if hard_errors: + print("\n[scan-npm] HARD ERRORS:", file = sys.stderr) + for pkg, err in hard_errors: + print(f" {pkg}: {err}", file = sys.stderr) + + threshold = { + "info": INFO, + "medium": MEDIUM, + "high": HIGH, + "critical": CRITICAL, + }[args.fail_on] + threshold_rank = _SEVERITY_RANK[threshold] + blocking = [f for f in all_findings if _SEVERITY_RANK[f.severity] <= threshold_rank] + if hard_errors or blocking: + if blocking: + print( + f"\n[scan-npm] FAIL: {len(blocking)} finding(s) " + f"at or above {threshold}", + file = sys.stderr, + ) + return 1 + print("\n[scan-npm] OK", flush = True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 9d47eb2e955041f581cdf3ca40dde7fac8d06986 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 11 May 2026 20:37:24 -0700 Subject: [PATCH 05/25] studio/tests: AbortSignal-bound in-page fetches and wall-clock watchdog for Playwright probes (#5391) * studio/tests: AbortSignal-bound in-page fetches + wall-clock watchdog Run 25696797934 / job 75446949358 on PR #5387 cancelled the "Chat UI Tests" macos-14 job at 30 min: studio.log went idle after the chat surface mounted, no further requests reached the server, and Playwright silently sat on a `page.evaluate(async () => fetch( /api/inference/load))` for 27+ minutes before the runner-level timeout fired. The two other Chat UI Tests jobs on the same SHA passed in 5-17 min, so this was a transient renderer wedge under --single-process Chromium, not a regression from the security bumps in that PR. Root cause: Playwright's `page.evaluate(...)` has no `timeout=` argument. If the JS body awaits a fetch whose promise never settles (the renderer's network thread stalls behind the busy main thread on the free macos-14 runner), the entire Python script hangs until something external kills it. Add two helpers in `_playwright_robust.py`: - `evaluate_fetch(page, url, *, method, headers, body, timeout_ms)` wraps `fetch()` in an `AbortController` so the JS resolves either with a real response or with `{status: 0, error: "AbortError..."}` after the budget elapses. Callers fail loud on a non-None `error` and the wedge surfaces as a one-line diagnostic instead of a 30-min cancel. - `install_wall_clock_watchdog(deadline_s)` starts a daemon Timer that hard-exits the process at the deadline. Belt-and- suspenders for any wedge inside the browser that the per- action timeouts cannot bound. Default 720s (12 min); healthy runs measure 5-9 min on macos-14 so the headroom is small without amplifying a wedge to the 30-min runner cap. Wire both into `playwright_chat_ui.py` and `playwright_extra_ui.py`: - Replace every `page.evaluate(async () => fetch(...))` site with `evaluate_fetch(...)`: refresh-token exchange, defaults fetch, inference load, health probe, post-rotation refresh. Five sites in chat_ui, two in extra_ui. - Arm the watchdog at the top of `with sync_playwright()` and cancel it on clean exit. Knobs (all default-safe, override only for slow runners): STUDIO_UI_WALL_TIMEOUT_S (default 720s) STUDIO_UI_FETCH_TIMEOUT_MS (default 30000ms) STUDIO_UI_LOAD_TIMEOUT_MS (default 180000ms) Verified locally with `python -c "ast.parse(...)"` on all three files and a unit smoke that confirms `evaluate_fetch`'s JS argument shape and that `install_wall_clock_watchdog` returns a daemonised Timer that responds to `.cancel()`. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- tests/studio/_playwright_robust.py | 141 ++++++++++++++++++++++++++++ tests/studio/playwright_chat_ui.py | 130 +++++++++++++++---------- tests/studio/playwright_extra_ui.py | 61 +++++++----- 3 files changed, 263 insertions(+), 69 deletions(-) diff --git a/tests/studio/_playwright_robust.py b/tests/studio/_playwright_robust.py index 928fa242eb..3deeb38cda 100644 --- a/tests/studio/_playwright_robust.py +++ b/tests/studio/_playwright_robust.py @@ -21,7 +21,9 @@ It does NOT depend on pytest -- both consumers run as plain Python. from __future__ import annotations import json +import os import sys +import threading import time import urllib.error import urllib.request @@ -404,3 +406,142 @@ def dump_diagnostics( except Exception as exc: if info is not None: info(f"diagnostics: json sidecar {name} failed: {exc}") + + +# ───────────────────────────────────────────────────────────────────── +# Bounded in-page fetch. +# ───────────────────────────────────────────────────────────────────── +# +# Playwright's `page.evaluate(...)` has no `timeout=` argument. If the +# JS body awaits a fetch that never resolves (the renderer's network +# thread wedges, the server accepts the connection but never replies, +# the macos-14 free runner under --single-process Chromium loses its +# IPC pipe), the entire Python script hangs until the runner-level +# timeout fires. Run 25696797934 / job 75446949358 on PR #5387 showed +# this exact failure: studio.log went idle after the chat surface +# mounted, no further requests reached the server, and Playwright +# burned 27+ minutes on a single page.evaluate(fetch /api/inference/ +# load) before the 30-min runner cancel. +# +# `evaluate_fetch` wraps the fetch in an AbortController.signal so the +# JS side resolves either with a real response or with a synthetic +# `{status: 0, error: "AbortError..."}` after `timeout_ms` ms. Either +# way page.evaluate returns and the script proceeds (or fails) with +# a debuggable signal instead of a silent wedge. +def evaluate_fetch( + page: Any, + url: str, + *, + method: str = "GET", + headers: dict[str, str] | None = None, + body: Any = None, + timeout_ms: int = 20_000, +) -> dict[str, Any]: + """Run `fetch(url, opts)` inside the page with an AbortSignal deadline. + + Returns `{"status": int, "body": parsed_or_text, "error": str|None}`. + On AbortSignal timeout returns `{"status": 0, "body": None, "error": + "AbortError: ..."}`. Callers should treat `status == 0` (or any + non-None `error`) as a transport failure rather than an HTTP + response. + + `body` may be a `str` (sent verbatim) or a `dict`/`list` (JSON- + encoded here). Pass headers explicitly when you need + `Content-Type: application/json` or an `Authorization` bearer. + """ + body_arg: str | None + if body is None: + body_arg = None + elif isinstance(body, (str, bytes)): + body_arg = body if isinstance(body, str) else body.decode("utf-8") + else: + body_arg = json.dumps(body) + js = """ + async ({url, method, headers, body, timeoutMs}) => { + const ctrl = new AbortController(); + const t = setTimeout(() => ctrl.abort(), timeoutMs); + try { + const opts = {method: method, headers: headers, signal: ctrl.signal}; + if (body !== null) opts.body = body; + const r = await fetch(url, opts); + clearTimeout(t); + let parsed; + try { + parsed = await r.json(); + } catch (_e) { + try { + parsed = await r.text(); + } catch (_e2) { + parsed = null; + } + } + return {status: r.status, body: parsed, error: null}; + } catch (e) { + clearTimeout(t); + return {status: 0, body: null, error: String(e)}; + } + } + """ + return page.evaluate( + js, + { + "url": url, + "method": method, + "headers": headers or {}, + "body": body_arg, + "timeoutMs": int(timeout_ms), + }, + ) + + +# ───────────────────────────────────────────────────────────────────── +# Wall-clock watchdog. +# ───────────────────────────────────────────────────────────────────── +# +# Even with every action and fetch bounded, a sufficiently strange +# wedge inside the browser (a CPU-pinned JS infinite loop, a renderer +# crash that doesn't propagate to Playwright, an asyncio deadlock in +# the sync wrapper) can still hang the script. The watchdog is a +# daemon Timer that calls `os._exit(2)` after `deadline_s` seconds, +# printing the wedge location to stderr so the CI log shows where the +# script was at force-kill time. The exit code matches "test failure +# by deadline" so the workflow's `set -e` propagates correctly. +# +# Pick `deadline_s` generously enough to cover the slowest healthy +# run -- macos-14 free runners with cold caches measure ~7-9 min for +# the comprehensive chat UI test. 12 minutes (720 s) leaves headroom +# without amplifying every real wedge to the 30-min runner-level cap. +def install_wall_clock_watchdog( + deadline_s: float, + *, + label: str = "playwright", + info: Callable[[str], None] | None = None, +) -> threading.Timer: + """Start a daemon Timer that hard-exits the process at `deadline_s`. + + Returns the Timer so the caller can `.cancel()` it on clean exit. + The Timer is daemonised; if the script exits normally before the + deadline the Timer dies with the process even without an explicit + cancel. + """ + + def _kaboom() -> None: + msg = ( + f"[{label}] WATCHDOG: hit {deadline_s:.0f}s wall-clock " + f"deadline; forcing exit(2). The script wedged somewhere " + f"the per-action timeouts could not bound. Inspect the " + f"most recent step printed above to localise." + ) + try: + sys.stderr.write(msg + "\n") + sys.stderr.flush() + except Exception: + pass + os._exit(2) + + timer = threading.Timer(deadline_s, _kaboom) + timer.daemon = True + timer.start() + if info is not None: + info(f"watchdog armed: hard-exit at {deadline_s:.0f}s") + return timer diff --git a/tests/studio/playwright_chat_ui.py b/tests/studio/playwright_chat_ui.py index 8f7dafa2a4..aa1d38c4e1 100644 --- a/tests/studio/playwright_chat_ui.py +++ b/tests/studio/playwright_chat_ui.py @@ -55,7 +55,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from _playwright_robust import ( # noqa: E402 chromium_launch_args, click_and_wait_for_response, + evaluate_fetch, install_view_transition_killer, + install_wall_clock_watchdog, is_benign_console_error, is_benign_page_error, recover_or_replace_page, @@ -85,6 +87,17 @@ STRICT = os.environ.get("STUDIO_UI_STRICT", "0") == "1" # CI bump this without hard-coding a Mac branch in the test. TURN_TIMEOUT_MS = int(os.environ.get("STUDIO_UI_TURN_TIMEOUT_MS", "180000")) +# Wall-clock cap for the entire script. A healthy comprehensive run is +# 5-9 min; 12 min leaves headroom. Tunable via STUDIO_UI_WALL_TIMEOUT_S. +# See _playwright_robust.install_wall_clock_watchdog for rationale. +WALL_TIMEOUT_S = float(os.environ.get("STUDIO_UI_WALL_TIMEOUT_S", "720")) + +# Per-fetch budget for in-page fetches. The /api/inference/load call is +# usually the slowest legitimate request: it pulls the model into the +# llama.cpp worker. Give it ~3 min on a cold cache, less elsewhere. +FETCH_TIMEOUT_MS = int(os.environ.get("STUDIO_UI_FETCH_TIMEOUT_MS", "30000")) +LOAD_FETCH_TIMEOUT_MS = int(os.environ.get("STUDIO_UI_LOAD_TIMEOUT_MS", "180000")) + _n = [0] @@ -132,6 +145,11 @@ def parse_rgb(s): with sync_playwright() as p: + _watchdog = install_wall_clock_watchdog( + WALL_TIMEOUT_S, + label = "ui", + info = info, + ) # Pre-flight: bash-side wait_for already gated on /api/health # before launching us, but the macos-14 free runner has been # observed to surface a 200 /api/health while the auth DB is @@ -424,18 +442,18 @@ with sync_playwright() as p: "() => localStorage.getItem('unsloth_auth_refresh_token')", ) if refresh_token: - refresh = page.evaluate( - f"""async (rt) => {{ - const r = await fetch("{BASE}/api/auth/refresh", {{ - method: "POST", - headers: {{"Content-Type": "application/json"}}, - body: JSON.stringify({{refresh_token: rt}}), - }}); - return await r.json(); - }}""", - refresh_token, + refresh_resp = evaluate_fetch( + page, + f"{BASE}/api/auth/refresh", + method = "POST", + headers = {"Content-Type": "application/json"}, + body = {"refresh_token": refresh_token}, + timeout_ms = FETCH_TIMEOUT_MS, ) - token = refresh.get("access_token") + if refresh_resp.get("error"): + fail(f"/api/auth/refresh wedged: {refresh_resp['error']!r}") + refresh = refresh_resp.get("body") or {} + token = (refresh or {}).get("access_token") if not token: fail("could not obtain auth token after change-password") @@ -450,15 +468,18 @@ with sync_playwright() as p: "EXPECTED_DEFAULT_MODEL", "unsloth/gemma-4-E2B-it-GGUF", ) - defaults = page.evaluate( - f"""async (token) => {{ - const r = await fetch("{BASE}/api/models/list", {{ - headers: {{ "Authorization": "Bearer " + token }}, - }}); - return await r.json(); - }}""", - token, + defaults_resp = evaluate_fetch( + page, + f"{BASE}/api/models/list", + headers = {"Authorization": f"Bearer {token}"}, + timeout_ms = FETCH_TIMEOUT_MS, ) + if defaults_resp.get("error") or defaults_resp.get("status") != 200: + fail( + f"/api/models/list failed: status={defaults_resp.get('status')!r} " + f"error={defaults_resp.get('error')!r}" + ) + defaults = defaults_resp["body"] or {} if not defaults.get("default_models"): fail(f"/api/models/list returned no default_models: {defaults}") if defaults["default_models"][0] != EXPECTED_DEFAULT: @@ -499,27 +520,35 @@ with sync_playwright() as p: # ───────────────────────────────────────────────────── step("load GGUF via /api/inference/load (uses session cookie)") # Token already fetched above; reuse it for the load call. - load_resp = page.evaluate(f"""async () => {{ - const r = await fetch("{BASE}/api/inference/load", {{ - method: "POST", - headers: {{ - "Authorization": "Bearer {token}", - "Content-Type": "application/json", - }}, - body: JSON.stringify({{ - model_path: "{GGUF_REPO}", - gguf_variant: "{GGUF_VARIANT}", - is_lora: false, - max_seq_length: 2048, - }}), - }}); - return {{status: r.status, body: await r.json()}}; - }}""") + # AbortSignal-bounded: the macos-14 --single-process Chromium had been + # observed wedging on this exact in-page fetch (run 25696797934 / job + # 75446949358) with zero further requests reaching the server. The + # 3-min budget is generous for a cold-cache GGUF load; on a wedge we + # surface a clean failure instead of a 30-min runner cancel. + load_resp = evaluate_fetch( + page, + f"{BASE}/api/inference/load", + method = "POST", + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + }, + body = { + "model_path": GGUF_REPO, + "gguf_variant": GGUF_VARIANT, + "is_lora": False, + "max_seq_length": 2048, + }, + timeout_ms = LOAD_FETCH_TIMEOUT_MS, + ) + if load_resp.get("error"): + fail(f"/api/inference/load wedged: {load_resp['error']!r}") if load_resp["status"] != 200: fail( - f"/api/inference/load returned {load_resp['status']}: {load_resp.get('body')!r}" + f"/api/inference/load returned {load_resp['status']}: " + f"{load_resp.get('body')!r}" ) - info(f"loaded model: {load_resp['body'].get('display_name')}") + info(f"loaded model: {(load_resp['body'] or {}).get('display_name')}") # Studio caches the per-context model state in zustand; reload # to make the chat composer pick up the loaded model. @@ -1185,10 +1214,13 @@ with sync_playwright() as p: # ───────────────────────────────────────────────────── # 14. /api/health stays healthy throughout. # ───────────────────────────────────────────────────── - health = page.evaluate(f"""async () => {{ - const r = await fetch("{BASE}/api/health"); - return {{status: r.status, body: await r.text()}}; - }}""") + health = evaluate_fetch( + page, + f"{BASE}/api/health", + timeout_ms = FETCH_TIMEOUT_MS, + ) + if health.get("error"): + fail(f"/api/health wedged: {health['error']!r}") if health["status"] != 200: fail(f"/api/health returned {health['status']}") @@ -1275,13 +1307,14 @@ with sync_playwright() as p: # The browser still has the pre-rotation access token. Refresh # tokens were revoked server-side by /change-password (auth.py), # so /api/auth/refresh from the browser context must now fail. - refresh_after = page.evaluate(f"""async () => {{ - const r = await fetch("{BASE}/api/auth/refresh", {{ - method: "POST", - credentials: "include", - }}); - return {{status: r.status}}; - }}""") + refresh_after = evaluate_fetch( + page, + f"{BASE}/api/auth/refresh", + method = "POST", + timeout_ms = FETCH_TIMEOUT_MS, + ) + if refresh_after.get("error"): + fail(f"/api/auth/refresh wedged: {refresh_after['error']!r}") if refresh_after["status"] == 200: fail(f"/api/auth/refresh should fail after CLI rotation; got 200") info( @@ -1392,4 +1425,5 @@ with sync_playwright() as p: ) info("PASS comprehensive UI flow") + _watchdog.cancel() browser.close() diff --git a/tests/studio/playwright_extra_ui.py b/tests/studio/playwright_extra_ui.py index 92025ed555..dccd2e423d 100644 --- a/tests/studio/playwright_extra_ui.py +++ b/tests/studio/playwright_extra_ui.py @@ -40,7 +40,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from _playwright_robust import ( # noqa: E402 chromium_launch_args, click_and_wait_for_response, + evaluate_fetch, install_view_transition_killer, + install_wall_clock_watchdog, is_benign_page_error, recover_or_replace_page, wait_for_health, @@ -59,6 +61,9 @@ STRICT = os.environ.get("STUDIO_UI_STRICT", "0") == "1" # turn timeout because gemma-3-270m CPU inference is 3-5x slower than # ubuntu-latest's. TURN_TIMEOUT_MS = int(os.environ.get("STUDIO_UI_TURN_TIMEOUT_MS", "180000")) +WALL_TIMEOUT_S = float(os.environ.get("STUDIO_UI_WALL_TIMEOUT_S", "720")) +FETCH_TIMEOUT_MS = int(os.environ.get("STUDIO_UI_FETCH_TIMEOUT_MS", "30000")) +LOAD_FETCH_TIMEOUT_MS = int(os.environ.get("STUDIO_UI_LOAD_TIMEOUT_MS", "180000")) _n = [0] _failed: list[str] = [] @@ -94,6 +99,11 @@ def runtime_warn(m: str) -> None: with sync_playwright() as p: + _watchdog = install_wall_clock_watchdog( + WALL_TIMEOUT_S, + label = "ui-extra", + info = info, + ) # Health pre-flight (best-effort). Same rationale as in # playwright_chat_ui.py: bash-side health wait can succeed before # the auth DB has finished migrating on macos-14 free runners. @@ -261,36 +271,44 @@ with sync_playwright() as p: if not token: fail("no access token after change-password") sys.exit(1) - load_resp = page.evaluate(f"""async () => {{ - const r = await fetch("{BASE}/api/inference/load", {{ - method: "POST", - headers: {{ - "Authorization": "Bearer {token}", - "Content-Type": "application/json", - }}, - body: JSON.stringify({{ - model_path: "{GGUF_REPO}", - gguf_variant: "{GGUF_VARIANT}", - is_lora: false, - max_seq_length: 2048, - }}), - }}); - return {{status: r.status, body: await r.json()}}; - }}""") + load_resp = evaluate_fetch( + page, + f"{BASE}/api/inference/load", + method = "POST", + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + }, + body = { + "model_path": GGUF_REPO, + "gguf_variant": GGUF_VARIANT, + "is_lora": False, + "max_seq_length": 2048, + }, + timeout_ms = LOAD_FETCH_TIMEOUT_MS, + ) + if load_resp.get("error"): + fail(f"/api/inference/load wedged: {load_resp['error']!r}") + sys.exit(1) if load_resp["status"] != 200: fail(f"/api/inference/load -> {load_resp['status']}: {load_resp.get('body')!r}") sys.exit(1) - info(f"loaded model: {load_resp['body'].get('display_name')}") + info(f"loaded model: {(load_resp['body'] or {}).get('display_name')}") page.reload() composer = page.locator('textarea[aria-label="Message input"]') composer.wait_for(state = "visible", timeout = 60_000) # Detect chat-only mode: /api/health.chat_only is the source of truth. # In chat-only mode, /studio + /export redirect to /chat. - health = page.evaluate(f"""async () => {{ - const r = await fetch("{BASE}/api/health"); - return await r.json(); - }}""") + health_resp = evaluate_fetch( + page, + f"{BASE}/api/health", + timeout_ms = FETCH_TIMEOUT_MS, + ) + if health_resp.get("error"): + fail(f"/api/health wedged: {health_resp['error']!r}") + sys.exit(1) + health = health_resp.get("body") or {} chat_only = bool(health.get("chat_only")) info(f"chat_only mode: {chat_only}") @@ -588,4 +606,5 @@ with sync_playwright() as p: info(f" - {m}") sys.exit(1) info("PASS extra UI flow") + _watchdog.cancel() browser.close() From a21e2d862d402671abfbfd6ff22c2801cc16ebd6 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 11 May 2026 22:42:29 -0700 Subject: [PATCH 06/25] chore: remove unused .semgrep/unsloth-rules.yml (#5395) The file's header claimed it was wired into security-audit.yml's Semgrep step, but that step only loads the four off-the-shelf packs (p/supply-chain, p/python, p/javascript, p/security-audit). The custom rules were never invoked by any upstream workflow, so the file is dead weight here. No CI changes needed; security-audit.yml is unaffected. --- .semgrep/unsloth-rules.yml | 183 ------------------------------------- 1 file changed, 183 deletions(-) delete mode 100644 .semgrep/unsloth-rules.yml diff --git a/.semgrep/unsloth-rules.yml b/.semgrep/unsloth-rules.yml deleted file mode 100644 index 654ff9a490..0000000000 --- a/.semgrep/unsloth-rules.yml +++ /dev/null @@ -1,183 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -# -# Custom Semgrep rules for unsloth + studio backend. The off-the-shelf -# rule packs (p/python, p/javascript, p/supply-chain, p/security-audit) -# wired into the security-audit workflow already cover the common -# patterns. These rules add catches for the *specific* shape of recent -# CVEs in the broader Python ML / dev-tools stack -- so if we ever -# introduce a similar bug ourselves, CI lights up. -# -# Run locally: -# pip install 'semgrep>=1.95' -# semgrep --config .semgrep/unsloth-rules.yml studio/backend unsloth scripts -# -# Wired into CI via .github/workflows/security-audit.yml's Semgrep step. - -rules: - # ───────────────────────────────────────────────────────────────── - # langchain-core CVE-2025-68664 shape: - # `dumps()` / `dumpd()` over a user-controlled dict that may carry - # the `lc` marker key -> deserialization injection on the round - # trip. Catch any json.dumps / pickle.dumps / yaml.dump on data - # that flowed through a Request/WebSocket payload. - # ───────────────────────────────────────────────────────────────── - - id: unsloth-deserialize-roundtrip - message: >- - Serializing user-controlled data with langchain-style `dumps` - can re-instantiate arbitrary classes when deserialized. See - langchain-core CVE-2025-68664. Sanitize / strip `lc` marker keys - before dumping, or use a strict schema (Pydantic) instead. - severity: WARNING - languages: [python] - patterns: - - pattern-either: - - pattern: langchain_core.load.dumps($DATA, ...) - - pattern: langchain_core.load.dumpd($DATA, ...) - - pattern: dumps($DATA) - - pattern: dumpd($DATA) - - metavariable-pattern: - metavariable: $DATA - patterns: - - pattern-either: - - pattern: request.$F - - pattern: payload - - pattern: body - - pattern: data - - pattern: input - - # ───────────────────────────────────────────────────────────────── - # n8n CVE-2025-68668 shape: - # `_pyodide._base.eval_code(...)` or any private/underscore call - # into pyodide internals that escapes the public sandbox API. - # ───────────────────────────────────────────────────────────────── - - id: unsloth-pyodide-private-eval - message: >- - Calling `_pyodide._base.eval_code` (or any `_pyodide.`) - bypasses the public Pyodide sandbox -- this is how n8n - CVE-2025-68668 (CVSS 9.9) escaped the Code Node's blocklist. - Use the documented sandbox API (`pyodide.runPython`) and rely - on web-worker isolation for untrusted input. - severity: ERROR - languages: [python, javascript, typescript] - patterns: - - pattern-either: - - pattern: _pyodide._base.eval_code(...) - - pattern: $X._pyodide.$Y(...) - - # ───────────────────────────────────────────────────────────────── - # marimo CVE-2026-39987 shape: - # FastAPI / Starlette WebSocket route that accepts connections - # without checking auth -- in marimo this dropped a PTY shell to - # any unauthenticated attacker. - # ───────────────────────────────────────────────────────────────── - - id: unsloth-websocket-no-auth - message: >- - WebSocket route accepts connections without an auth check. - marimo CVE-2026-39987 was a pre-auth WebSocket on - `/terminal/ws` that handed a full PTY shell to any - unauthenticated peer. Add a Depends(get_current_user) / - `await websocket.headers.get("authorization")` gate before - `await websocket.accept()`. - severity: WARNING - languages: [python] - patterns: - - pattern: | - @$APP.websocket("...") - async def $F(websocket: WebSocket, ...): - ... - await websocket.accept() - ... - - pattern-not-inside: | - @$APP.websocket("...") - async def $F(websocket: WebSocket, ..., $USER = Depends(...)): - ... - - pattern-not-inside: | - @$APP.websocket("...") - async def $F(websocket: WebSocket, ...): - ... - if not $AUTH: - ... - await websocket.accept() - - # ───────────────────────────────────────────────────────────────── - # litellm 1.82.7 shape: - # `subprocess.Popen` of a child Python interpreter that reads - # stdin from a network response (the C2-fetch-then-exec dropper - # pattern). Catches both `Popen([sys.executable, ...], stdin=...)` - # and `Popen("python ...", stdin=...)` variants. - # ───────────────────────────────────────────────────────────────── - - id: unsloth-popen-network-stdin - message: >- - Spawning a Python interpreter that reads its program from a - network call is the canonical fetch-and-exec dropper (litellm - 1.82.7 used this exact shape). Almost never legitimate inside a - package's import path. - severity: ERROR - languages: [python] - pattern-either: - - pattern: | - subprocess.Popen([..., $PY, ...], stdin=$NET, ...) - - pattern: | - subprocess.run([..., $PY, ...], input=$NET, ...) - - # ───────────────────────────────────────────────────────────────── - # Shai-Hulud / ForceMemo shape: - # programmatic write of a `.github/workflows/*.yml` file from - # inside our own Python source. We never write workflows - # programmatically; if a contributor ever does, they're probably - # re-implementing the worm pattern. - # ───────────────────────────────────────────────────────────────── - - id: unsloth-write-github-workflow - message: >- - Code that programmatically writes into `.github/workflows/` - from within unsloth itself is the Shai-Hulud / ForceMemo - self-propagation pattern. If you legitimately need a workflow - template, ship it under examples/ or templates/ instead. - severity: ERROR - languages: [python] - patterns: - - pattern-either: - - pattern: open("$P", ...) - - pattern: Path("$P").write_text(...) - - pattern: open("$P", "w", ...) - - metavariable-regex: - metavariable: $P - regex: \.github/workflows/.*\.ya?ml - - # ───────────────────────────────────────────────────────────────── - # Pickle-from-network shape: classic deserialization sink that - # several recent ML pipeline CVEs hit (mlflow, pyzmq, ray serve). - # ───────────────────────────────────────────────────────────────── - - id: unsloth-pickle-from-network - message: >- - `pickle.loads` on bytes that flowed from a network response is - arbitrary code execution. Use `safetensors` or a strict - schema (Pydantic / msgspec) instead. ML frameworks have shipped - multiple CVEs of this exact shape (mlflow, ray serve, pyzmq). - severity: ERROR - languages: [python] - pattern-either: - - pattern: pickle.loads($X.content) - - pattern: pickle.loads($X.text.encode(...)) - - pattern: pickle.loads(requests.get(...).content) - - pattern: pickle.load(urllib.request.urlopen(...)) - - # ───────────────────────────────────────────────────────────────── - # Subprocess shell=True with f-string / format / concat -- command - # injection if any interpolated value comes from user input. - # ───────────────────────────────────────────────────────────────── - - id: unsloth-shell-true-interpolation - message: >- - `subprocess` call with `shell=True` and an interpolated command - string is command injection if any input is user-controlled. - Pass argv list instead, or use shlex.quote on each part. - severity: WARNING - languages: [python] - pattern-either: - - pattern: subprocess.run(f"...", shell=True, ...) - - pattern: subprocess.Popen(f"...", shell=True, ...) - - pattern: subprocess.call(f"...", shell=True, ...) - - pattern: os.system(f"...") - - pattern: subprocess.run("..." + $X, shell=True, ...) - - pattern: subprocess.run("...{}...".format(...), shell=True, ...) From 8ca0455be4ef2272e70348c62e8b760a232c4518 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 12 May 2026 05:47:41 -0700 Subject: [PATCH 07/25] studio/ci: sweep actions/cache v5 hardening across sibling smoke workflows (#5399) * studio/ci: sweep actions/cache@v5 hardening across sibling smoke workflows Follow-up to PR 5396, which fixed the same flake in studio-windows-inference-smoke.yml. actions/cache@v5 has a recurring mode where it logs `Cache hit for: ` and then exits non-zero without extracting the archive (see actions/cache#1621 and github community discussion #163260). 12 cache blocks across 8 sibling Studio smoke workflows remained on the vulnerable one-step pattern and would abort before priming HF_HOME / installing Studio on the same flake. Apply the same restore + save split mechanically to every block: - actions/cache/restore@ with continue-on-error: true - Prime/Download gate widened to also fire on outcome != 'success' so the silent-restore-failure path re-downloads - actions/cache/save@ with continue-on-error: true, gated on the Prime/Download outcome so we only write a fresh entry when we actually rebuilt the directory Same SHA-pinned action, same cache keys (character-identical), same paths. Existing cache entries keep matching. Only behavior change is that a transient restore-side or save-side failure now falls through to a re-download instead of failing the job. Files touched (12 cache blocks total): studio-api-smoke.yml (1 block) studio-mac-api-smoke.yml (1 block) studio-mac-ui-smoke.yml (1 block) studio-ui-smoke.yml (1 block) studio-windows-api-smoke.yml (1 block) studio-windows-ui-smoke.yml (1 block) studio-inference-smoke.yml (3 blocks: HF, GGUF flat, HF+mmproj) studio-mac-inference-smoke.yml (3 blocks: HF, GGUF flat, HF+mmproj) Verification: all 12 single-step actions/cache@ uses removed, replaced by 12 restore@ + 12 save@; every file parses as valid YAML. * studio/ci: drop continue-on-error from cache/save steps Reverting the save-side continue-on-error addition. Defensive masking of save failures was correct in principle but loses signal: - cache/save@v5.0.5 already swallows ReserveCacheError (the most common save flake) as a non-fatal core.info, so the mask was rarely doing anything today. - A real save-side failure (sustained cache backend outage, blob server 5xx storm) is something we want to see, not hide. Without the signal we would see slow CI for days without knowing the cache layer is broken. - If save flakes start showing up in practice we add this back with concrete evidence. The restore-side continue-on-error stays -- that is the actual fix for actions/cache#1621 silent-restore-failures and removing it would re-introduce the bug. --- .github/workflows/studio-api-smoke.yml | 15 +++++-- .github/workflows/studio-inference-smoke.yml | 45 +++++++++++++++---- .github/workflows/studio-mac-api-smoke.yml | 15 +++++-- .../workflows/studio-mac-inference-smoke.yml | 45 +++++++++++++++---- .github/workflows/studio-mac-ui-smoke.yml | 15 +++++-- .github/workflows/studio-ui-smoke.yml | 15 +++++-- .../workflows/studio-windows-api-smoke.yml | 15 +++++-- .github/workflows/studio-windows-ui-smoke.yml | 15 +++++-- 8 files changed, 144 insertions(+), 36 deletions(-) diff --git a/.github/workflows/studio-api-smoke.yml b/.github/workflows/studio-api-smoke.yml index 742cba9ed9..668111d1dc 100644 --- a/.github/workflows/studio-api-smoke.yml +++ b/.github/workflows/studio-api-smoke.yml @@ -69,9 +69,10 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + - name: Restore HF_HOME for ${{ env.GGUF_REPO }} id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache # Same key as studio-ui-smoke.yml so the two jobs share a @@ -79,7 +80,8 @@ jobs: key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 - name: Prime HF_HOME with the GGUF - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -88,6 +90,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" + - name: Save HF_HOME for ${{ env.GGUF_REPO }} + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + - name: Install Studio (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/studio-inference-smoke.yml b/.github/workflows/studio-inference-smoke.yml index 19e1ab9cc5..922d883cc9 100644 --- a/.github/workflows/studio-inference-smoke.yml +++ b/.github/workflows/studio-inference-smoke.yml @@ -85,15 +85,17 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + - name: Restore HF_HOME for ${{ env.GGUF_REPO }} id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 - name: Prime HF_HOME with the GGUF - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -102,6 +104,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" + - name: Save HF_HOME for ${{ env.GGUF_REPO }} + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + - name: Install Studio (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -324,15 +333,17 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Cache GGUF model file + - name: Restore GGUF model file id: cache-gguf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: gguf-cache key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 - name: Download GGUF if cache miss - if: steps.cache-gguf.outputs.cache-hit != 'true' + id: download-gguf + if: steps.cache-gguf.outputs.cache-hit != 'true' || steps.cache-gguf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -341,6 +352,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" --local-dir gguf-cache + - name: Save GGUF model file + if: always() && steps.download-gguf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: gguf-cache + key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 + - name: Install Studio (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -632,15 +650,17 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} (model + mmproj) + - name: Restore HF_HOME for ${{ env.GGUF_REPO }} (model + mmproj) id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-${{ env.MMPROJ_FILE }}-v1 - name: Prime HF_HOME with the GGUF + mmproj - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -651,6 +671,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$MMPROJ_FILE" + - name: Save HF_HOME for ${{ env.GGUF_REPO }} (model + mmproj) + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-${{ env.MMPROJ_FILE }}-v1 + - name: Install Studio (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/studio-mac-api-smoke.yml b/.github/workflows/studio-mac-api-smoke.yml index 98596f374a..6a98776fa3 100644 --- a/.github/workflows/studio-mac-api-smoke.yml +++ b/.github/workflows/studio-mac-api-smoke.yml @@ -56,15 +56,17 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + - name: Restore HF_HOME for ${{ env.GGUF_REPO }} id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 - name: Prime HF_HOME with the GGUF - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -73,6 +75,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" + - name: Save HF_HOME for ${{ env.GGUF_REPO }} + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + - name: Install Studio (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/studio-mac-inference-smoke.yml b/.github/workflows/studio-mac-inference-smoke.yml index 97efe3e74d..82438f0c27 100644 --- a/.github/workflows/studio-mac-inference-smoke.yml +++ b/.github/workflows/studio-mac-inference-smoke.yml @@ -79,15 +79,17 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + - name: Restore HF_HOME for ${{ env.GGUF_REPO }} id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 - name: Prime HF_HOME with the GGUF - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -96,6 +98,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" + - name: Save HF_HOME for ${{ env.GGUF_REPO }} + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + - name: Install Studio (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -318,15 +327,17 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Cache GGUF model file + - name: Restore GGUF model file id: cache-gguf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: gguf-cache key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 - name: Download GGUF if cache miss - if: steps.cache-gguf.outputs.cache-hit != 'true' + id: download-gguf + if: steps.cache-gguf.outputs.cache-hit != 'true' || steps.cache-gguf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -335,6 +346,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" --local-dir gguf-cache + - name: Save GGUF model file + if: always() && steps.download-gguf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: gguf-cache + key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 + - name: Install Studio (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -674,15 +692,17 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} (model + mmproj) + - name: Restore HF_HOME for ${{ env.GGUF_REPO }} (model + mmproj) id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-${{ env.MMPROJ_FILE }}-v1 - name: Prime HF_HOME with the GGUF + mmproj - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' # Authenticated + parallel: shared macos-14 NAT egress stalls # multi-GB anonymous downloads. env: @@ -702,6 +722,13 @@ jobs: find hf-cache -name "$GGUF_FILE" -o -name "$MMPROJ_FILE" \ | xargs -I{} ls -lhL {} + - name: Save HF_HOME for ${{ env.GGUF_REPO }} (model + mmproj) + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-${{ env.MMPROJ_FILE }}-v1 + - name: Install Studio (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/studio-mac-ui-smoke.yml b/.github/workflows/studio-mac-ui-smoke.yml index c921ddf63e..df3654277f 100644 --- a/.github/workflows/studio-mac-ui-smoke.yml +++ b/.github/workflows/studio-mac-ui-smoke.yml @@ -56,15 +56,17 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + - name: Restore HF_HOME for ${{ env.GGUF_REPO }} id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 - name: Prime HF_HOME with the GGUF - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -73,6 +75,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" + - name: Save HF_HOME for ${{ env.GGUF_REPO }} + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + - name: Install Studio (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/studio-ui-smoke.yml b/.github/workflows/studio-ui-smoke.yml index 756eea64b2..82496c3665 100644 --- a/.github/workflows/studio-ui-smoke.yml +++ b/.github/workflows/studio-ui-smoke.yml @@ -70,15 +70,17 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + - name: Restore HF_HOME for ${{ env.GGUF_REPO }} id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 - name: Prime HF_HOME with the GGUF - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -87,6 +89,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" + - name: Save HF_HOME for ${{ env.GGUF_REPO }} + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + - name: Install Studio (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/studio-windows-api-smoke.yml b/.github/workflows/studio-windows-api-smoke.yml index d9ed5d5594..fd80377352 100644 --- a/.github/workflows/studio-windows-api-smoke.yml +++ b/.github/workflows/studio-windows-api-smoke.yml @@ -63,15 +63,17 @@ jobs: with: python-version: '3.12' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + - name: Restore HF_HOME for ${{ env.GGUF_REPO }} id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 - name: Prime HF_HOME with the GGUF - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -80,6 +82,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" + - name: Save HF_HOME for ${{ env.GGUF_REPO }} + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + - name: Pre-install Windows tweaks (npm 11 + Defender exclusions) shell: pwsh # See studio-windows-update-smoke.yml for the full rationale. diff --git a/.github/workflows/studio-windows-ui-smoke.yml b/.github/workflows/studio-windows-ui-smoke.yml index a5a4753ba5..6ee262163b 100644 --- a/.github/workflows/studio-windows-ui-smoke.yml +++ b/.github/workflows/studio-windows-ui-smoke.yml @@ -72,15 +72,17 @@ jobs: # then fatal-errors with "Cache folder path is retrieved # for pip but doesn't exist on disk". - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + - name: Restore HF_HOME for ${{ env.GGUF_REPO }} id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 - name: Prime HF_HOME with the GGUF - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -89,6 +91,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" + - name: Save HF_HOME for ${{ env.GGUF_REPO }} + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + - name: Pre-install Windows tweaks (npm 11 + Defender exclusions) shell: pwsh # See studio-windows-update-smoke.yml for the full rationale. From 040b80a60e6bf0278c0b81ee986cf9b87b9d4f6d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 12 May 2026 05:47:44 -0700 Subject: [PATCH 08/25] studio/ci: harden HF_HOME cache against actions/cache v5 silent restore failures (#5396) * studio/ci: harden HF_HOME/GGUF cache against actions/cache@v5 silent restore failures actions/cache@v5 has a recurring flake where it logs "Cache hit for: " and then exits non-zero in well under a second without actually extracting the archive (see actions/cache#1621 and github community discussion #163260). When that happens to the JSON, images job the cache step is marked failure, all downstream steps are skipped (only the if: always() ones run), and the job never even tries to install Studio. Example: run 25713577488 / job 75498714730 took 23 s total and bailed at the cache step despite the cache having been written successfully ~30 min earlier. Replace the single-step actions/cache usage in all three jobs with the documented restore + save split: - actions/cache/restore with continue-on-error: true on the way in - Prime/Download step gated on cache-hit != 'true' OR outcome != 'success' so the silent-failure path re-downloads from HF instead of skipping - actions/cache/save on the way out, gated on the Prime step's outcome so we only write a fresh entry when we actually rebuilt the directory Same SHA-pinned action (v5.0.5), same cache keys, same paths -- so existing cache entries keep matching. Only behavior change is that a transient restore-side failure now falls through to a re-download instead of failing the job. * studio/ci: add continue-on-error to the new actions/cache/save steps Per review of PR 5396: a save-side flake (upload timeout, 5xx from the cache backend, future-fatal ReserveCacheError) is strictly recoverable because next run just re-downloads, so it should never fail the job. Today actions/cache/save@v5.0.5 already swallows ReserveCacheError as a non-fatal warning, so this is defense in depth. Aligns the save steps with their matching restore steps which already mask transient failures via continue-on-error. * studio/ci: drop continue-on-error from cache/save steps Reverting the save-side continue-on-error addition from the previous commit. cache/save@v5.0.5 already swallows ReserveCacheError (the most common save flake) as a non-fatal core.info, so the mask was rarely doing anything in practice. A real save-side failure (cache backend outage, blob server 5xx storm) is signal we want to keep -- without it we would see slow CI for days without knowing the cache layer is broken. If save flakes start showing up in practice we add this back with concrete evidence. The restore-side continue-on-error stays -- that is the actual fix for the actions/cache#1621 silent-restore-failure mode. Also strip the now-stale "continue-on-error" comments above the three save blocks. * studio/ci: clarify cache split header comment Per re-review: the prior wording "the Save step re-uploads on the way out" implied actions/cache/save would replace a broken existing cache entry, which is wrong -- cache keys are immutable, so save logs a warning when the key already exists and the corrupted entry stays until the -v1 suffix is bumped. Rewrite to spell out the actual behavior and the escape hatch (bump the suffix). --- .../studio-windows-inference-smoke.yml | 69 ++++++++++++++++--- 1 file changed, 60 insertions(+), 9 deletions(-) diff --git a/.github/workflows/studio-windows-inference-smoke.yml b/.github/workflows/studio-windows-inference-smoke.yml index b33b6f9563..13bd8e58e2 100644 --- a/.github/workflows/studio-windows-inference-smoke.yml +++ b/.github/workflows/studio-windows-inference-smoke.yml @@ -73,15 +73,29 @@ jobs: with: python-version: '3.12' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + # Split restore + save (rather than the one-step actions/cache) so a + # transient restore-side failure does not kill the whole job. v5 has a + # known flake where it logs "Cache hit for: " and then exits + # non-zero without actually extracting the archive (see + # actions/cache#1621 and github community discussion #163260). + # continue-on-error on restore masks that failure so the Prime step + # below can re-download from HF and the job keeps running. Save then + # populates the cache key on a real miss only; cache keys are + # immutable, so a corrupted cached entry persists until the -v1 + # suffix below is bumped. + - name: Restore HF_HOME cache for ${{ env.GGUF_REPO }} id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 - name: Prime HF_HOME with the GGUF - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + # Run on a real cache miss AND on the silent-restore-failure mode + # described above (outcome != success). + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -90,6 +104,16 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" + - name: Save HF_HOME cache for ${{ env.GGUF_REPO }} + # Only write a fresh cache entry when we actually rebuilt the + # directory (Prime ran and succeeded). Skipping when Prime is + # skipped avoids "already exists" save warnings on the happy path. + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + - name: Pre-install Windows tweaks (npm 11 + Defender exclusions) shell: pwsh # See studio-windows-update-smoke.yml for the full rationale. @@ -375,15 +399,20 @@ jobs: with: python-version: '3.12' - - name: Cache GGUF model file + # Split restore + save so a transient restore-side failure does not + # kill the whole job. See the matching block in the tool-calling job + # above for the full rationale (actions/cache#1621). + - name: Restore GGUF model cache id: cache-gguf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: gguf-cache key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 - name: Download GGUF if cache miss - if: steps.cache-gguf.outputs.cache-hit != 'true' + id: download-gguf + if: steps.cache-gguf.outputs.cache-hit != 'true' || steps.cache-gguf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -392,6 +421,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" --local-dir gguf-cache + - name: Save GGUF model cache + if: always() && steps.download-gguf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: gguf-cache + key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 + - name: Pre-install Windows tweaks (npm 11 + Defender exclusions) shell: pwsh # See studio-windows-update-smoke.yml for the full rationale. @@ -771,15 +807,23 @@ jobs: with: python-version: '3.12' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} (model + mmproj) + # Split restore + save so a transient restore-side failure does not + # kill the whole job. See the matching block in the tool-calling job + # for the full rationale (actions/cache#1621). This is the block that + # actually broke in run 25713577488: "Cache hit for: " was + # logged, the step exited non-zero in ~0.3 s without extracting the + # 3.4 GiB archive, and steps 6-15 were skipped. + - name: Restore HF_HOME cache for ${{ env.GGUF_REPO }} (model + mmproj) id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-${{ env.MMPROJ_FILE }}-v1 - name: Prime HF_HOME with the GGUF + mmproj - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -790,6 +834,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$MMPROJ_FILE" + - name: Save HF_HOME cache for ${{ env.GGUF_REPO }} (model + mmproj) + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-${{ env.MMPROJ_FILE }}-v1 + - name: Pre-install Windows tweaks (npm 11 + Defender exclusions) shell: pwsh # See studio-windows-update-smoke.yml for the full rationale. From 0a54d001ec0f6d65cc766480c76588cd8370f5a0 Mon Sep 17 00:00:00 2001 From: Wasim Yousef Said Date: Wed, 13 May 2026 05:30:20 +0200 Subject: [PATCH 09/25] Harden Tauri release flow (#5341) * Harden Tauri backend preflight and startup Require managed Studio root IDs to match before attaching to existing backends, close the concurrent backend-start window, and tighten frontend Tauri detection to Tauri-specific signals. * Add Tauri backend manageability guards Gate desktop backend compatibility on explicit manageability fields, add external-conflict handling for unsafe backend states, and protect update/repair paths from mutating active non-owned Studio backends. Track Tauri-owned backends with local owner metadata for verified orphan cleanup only. * Split Tauri preflight probes into modules Move preflight types, version checks, managed install probing, and backend probing into focused submodules while preserving behavior and keeping implementation files under the release-readiness size target. * Use desktop-specific Tauri updater channel Point the desktop updater at a same-repo desktop-latest manifest and publish that channel from non-draft desktop releases after validating the Tauri-generated latest.json. * Add Linux desktop update policy * Add owned backend lifecycle guards * Adopt verified desktop-owned backends * Validate desktop backend readiness * Trim Tauri release hardening code * Require desktop backend 2026.5.3 * Handle desktop backend edge cases * Fail stalled desktop backend startup * Fix desktop update edge cases * Avoid secret-gating adopted watchdog * Fix desktop update comparison guards * Automate desktop release versioning * Serialize desktop release workflow * tests: follow preflight.rs split into preflight/{backend,managed,types,version}.rs PR #5341 splits studio/src-tauri/src/preflight.rs into a directory of submodules. The cmd.env_remove("UNSLOTH_STUDIO_HOME") + STUDIO_HOME calls now live in preflight/managed.rs instead of preflight.rs, so test_tauri_preflight_scrubs_studio_home_env counted zero matches in the old single-file location and failed with "assert 0 >= 2". Read whichever shape is on disk: preflight.rs at the old path plus every *.rs under preflight/ (current PR has 2 occurrences in preflight/managed.rs). The guard intent is unchanged: at least 2 env_remove calls covering run_cli_probe and probe_cli_capability, plus the single commands.rs scrub in check_install_status. Verified locally: pytest tests/test_studio_install_workspace_guard.py::test_tauri_preflight_scrubs_studio_home_env passes. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Avoid browser Tauri hostname detection * Restore shutdown flag after failed stop --------- Co-authored-by: Daniel Han Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/release-desktop.yml | 693 +++++++++- studio/backend/main.py | 22 + studio/backend/run.py | 75 +- studio/frontend/src/app/provider.tsx | 17 +- .../src/components/tauri/update-banner.tsx | 45 +- .../frontend/src/hooks/use-tauri-backend.ts | 100 +- studio/frontend/src/hooks/use-tauri-update.ts | 132 +- studio/frontend/src/lib/api-base.ts | 14 +- studio/src-tauri/linux/postremove.sh | 2 +- studio/src-tauri/src/commands.rs | 395 ++++-- studio/src-tauri/src/desktop_auth.rs | 163 +-- studio/src-tauri/src/desktop_backend_owner.rs | 1018 +++++++++++++++ studio/src-tauri/src/desktop_update_policy.rs | 420 ++++++ studio/src-tauri/src/diagnostics/mod.rs | 16 +- studio/src-tauri/src/diagnostics/report.rs | 2 + studio/src-tauri/src/diagnostics/state.rs | 24 + studio/src-tauri/src/main.rs | 4 + studio/src-tauri/src/preflight.rs | 1122 ++++++++--------- studio/src-tauri/src/preflight/backend.rs | 336 +++++ studio/src-tauri/src/preflight/managed.rs | 169 +++ studio/src-tauri/src/preflight/types.rs | 44 + studio/src-tauri/src/preflight/version.rs | 163 +++ studio/src-tauri/src/process.rs | 1018 ++++++++++++--- studio/src-tauri/tauri.conf.json | 5 +- tests/test_studio_install_workspace_guard.py | 22 +- unsloth_cli/commands/studio.py | 2 + 26 files changed, 4917 insertions(+), 1106 deletions(-) create mode 100644 studio/src-tauri/src/desktop_backend_owner.rs create mode 100644 studio/src-tauri/src/desktop_update_policy.rs create mode 100644 studio/src-tauri/src/preflight/backend.rs create mode 100644 studio/src-tauri/src/preflight/managed.rs create mode 100644 studio/src-tauri/src/preflight/types.rs create mode 100644 studio/src-tauri/src/preflight/version.rs diff --git a/.github/workflows/release-desktop.yml b/.github/workflows/release-desktop.yml index ea82739968..13afacc2a8 100644 --- a/.github/workflows/release-desktop.yml +++ b/.github/workflows/release-desktop.yml @@ -3,15 +3,295 @@ name: Release Desktop App on: workflow_dispatch: inputs: + studio_version: + description: 'Studio version tag to release (for example, v0.1.39-beta)' + type: string + required: true + pypi_version: + description: 'Exact PyPI unsloth version just published/stamped (for example, 2026.5.3); leave blank to use MIN_DESKTOP_BACKEND_VERSION' + type: string + required: false draft: - description: 'Create as draft release' + description: 'Create as draft release; draft runs do not advance desktop-latest updater channel' type: boolean default: true permissions: contents: write +concurrency: + group: release-desktop-${{ github.repository }} + cancel-in-progress: false + jobs: + prepare-version: + name: Prepare release versions + runs-on: ubuntu-latest + outputs: + studio_version: ${{ steps.prepare.outputs.studio_version }} + app_version: ${{ steps.prepare.outputs.app_version }} + desktop_release_tag: ${{ steps.prepare.outputs.desktop_release_tag }} + prerelease: ${{ steps.prepare.outputs.prerelease }} + pypi_version: ${{ steps.prepare.outputs.pypi_version }} + + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + + - name: Validate release versions + id: prepare + shell: bash + env: + INPUT_STUDIO_VERSION: ${{ inputs.studio_version }} + INPUT_PYPI_VERSION: ${{ inputs.pypi_version }} + run: | + python3 <<'PY' + import os + import pathlib + import re + import sys + + studio_version = os.environ['INPUT_STUDIO_VERSION'].strip() + if not studio_version: + sys.exit('studio_version is required, for example v0.1.39-beta') + if re.fullmatch(r'v?20\d{2}\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?', studio_version): + sys.exit(f'studio_version must be a Studio SemVer tag, not a date-style backend version: {studio_version}') + + semver_tag = re.compile( + r'^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)' + r'(?:-[0-9A-Za-z.][0-9A-Za-z.-]*)?$' + ) + if not semver_tag.fullmatch(studio_version): + sys.exit(f'studio_version must be a SemVer tag with leading v, for example v0.1.39-beta: {studio_version}') + + app_version = studio_version.removeprefix('v') + desktop_release_tag = f'desktop-v{app_version}' + prerelease = 'true' if '-' in app_version.split('+', 1)[0] else 'false' + + def parse_backend_version(version): + match = re.fullmatch( + r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)' + r'(?:([a-zA-Z]|\.dev|dev|\.rc|rc|\.post|post)(\d*))?' + r'(?:[-+]([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?', + version, + ) + if not match: + return None + major, minor, patch, suffix_name, suffix_number, suffix_text = match.groups() + if suffix_name: + normalized = suffix_name.lower().lstrip('.') + order = {'dev': 0, 'a': 1, 'b': 2, 'rc': 3, 'post': 5}.get(normalized) + if order is None: + return None + number = int(suffix_number or '0') + elif suffix_text: + order = 3 if version[version.find(suffix_text) - 1] == '-' else 4 + number = 0 + else: + order = 4 + number = 0 + return (int(major), int(minor), int(patch), order, number) + + preflight = pathlib.Path('studio/src-tauri/src/preflight/version.rs').read_text() + match = re.search(r'MIN_DESKTOP_BACKEND_VERSION:\s*&str\s*=\s*"([^"]+)"', preflight) + if not match: + sys.exit('Could not read MIN_DESKTOP_BACKEND_VERSION') + min_backend_version = match.group(1) + + input_pypi_version = os.environ.get('INPUT_PYPI_VERSION', '').strip() + parsed_min_backend = parse_backend_version(min_backend_version) + if parsed_min_backend is None: + sys.exit(f'MIN_DESKTOP_BACKEND_VERSION is not a supported backend package version: {min_backend_version}') + + pypi_version = input_pypi_version or min_backend_version + parsed_pypi = parse_backend_version(pypi_version) + if parsed_pypi is None: + sys.exit(f'pypi_version is not a supported backend package version: {pypi_version}') + if parsed_pypi < parsed_min_backend: + sys.exit( + f'pypi_version {pypi_version} is lower than desktop minimum ' + f'MIN_DESKTOP_BACKEND_VERSION {min_backend_version}' + ) + + if input_pypi_version: + print( + 'Using exact PyPI unsloth version from pypi_version input: ' + f'{pypi_version} (desktop minimum: {min_backend_version})' + ) + else: + print( + 'Using exact PyPI unsloth version from MIN_DESKTOP_BACKEND_VERSION: ' + f'{pypi_version}' + ) + + with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as output: + print(f'studio_version={studio_version}', file=output) + print(f'app_version={app_version}', file=output) + print(f'desktop_release_tag={desktop_release_tag}', file=output) + print(f'prerelease={prerelease}', file=output) + print(f'pypi_version={pypi_version}', file=output) + PY + + - name: Verify PyPI package and Studio stamp + shell: bash + env: + STUDIO_VERSION: ${{ steps.prepare.outputs.studio_version }} + PYPI_VERSION: ${{ steps.prepare.outputs.pypi_version }} + run: | + set -euo pipefail + python3 <<'PY' + import json + import os + import pathlib + import sys + import time + import urllib.error + import urllib.request + + pypi_version = os.environ['PYPI_VERSION'] + dist_dir = pathlib.Path(os.environ['RUNNER_TEMP'], 'pypi-unsloth-dist') + dist_dir.mkdir(parents=True, exist_ok=True) + metadata_url = f'https://pypi.org/pypi/unsloth/{pypi_version}/json' + + last_error = None + for attempt in range(1, 6): + try: + with urllib.request.urlopen(metadata_url, timeout=30) as response: + metadata = json.load(response) + break + except Exception as exc: + last_error = exc + if attempt < 5: + time.sleep(10 * attempt) + else: + sys.exit(f'Publish unsloth=={pypi_version} to PyPI before the desktop release ({last_error})') + + files = metadata.get('urls') or [] + if not files: + sys.exit(f'PyPI returned no distribution files for unsloth=={pypi_version}') + + for file_info in files: + filename = file_info.get('filename') + url = file_info.get('url') + if not filename or '/' in filename or not url: + sys.exit(f'Unexpected PyPI file entry for unsloth=={pypi_version}: {file_info!r}') + target = dist_dir / filename + for attempt in range(1, 4): + try: + with urllib.request.urlopen(url, timeout=60) as response: + target.write_bytes(response.read()) + break + except Exception as exc: + last_error = exc + if attempt < 3: + time.sleep(5 * attempt) + else: + sys.exit(f'Could not download {filename} from PyPI ({last_error})') + PY + + if [ -f scripts/stamp_studio_release.py ]; then + mapfile -t dists < <(find "$RUNNER_TEMP/pypi-unsloth-dist" -type f \( -name '*.whl' -o -name '*.tar.gz' \) | sort) + if [ "${#dists[@]}" -eq 0 ]; then + echo "No PyPI wheel/sdist artifacts downloaded for unsloth==$PYPI_VERSION" >&2 + exit 1 + fi + python3 scripts/stamp_studio_release.py --verify-dist "$RUNNER_TEMP/pypi-unsloth-dist" --expected "$STUDIO_VERSION" + else + echo "scripts/stamp_studio_release.py not found; release-desktop requires #5308 to verify the PyPI Studio stamp." >&2 + exit 1 + fi + + - name: Guard public updater channel version + if: ${{ !inputs.draft }} + shell: bash + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} + APP_VERSION: ${{ steps.prepare.outputs.app_version }} + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/desktop-current" + if ! gh release download desktop-latest --pattern latest.json --dir "$RUNNER_TEMP/desktop-current" --clobber 2>/dev/null; then + echo "No existing desktop-latest latest.json found; allowing first channel publish." + exit 0 + fi + python3 <<'PY' + import json + import os + import pathlib + import re + import sys + + def parse(value: str): + value = value.removeprefix('v') + match = re.fullmatch( + r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)' + r'(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?' + r'(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?', + value, + ) + if not match: + sys.exit(f'desktop-latest latest.json has invalid version: {value}') + major, minor, patch, prerelease = match.groups() + return (int(major), int(minor), int(patch), prerelease) + + def numeric_tail(identifier: str) -> tuple[str, int] | None: + match = re.fullmatch(r'([A-Za-z-]+)(\d+)', identifier) + if not match: + return None + return (match.group(1).lower(), int(match.group(2))) + + def compare_identifier(left: str, right: str) -> int: + left_num = left.isdigit() + right_num = right.isdigit() + if left_num and right_num: + return (int(left) > int(right)) - (int(left) < int(right)) + if left_num: + return -1 + if right_num: + return 1 + + left_tail = numeric_tail(left) + right_tail = numeric_tail(right) + if left_tail and right_tail and left_tail[0] == right_tail[0]: + return (left_tail[1] > right_tail[1]) - (left_tail[1] < right_tail[1]) + + return (left > right) - (left < right) + + def compare_prerelease(left: str | None, right: str | None) -> int: + if left == right: + return 0 + if left is None: + return 1 + if right is None: + return -1 + left_parts = left.split('.') + right_parts = right.split('.') + for left_part, right_part in zip(left_parts, right_parts): + order = compare_identifier(left_part, right_part) + if order: + return order + return (len(left_parts) > len(right_parts)) - (len(left_parts) < len(right_parts)) + + def compare(left: str, right: str) -> int: + left_major, left_minor, left_patch, left_pre = parse(left) + right_major, right_minor, right_patch, right_pre = parse(right) + left_core = (left_major, left_minor, left_patch) + right_core = (right_major, right_minor, right_patch) + if left_core != right_core: + return (left_core > right_core) - (left_core < right_core) + return compare_prerelease(left_pre, right_pre) + + current_path = pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-current', 'latest.json') + current = json.loads(current_path.read_text()).get('version') + next_version = os.environ['APP_VERSION'] + if not isinstance(current, str): + sys.exit('desktop-latest latest.json has missing version') + if compare(next_version, current) < 0: + sys.exit( + f'Refusing to publish {next_version}; desktop-latest currently points at newer version {current}.' + ) + PY + build: strategy: fail-fast: false @@ -32,11 +312,15 @@ jobs: label: Windows (x64) name: Build ${{ matrix.label }} + needs: prepare-version runs-on: ${{ matrix.platform }} env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true - + APP_VERSION: ${{ needs.prepare-version.outputs.app_version }} + STUDIO_VERSION: ${{ needs.prepare-version.outputs.studio_version }} + DESKTOP_RELEASE_TAG: ${{ needs.prepare-version.outputs.desktop_release_tag }} + DESKTOP_PRERELEASE: ${{ needs.prepare-version.outputs.prerelease }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 @@ -67,36 +351,144 @@ jobs: exit 1 fi - - name: Install frontend dependencies - working-directory: studio/frontend - run: npm install - - - name: Verify backend package is published + - name: Verify desktop updater and Linux package config shell: bash run: | node <<'JS' const { readFileSync } = require('node:fs'); - (async () => { - const cargo = readFileSync('studio/src-tauri/Cargo.toml', 'utf8'); - const match = cargo.match(/^version\s*=\s*"([^"]+)"/m); - if (!match) throw new Error('Could not read desktop app version'); + const expected = 'https://github.com/unslothai/unsloth/releases/download/desktop-latest/latest.json'; + const config = JSON.parse(readFileSync('studio/src-tauri/tauri.conf.json', 'utf8')); + const endpoints = config.plugins?.updater?.endpoints; + if (!Array.isArray(endpoints) || endpoints.length !== 1) { + throw new Error('Expected exactly one desktop updater endpoint'); + } + if (endpoints[0] !== expected) { + throw new Error('Desktop updater endpoint must be ' + expected + ', got ' + endpoints[0]); + } + if (endpoints.some((endpoint) => endpoint.includes('/releases/latest/'))) { + throw new Error('Desktop updater endpoint must not use repo-wide /releases/latest/'); + } - const appVersion = match[1]; - const response = await fetch(`https://pypi.org/pypi/unsloth/${appVersion}/json`); - if (!response.ok) { - const message = 'Publish unsloth=={app_version} to PyPI before the desktop release'; - throw new Error(`${message.replace('{app_version}', appVersion)} (HTTP ${response.status})`); + const targets = config.bundle?.targets; + if (Array.isArray(targets) && targets.some((target) => String(target).toLowerCase() === 'rpm')) { + throw new Error('Desktop release must not target RPM packages'); + } + if (config.bundle?.linux?.rpm) { + throw new Error('bundle.linux.rpm must not be configured'); + } + + const workflow = readFileSync('.github/workflows/release-desktop.yml', 'utf8'); + const lines = workflow.split(/\r?\n/); + const releaseBodies = []; + for (let i = 0; i < lines.length; i += 1) { + const match = lines[i].match(/^(\s*)releaseBody:\s*\|\s*$/); + if (!match) continue; + const baseIndent = match[1].length; + const bodyLines = []; + i += 1; + for (; i < lines.length; i += 1) { + const line = lines[i]; + if (line.trim() === '') { + bodyLines.push(''); + continue; + } + const indent = line.match(/^\s*/)[0].length; + if (indent <= baseIndent) { + i -= 1; + break; + } + bodyLines.push(line.slice(baseIndent + 2)); } - })(); + releaseBodies.push(bodyLines.join('\n')); + } + if (releaseBodies.length === 0) { + throw new Error('Expected at least one desktop release body'); + } + for (const body of releaseBodies) { + if (/\brpm\b|\.rpm/i.test(body)) { + throw new Error('Desktop release body must not advertise RPM packages'); + } + } JS + - name: Install frontend dependencies + working-directory: studio/frontend + run: npm install + # ── Rust ── - name: Install Rust stable uses: dtolnay/rust-toolchain@stable with: targets: ${{ matrix.platform == 'macos-latest' && 'aarch64-apple-darwin,x86_64-apple-darwin' || '' }} + - name: Patch desktop app version + shell: bash + working-directory: studio/src-tauri + run: | + set -euo pipefail + if command -v python3 >/dev/null 2>&1; then + PYTHON=python3 + else + PYTHON=python + fi + "$PYTHON" <<'PY' + import os + import pathlib + import re + import sys + + app_version = os.environ['APP_VERSION'] + if not app_version: + sys.exit('APP_VERSION is required') + + cargo_toml = pathlib.Path('Cargo.toml') + lines = cargo_toml.read_text().splitlines(keepends=True) + in_package = False + patched = False + for index, line in enumerate(lines): + stripped = line.strip() + if stripped == '[package]': + in_package = True + continue + if stripped.startswith('[') and stripped.endswith(']'): + in_package = False + if in_package and re.fullmatch(r'version\s*=\s*"[^"]+"\s*', stripped): + lines[index] = f'version = "{app_version}"\n' + patched = True + break + if not patched: + sys.exit('Could not patch [package] version in Cargo.toml') + cargo_toml.write_text(''.join(lines)) + + cargo_lock = pathlib.Path('Cargo.lock') + lock_text = cargo_lock.read_text() + lock_text, count = re.subn( + r'(?m)(^\[\[package\]\]\nname = "unsloth-studio"\nversion = ")[^"]+(")', + lambda match: f'{match.group(1)}{app_version}{match.group(2)}', + lock_text, + ) + if count != 1: + sys.exit(f'Could not patch unsloth-studio version in Cargo.lock (matches={count})') + cargo_lock.write_text(lock_text) + PY + + cargo metadata --locked --no-deps --format-version 1 > "$RUNNER_TEMP/cargo-metadata.json" + "$PYTHON" <<'PY' + import json + import os + import pathlib + import sys + + app_version = os.environ['APP_VERSION'] + metadata = json.loads(pathlib.Path(os.environ['RUNNER_TEMP'], 'cargo-metadata.json').read_text()) + versions = [package['version'] for package in metadata.get('packages', []) if package.get('name') == 'unsloth-studio'] + if versions != [app_version]: + sys.exit(f'cargo metadata unsloth-studio version mismatch: expected {app_version}, got {versions}') + PY + + git diff -- Cargo.toml Cargo.lock + - name: Rust cache uses: swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae with: @@ -146,8 +538,8 @@ jobs: with: projectPath: studio tauriScript: npx --prefix . tauri - tagName: desktop-v__VERSION__ - releaseName: 'Unsloth Studio (Desktop) v__VERSION__' + tagName: ${{ needs.prepare-version.outputs.desktop_release_tag }} + releaseName: 'Unsloth Studio (Desktop) ${{ needs.prepare-version.outputs.studio_version }}' releaseBody: | Desktop app for Unsloth Studio. @@ -159,7 +551,7 @@ jobs: > Linux AppImage on Ubuntu 24.04+ may require: `sudo apt install libfuse2t64` > First-run system dependency elevation is supported on Ubuntu/Debian. Other Linux distributions should install system packages manually. releaseDraft: ${{ inputs.draft }} - prerelease: false + prerelease: ${{ needs.prepare-version.outputs.prerelease }} args: -v ${{ matrix.args }} # ── macOS: build + sign + notarize + upload ── @@ -177,8 +569,8 @@ jobs: with: projectPath: studio tauriScript: npx --prefix . tauri - tagName: desktop-v__VERSION__ - releaseName: 'Unsloth Studio (Desktop) v__VERSION__' + tagName: ${{ needs.prepare-version.outputs.desktop_release_tag }} + releaseName: 'Unsloth Studio (Desktop) ${{ needs.prepare-version.outputs.studio_version }}' releaseBody: | Desktop app for Unsloth Studio. @@ -190,7 +582,7 @@ jobs: > Linux AppImage on Ubuntu 24.04+ may require: `sudo apt install libfuse2t64` > First-run system dependency elevation is supported on Ubuntu/Debian. Other Linux distributions should install system packages manually. releaseDraft: ${{ inputs.draft }} - prerelease: false + prerelease: ${{ needs.prepare-version.outputs.prerelease }} args: -v ${{ matrix.args }} # ── Windows: build + sign + upload ── @@ -209,8 +601,8 @@ jobs: with: projectPath: studio tauriScript: npx --prefix . tauri - tagName: desktop-v__VERSION__ - releaseName: 'Unsloth Studio (Desktop) v__VERSION__' + tagName: ${{ needs.prepare-version.outputs.desktop_release_tag }} + releaseName: 'Unsloth Studio (Desktop) ${{ needs.prepare-version.outputs.studio_version }}' releaseBody: | Desktop app for Unsloth Studio. @@ -222,5 +614,254 @@ jobs: > Linux AppImage on Ubuntu 24.04+ may require: `sudo apt install libfuse2t64` > First-run system dependency elevation is supported on Ubuntu/Debian. Other Linux distributions should install system packages manually. releaseDraft: ${{ inputs.draft }} - prerelease: false + prerelease: ${{ needs.prepare-version.outputs.prerelease }} args: -v ${{ matrix.args }} + + # Release process note: only non-draft workflow runs advance the public + # desktop-latest updater channel. Draft builds are for private review; if a + # draft is manually published later, this channel intentionally remains + # unchanged until a narrow manual channel-publish flow is added or a public + # desktop release is created by running this workflow with draft=false. + publish-updater-channel: + name: Publish desktop updater channel + needs: [prepare-version, build] + if: ${{ !inputs.draft }} + runs-on: ubuntu-latest + permissions: + contents: write + env: + GH_REPO: ${{ github.repository }} + APP_VERSION: ${{ needs.prepare-version.outputs.app_version }} + STUDIO_VERSION: ${{ needs.prepare-version.outputs.studio_version }} + DESKTOP_RELEASE_TAG: ${{ needs.prepare-version.outputs.desktop_release_tag }} + DESKTOP_PRERELEASE: ${{ needs.prepare-version.outputs.prerelease }} + + steps: + - name: Download versioned updater metadata + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/desktop-updater" + gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${DESKTOP_RELEASE_TAG}" > "$RUNNER_TEMP/source-release.json" + python3 <<'PY' + import json + import os + import pathlib + import sys + + source = json.loads(pathlib.Path(os.environ['RUNNER_TEMP'], 'source-release.json').read_text()) + expected_tag = os.environ['DESKTOP_RELEASE_TAG'] + if source.get('tag_name') != expected_tag: + sys.exit(f'Expected source release {expected_tag}, got {source.get("tag_name")}') + if source.get('draft'): + sys.exit(f'Source desktop release {expected_tag} is draft; refusing to publish public updater channel') + PY + gh release download "$DESKTOP_RELEASE_TAG" --pattern latest.json --dir "$RUNNER_TEMP/desktop-updater" --clobber + test -s "$RUNNER_TEMP/desktop-updater/latest.json" + + - name: Validate versioned updater metadata + shell: bash + run: | + python3 <<'PY' + import json + import os + import pathlib + import re + import sys + + app_version = os.environ['APP_VERSION'] + release_tag = os.environ['DESKTOP_RELEASE_TAG'] + latest_path = pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-updater', 'latest.json') + data = json.loads(latest_path.read_text()) + if not isinstance(data, dict): + sys.exit('latest.json must be a JSON object') + + version = data.get('version') + if not isinstance(version, str) or not version: + sys.exit('latest.json missing version') + if not re.fullmatch(r'v?\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?', version): + sys.exit(f'latest.json version is not SemVer-like: {version}') + if version.removeprefix('v') != app_version: + sys.exit(f'latest.json version {version} does not match desktop app version {app_version}') + + platforms = data.get('platforms') + if not isinstance(platforms, dict) or not platforms: + sys.exit('latest.json missing platforms') + + required_families = { + 'darwin-aarch64': False, + 'linux-x86_64': False, + 'windows-x86_64': False, + } + expected_prefix = f'https://github.com/unslothai/unsloth/releases/download/{release_tag}/' + forbidden_fragments = ('/releases/latest/', '/releases/download/desktop-latest/') + + for platform, entry in platforms.items(): + if not isinstance(entry, dict): + sys.exit(f'Platform {platform} must be an object') + url = entry.get('url') + signature = entry.get('signature') + if not isinstance(url, str) or not url.strip(): + sys.exit(f'Platform {platform} missing url') + if not isinstance(signature, str) or not signature.strip(): + sys.exit(f'Platform {platform} missing signature') + if any(fragment in url for fragment in forbidden_fragments): + sys.exit(f'Platform {platform} points at a moving updater channel: {url}') + if not url.startswith(expected_prefix): + sys.exit(f'Platform {platform} URL must point at {release_tag}: {url}') + for family in required_families: + if platform == family or platform.startswith(family + '-'): + required_families[family] = True + + missing = [family for family, found in required_families.items() if not found] + if missing: + sys.exit('latest.json missing required platform families: ' + ', '.join(missing)) + PY + + - name: Ensure desktop updater channel release + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + channel_json="$RUNNER_TEMP/desktop-latest-release.json" + if ! gh api "repos/${GITHUB_REPOSITORY}/releases/tags/desktop-latest" > "$channel_json" 2>/dev/null; then + gh release create desktop-latest \ + --title "Unsloth Studio Desktop updater channel" \ + --notes "Machine-managed desktop updater channel; latest.json is replaced by release-desktop.yml." \ + --prerelease \ + --latest=false \ + --target "$GITHUB_SHA" + gh api "repos/${GITHUB_REPOSITORY}/releases/tags/desktop-latest" > "$channel_json" + fi + + python3 <<'PY' + import json + import os + import pathlib + import sys + + channel = json.loads(pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-latest-release.json').read_text()) + if channel.get('draft'): + sys.exit('desktop-latest release is draft; refusing to publish updater channel') + if channel.get('immutable'): + sys.exit('desktop-latest release is immutable; cannot replace latest.json') + if not channel.get('prerelease'): + sys.exit('desktop-latest release must be a prerelease so it cannot compete with repo-wide latest') + PY + + - name: Prevent updater channel downgrade + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/desktop-current" + if ! gh release download desktop-latest --pattern latest.json --dir "$RUNNER_TEMP/desktop-current" --clobber 2>/dev/null; then + echo "No existing desktop-latest latest.json found; allowing first channel publish." + exit 0 + fi + python3 <<'PY' + import json + import os + import pathlib + import re + import sys + + def parse(value: str): + value = value.removeprefix('v') + match = re.fullmatch( + r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)' + r'(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?' + r'(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?', + value, + ) + if not match: + sys.exit(f'desktop-latest latest.json has invalid version: {value}') + major, minor, patch, prerelease = match.groups() + return (int(major), int(minor), int(patch), prerelease) + + def numeric_tail(identifier: str) -> tuple[str, int] | None: + match = re.fullmatch(r'([A-Za-z-]+)(\d+)', identifier) + if not match: + return None + return (match.group(1).lower(), int(match.group(2))) + + def compare_identifier(left: str, right: str) -> int: + left_num = left.isdigit() + right_num = right.isdigit() + if left_num and right_num: + return (int(left) > int(right)) - (int(left) < int(right)) + if left_num: + return -1 + if right_num: + return 1 + + left_tail = numeric_tail(left) + right_tail = numeric_tail(right) + if left_tail and right_tail and left_tail[0] == right_tail[0]: + return (left_tail[1] > right_tail[1]) - (left_tail[1] < right_tail[1]) + + return (left > right) - (left < right) + + def compare_prerelease(left: str | None, right: str | None) -> int: + if left == right: + return 0 + if left is None: + return 1 + if right is None: + return -1 + left_parts = left.split('.') + right_parts = right.split('.') + for left_part, right_part in zip(left_parts, right_parts): + order = compare_identifier(left_part, right_part) + if order: + return order + return (len(left_parts) > len(right_parts)) - (len(left_parts) < len(right_parts)) + + def compare(left: str, right: str) -> int: + left_major, left_minor, left_patch, left_pre = parse(left) + right_major, right_minor, right_patch, right_pre = parse(right) + left_core = (left_major, left_minor, left_patch) + right_core = (right_major, right_minor, right_patch) + if left_core != right_core: + return (left_core > right_core) - (left_core < right_core) + return compare_prerelease(left_pre, right_pre) + + current_path = pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-current', 'latest.json') + next_path = pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-updater', 'latest.json') + current = json.loads(current_path.read_text()).get('version') + next_version = json.loads(next_path.read_text()).get('version') + if not isinstance(current, str) or not isinstance(next_version, str): + sys.exit('Could not compare desktop-latest channel versions') + if compare(next_version, current) < 0: + sys.exit( + f'Refusing to move desktop-latest from {current} to older version {next_version}.' + ) + PY + + - name: Publish desktop updater channel metadata + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + gh release upload desktop-latest "$RUNNER_TEMP/desktop-updater/latest.json" --clobber + gh api "repos/${GITHUB_REPOSITORY}/releases/tags/desktop-latest" > "$RUNNER_TEMP/desktop-latest-release.json" + python3 <<'PY' + import json + import os + import pathlib + import sys + + channel = json.loads(pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-latest-release.json').read_text()) + assets = [asset for asset in channel.get('assets', []) if asset.get('name') == 'latest.json'] + if len(assets) != 1: + sys.exit(f'Expected exactly one desktop-latest latest.json asset, found {len(assets)}') + expected_url = f'https://github.com/{os.environ["GITHUB_REPOSITORY"]}/releases/download/desktop-latest/latest.json' + actual_url = assets[0].get('browser_download_url') + if actual_url != expected_url: + sys.exit(f'desktop-latest latest.json URL mismatch: expected {expected_url}, got {actual_url}') + PY diff --git a/studio/backend/main.py b/studio/backend/main.py index 650a488212..2fba2756e7 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -42,6 +42,7 @@ if _STUDIO_ROOT_RESOLVED != _LEGACY_STUDIO_ROOT: if not os.environ.get("UNSLOTH_LLAMA_CPP_PATH"): os.environ["UNSLOTH_LLAMA_CPP_PATH"] = str(_STUDIO_ROOT_RESOLVED / "llama.cpp") +import hashlib import mimetypes import re as _re import shutil @@ -163,6 +164,24 @@ UNSLOTH_VERSION = get_unsloth_version() STUDIO_VERSION = get_studio_version() +def _load_desktop_owner() -> dict[str, str] | None: + token = os.environ.pop("UNSLOTH_STUDIO_DESKTOP_OWNER_TOKEN", "") + kind = os.environ.pop("UNSLOTH_STUDIO_DESKTOP_OWNER_KIND", "") + if kind != "tauri" or not token: + return None + return { + "kind": "tauri", + "token_sha256": hashlib.sha256(token.encode("utf-8")).hexdigest(), + } + + +_DESKTOP_OWNER = _load_desktop_owner() + + +def _desktop_owner() -> dict[str, str] | None: + return _DESKTOP_OWNER + + @asynccontextmanager async def lifespan(app: FastAPI): """Startup: detect hardware, seed default admin if needed. Shutdown: clean up compiled cache.""" @@ -306,12 +325,15 @@ async def health_check(): "device_type": device_type, "chat_only": _hw_module.CHAT_ONLY, "desktop_protocol_version": 1, + "desktop_manageability_version": 1, "supports_desktop_auth": True, + "supports_desktop_backend_ownership": True, # why: launchers compare against an install-time hash so a sibling # Studio on the same port is rejected; hex digest avoids leaking the # raw install path on -H 0.0.0.0. "studio_root_id": _studio_root_id(), "native_path_leases_supported": native_path_leases_supported(), + **({"desktop_owner": owner} if (owner := _desktop_owner()) else {}), } diff --git a/studio/backend/run.py b/studio/backend/run.py index 1dd1230a17..dfd4b7453e 100644 --- a/studio/backend/run.py +++ b/studio/backend/run.py @@ -307,7 +307,6 @@ def run_server( import asyncio from threading import Thread, Event - import time import uvicorn from main import app, setup_frontend @@ -336,10 +335,6 @@ def run_server( print("=" * 50) print("") - # Output port for Tauri to parse when in api-only mode - if api_only: - print(f"TAURI_PORT={port}", flush = True) - # Setup frontend if path provided (skip in api-only mode) if frontend_path and not api_only: if setup_frontend(app, frontend_path): @@ -349,11 +344,21 @@ def run_server( if not silent: print(f"[WARNING] Frontend not found at {frontend_path}") + ready_event = Event() + startup_failed = Event() + startup_errors = [] + + class _ReadyServer(uvicorn.Server): + async def startup(self, *args, **kwargs): + await super().startup(*args, **kwargs) + if getattr(self, "started", False) and not self.should_exit: + ready_event.set() + # Create the uvicorn server and expose it for signal handlers config = uvicorn.Config( app, host = host, port = port, log_level = "info", access_log = False ) - _server = uvicorn.Server(config) + _server = _ReadyServer(config) _shutdown_event = Event() # Expose the actual bound port so request-handling code can build @@ -365,21 +370,8 @@ def run_server( app.state.server_port = port if port and port > 0 else None app.state.llama_parallel_slots = llama_parallel_slots - # Run server in a daemon thread - def _run(): - asyncio.run(_server.serve()) - - thread = Thread(target = _run, daemon = True) - thread.start() - time.sleep(3) - - _write_pid_file() - import atexit - - atexit.register(_remove_pid_file) - - # Expose a shutdown callable via app.state so the /api/shutdown endpoint - # can trigger graceful shutdown without circular imports. + # Expose a shutdown callable via app.state before the server can accept + # requests so /api/shutdown is available as soon as readiness is published. def _trigger_shutdown(): _graceful_shutdown(_server) if _shutdown_event is not None: @@ -387,6 +379,47 @@ def run_server( app.state.trigger_shutdown = _trigger_shutdown + # Run server in a daemon thread + def _run(): + try: + asyncio.run(_server.serve()) + except BaseException as exc: + startup_errors.append(exc) + startup_failed.set() + finally: + if not ready_event.is_set(): + startup_failed.set() + + thread = Thread(target = _run, daemon = True) + thread.start() + + # Wait until uvicorn has completed lifespan startup and bound sockets, or + # until the server exits/fails before startup. This intentionally has no + # correctness deadline: a slow but live startup should remain in progress. + try: + while not ready_event.is_set(): + if startup_failed.is_set() or not thread.is_alive(): + if startup_errors: + raise RuntimeError( + "Uvicorn server failed before startup completed" + ) from startup_errors[0] + raise RuntimeError("Uvicorn server exited before startup completed") + ready_event.wait(timeout = 0.1) + except KeyboardInterrupt: + _graceful_shutdown(_server) + _shutdown_event.set() + raise + + _write_pid_file() + import atexit + + atexit.register(_remove_pid_file) + + # Output port for Tauri to parse when in api-only mode. Emit only after + # uvicorn sockets are bound and FastAPI lifespan/startup has completed. + if api_only: + print(f"TAURI_PORT={port}", flush = True) + if not silent: display_host = _resolve_external_ip() if host == "0.0.0.0" else host print_studio_access_banner( diff --git a/studio/frontend/src/app/provider.tsx b/studio/frontend/src/app/provider.tsx index fe3173aa41..8360186d1e 100644 --- a/studio/frontend/src/app/provider.tsx +++ b/studio/frontend/src/app/provider.tsx @@ -23,10 +23,6 @@ interface AppProviderProps { children: ReactNode; } -// --------------------------------------------------------------------------- -// Tauri window helpers (only imported in Tauri mode) -// --------------------------------------------------------------------------- - type TauriWindowMode = "setup" | "app"; type WindowLayoutGuard = () => boolean; @@ -53,19 +49,15 @@ async function applyAppWindowLayout(isCurrent: WindowLayoutGuard): Promise let finalH = 600; if (monitor) { - // Convert physical pixels to logical using scale factor const scale = monitor.scaleFactor; const screenW = monitor.size.width / scale; const screenH = monitor.size.height / scale; - // Target: 75% of screen width, golden ratio height, capped at min 900x600 finalW = Math.max(900, Math.round(screenW * 0.75)); const targetH = Math.max(600, Math.round(finalW / 1.618)); - // Don't exceed screen height finalH = Math.min(targetH, Math.round(screenH * 0.85)); } - // Apply constraints and finalize without animating through intermediate sizes if (!isCurrent()) return; await win.setSize(new LogicalSize(finalW, finalH)); if (!isCurrent()) return; @@ -108,10 +100,6 @@ function getTauriWindowMode( } } -// --------------------------------------------------------------------------- -// TauriWrapper -// --------------------------------------------------------------------------- - function TauriUpdateLayer({ isExternalServer }: { isExternalServer: boolean }) { const update = useTauriUpdate(isExternalServer); const isUpdating = @@ -141,6 +129,8 @@ function TauriUpdateLayer({ isExternalServer }: { isExternalServer: boolean }) { dismissed={update.dismissed} lastFailure={update.lastFailure} isExternalServer={isExternalServer} + updatePolicyMode={update.updatePolicyMode} + manualReleaseUrl={update.manualReleaseUrl} onInstall={update.installUpdate} onDismiss={update.dismiss} onCopyDiagnostics={update.copyDiagnostics} @@ -184,8 +174,7 @@ function TauriWrapper({ children }: { children: ReactNode }) { }; }, []); - // Keep the Tauri window hidden during preflight, then show it centered in setup - // mode or apply the final app layout in one instant step. + // Keep the Tauri window hidden until setup or app layout is ready. useEffect(() => { if (!isTauri) return; diff --git a/studio/frontend/src/components/tauri/update-banner.tsx b/studio/frontend/src/components/tauri/update-banner.tsx index 79fbae6bb7..62038f92a9 100644 --- a/studio/frontend/src/components/tauri/update-banner.tsx +++ b/studio/frontend/src/components/tauri/update-banner.tsx @@ -2,7 +2,12 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import { Button } from "@/components/ui/button"; -import type { RetainedUpdateFailure, UpdateInfo, UpdateStatus } from "@/hooks/use-tauri-update"; +import type { + DesktopUpdatePolicyMode, + RetainedUpdateFailure, + UpdateInfo, + UpdateStatus, +} from "@/hooks/use-tauri-update"; import type { CopySupportDiagnosticsResult } from "@/lib/tauri-diagnostics"; import { AnimatePresence, motion } from "motion/react"; import { useState } from "react"; @@ -13,6 +18,8 @@ interface UpdateBannerProps { dismissed: boolean; lastFailure: RetainedUpdateFailure | null; isExternalServer?: boolean; + updatePolicyMode: DesktopUpdatePolicyMode; + manualReleaseUrl: string | null; onInstall: () => void; onDismiss: () => void; onCopyDiagnostics: () => Promise; @@ -26,6 +33,8 @@ export function UpdateBanner({ dismissed, lastFailure, isExternalServer = false, + updatePolicyMode, + manualReleaseUrl, onInstall, onDismiss, onCopyDiagnostics, @@ -36,6 +45,10 @@ export function UpdateBanner({ const showFailure = Boolean(lastFailure) && !dismissed; const showAvailable = status === "available" && !dismissed && !showFailure; const show = showFailure || (showAvailable && Boolean(info)); + const isManualLinuxPackage = updatePolicyMode === "manual_linux_package"; + const installDisabled = isManualLinuxPackage + ? manualReleaseUrl === null + : isExternalServer; async function handleCopyDiagnostics() { setCopying(true); @@ -67,18 +80,16 @@ export function UpdateBanner({ className="fixed top-4 right-4 z-[9999] w-[380px]" >
- {/* Close button */} - {/* Header */}
đŸ¦¥
@@ -88,37 +99,39 @@ export function UpdateBanner({

{showFailure ? "Backend recovered. Diagnostics are still available." - : isExternalServer - ? "Run `unsloth studio update` from your terminal" - : "A new app update is available"} + : isManualLinuxPackage + ? "Open the GitHub release page to install the Linux package" + : isExternalServer + ? "Run `unsloth studio update` from your terminal" + : "A new app update is available"}

- {/* Retained failure */} {showFailure && lastFailure && (

{lastFailure.error}

)} - {/* Actions */}
{showFailure ? ( <> - - ) : ( <> - - @@ -132,7 +145,7 @@ export function UpdateBanner({ )} {manualReport && (