diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 42fb3a1373..2802f461b6 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -53,3 +53,10 @@ /studio/backend/tests/ @rolandtannous @danielhanchen /tests/ @rolandtannous @danielhanchen /scripts/ @rolandtannous @danielhanchen + +# Snapshot data for the notebook linter / Colab oracle. Drift in these +# files changes the pin floor for every Unsloth notebook, so refreshes +# must be reviewed by the notebook owners directly. CODEOWNERS later +# wins, so this overrides the broader /scripts/ rule above. +/scripts/data/colab_*.txt @danielhanchen @shimmyshimmer +/scripts/data/colab_*.json @danielhanchen @shimmyshimmer diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 51f9e724d9..490838285e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,6 +5,12 @@ updates: directory: "/" schedule: interval: "weekly" + cooldown: + # github-actions refs are git tags / SHAs, not semver -- the + # `semver-minor-days` / `semver-patch-days` knobs are rejected + # by Dependabot's validator for this ecosystem. Only the + # `default-days` floor applies. + default-days: 7 groups: actions: patterns: ["*"] @@ -12,21 +18,24 @@ updates: applies-to: security-updates patterns: ["*"] - - package-ecosystem: "bun" - directory: "/studio/frontend" - schedule: - interval: "weekly" - groups: - bun-frontend: - patterns: ["*"] - bun-frontend-security: - applies-to: security-updates - patterns: ["*"] + # Removed a stray `package-ecosystem: "bun"` entry for + # /studio/frontend: that path has no bun.lock / bun.lockb, so + # Dependabot's bun ecosystem silently no-ops on it. The actual + # lockfile committed at /studio/frontend is package-lock.json + # (npm), and the npm entry further below already catches + # npm_and_yarn security advisories for that directory. Version + # updates for /studio/frontend stay suppressed (open-pull- + # requests-limit: 0 in that entry) -- security PRs flow through + # regardless. Add a real bun entry IF and WHEN bun.lock lands. - package-ecosystem: "npm" directory: "/studio/backend/core/data_recipe/oxc-validator" schedule: interval: "weekly" + cooldown: + default-days: 7 + semver-minor-days: 3 + semver-patch-days: 3 groups: npm-oxc-validator: patterns: ["*"] @@ -41,6 +50,8 @@ updates: schedule: interval: "weekly" open-pull-requests-limit: 5 + cooldown: + default-days: 7 groups: python: patterns: ["*"] @@ -52,6 +63,10 @@ updates: directory: "/studio/src-tauri" schedule: interval: "weekly" + cooldown: + default-days: 7 + semver-minor-days: 3 + semver-patch-days: 3 groups: cargo-tauri: patterns: ["*"] @@ -59,15 +74,25 @@ updates: applies-to: security-updates patterns: ["*"] - # bun owns version updates for /studio/frontend (above); GitHub - # fires npm-package advisories under npm_and_yarn, so this entry - # catches and groups them. limit: 0 suppresses version-update - # PRs, security updates flow through regardless. + # /studio/frontend npm dependencies. Version-update PRs are + # deliberately suppressed (open-pull-requests-limit: 0) -- the + # frontend dep tree is large, the lockfile is the authoritative + # pin, and `min-release-age=7` in studio/frontend/.npmrc already + # blocks fresh tarballs at install time. Security advisories + # arrive via GitHub's npm_and_yarn channel and are NOT capped by + # `open-pull-requests-limit` per Dependabot's documented + # behaviour; they flow through this entry, group together, and + # still respect the cooldown below so we never ingest a tarball + # that was hot-published less than 3 days ago. - package-ecosystem: "npm" directory: "/studio/frontend" schedule: interval: "weekly" open-pull-requests-limit: 0 + cooldown: + default-days: 7 + semver-minor-days: 3 + semver-patch-days: 3 groups: npm-frontend-security: applies-to: security-updates diff --git a/.github/workflows/consolidated-tests-ci.yml b/.github/workflows/consolidated-tests-ci.yml index 4ad3d9f16a..abceb91567 100644 --- a/.github/workflows/consolidated-tests-ci.yml +++ b/.github/workflows/consolidated-tests-ci.yml @@ -121,6 +121,8 @@ jobs: UNSLOTH_IS_PRESENT: '1' steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: @@ -234,9 +236,23 @@ jobs: # tests/conftest.py spoof which handles that. run: | set -euxo pipefail - git clone --depth=1 --branch="$UNSLOTH_ZOO_REF" \ - https://github.com/unslothai/unsloth-zoo \ - "$RUNNER_TEMP/unsloth-zoo" + # github.com occasionally 500s on the git fetch; retry so a + # single upstream blip does not fail CI. + for attempt in 1 2 3; do + rm -rf "$RUNNER_TEMP/unsloth-zoo" + if git clone --depth=1 --branch="$UNSLOTH_ZOO_REF" \ + https://github.com/unslothai/unsloth-zoo \ + "$RUNNER_TEMP/unsloth-zoo"; then + break + fi + if [ "$attempt" -eq 3 ]; then + echo "::error::git clone unsloth-zoo failed after 3 attempts" + exit 1 + fi + delay=$((5 * attempt)) + echo "::warning::clone failed (attempt $attempt/3), retrying in ${delay}s..." + sleep "$delay" + done pip install -e "$RUNNER_TEMP/unsloth-zoo" --no-deps pip show unsloth_zoo @@ -255,6 +271,39 @@ jobs: tests/utils/test_trunc_normal_patch.py python -m pytest --collect-only -q "$RUNNER_TEMP/unsloth-zoo/tests/" + - name: import_fixes drift detectors (18 tests, HARD GATE) + # One drift detector per fix_* / patch_* function in + # unsloth/import_fixes.py. The detectors assert the *healthy* + # upstream shape that the fix expects ABSENT the regression; + # ANY DRIFT DETECTED -> pytest.fail (NEVER skip) so the + # matrix cell goes red and the maintainer triages on the + # next PR, not in a downstream user's crash report. + # + # Pathologies covered by the suite (each maps to one fix + # function with the line range cited in the test docstring): + # * protobuf MessageFactory GetPrototype / GetMessageClass + # * datasets 4.4.x recursion range + # * TRL tuple-vs-bool _*_available caching + # * transformers PreTrainedModel.enable_input_require_grads + # source pattern flip + # * transformers torchcodec / causal_conv1d availability + # flags + # * transformers + accelerate is_wandb_available + # * peft.utils.transformers_weight_conversion importability + # + build_peft_weight_mapping signature + # * triton 3.6+ CompiledKernel num_ctas / cluster_dims + # * torch / torchvision pinned compatibility table + # * vllm guided_decoding_params / structured_outputs + + # aimv2 ovis config version + # * huggingface_hub is_offline_mode / HF_HUB_OFFLINE + # * torch.nn.init.trunc_normal_ presence (patch site for + # patch_trunc_normal_precision_issue) + # * xformers post-num_splits-key fix version + # HARD GATE: a red cell here is a real upstream regression + # without a corresponding zoo / unsloth-side workaround. + run: | + python -m pytest -v --tb=short tests/test_import_fixes_drift.py + - name: unsloth Bucket-A — CPU tests not in Repo tests (CPU) # 16 tests across 5 files. They live inside tests/saving/ and # tests/utils/, both of which Repo tests (CPU) excludes via --ignore @@ -1999,6 +2048,8 @@ jobs: UNSLOTH_IS_PRESENT: '1' steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: @@ -2040,9 +2091,23 @@ jobs: # main-branch fixes flow into the smoke without a release). run: | set -euxo pipefail - git clone --depth=1 --branch="$UNSLOTH_ZOO_REF" \ - https://github.com/unslothai/unsloth-zoo \ - "$RUNNER_TEMP/unsloth-zoo" + # github.com occasionally 500s on the git fetch; retry so a + # single upstream blip does not fail CI. + for attempt in 1 2 3; do + rm -rf "$RUNNER_TEMP/unsloth-zoo" + if git clone --depth=1 --branch="$UNSLOTH_ZOO_REF" \ + https://github.com/unslothai/unsloth-zoo \ + "$RUNNER_TEMP/unsloth-zoo"; then + break + fi + if [ "$attempt" -eq 3 ]; then + echo "::error::git clone unsloth-zoo failed after 3 attempts" + exit 1 + fi + delay=$((5 * attempt)) + echo "::warning::clone failed (attempt $attempt/3), retrying in ${delay}s..." + sleep "$delay" + done pip install -e "$RUNNER_TEMP/unsloth-zoo" --no-deps pip show unsloth_zoo diff --git a/.github/workflows/lint-ci.yml b/.github/workflows/lint-ci.yml index 49b7f7d9b2..00e6e357e2 100644 --- a/.github/workflows/lint-ci.yml +++ b/.github/workflows/lint-ci.yml @@ -44,6 +44,8 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: diff --git a/.github/workflows/mlx-ci.yml b/.github/workflows/mlx-ci.yml index 61e0566903..8cd95bd30a 100644 --- a/.github/workflows/mlx-ci.yml +++ b/.github/workflows/mlx-ci.yml @@ -89,7 +89,19 @@ jobs: # Silicon (~5-7 min), so we budget headroom. timeout-minutes: 25 steps: + # harden-runner audit mode: macOS runners cannot use blocking mode + # today (eBPF egress enforcement is Linux-only), but audit mode is + # supported cross-platform and surfaces the egress destinations in + # the runner log. This produces the data needed to graduate this + # job to a block-mode allowlist once macOS support lands. + - name: Harden runner (audit) + uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1 + with: + egress-policy: audit + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: @@ -153,7 +165,20 @@ jobs: 'httpx==0.28.1' pip install --index-url https://download.pytorch.org/whl/cpu \ 'torch==2.10.0' - pip install "unsloth_zoo @ git+https://github.com/unslothai/unsloth-zoo" + # github.com occasionally 500s on the git fetch; retry the + # zoo install so a single upstream blip does not fail CI. + for attempt in 1 2 3; do + if pip install "unsloth_zoo @ git+https://github.com/unslothai/unsloth-zoo"; then + break + fi + if [ "$attempt" -eq 3 ]; then + echo "::error::pip install unsloth_zoo failed after 3 attempts" + exit 1 + fi + delay=$((5 * attempt)) + echo "::warning::unsloth_zoo install failed (attempt $attempt/3), retrying in ${delay}s..." + sleep "$delay" + done pip install -e . --no-deps # Real Apple Silicon sanity: confirm _IS_MLX activates on real diff --git a/.github/workflows/notebooks-ci.yml b/.github/workflows/notebooks-ci.yml index 29c6ea4d1d..0881c5ef3a 100644 --- a/.github/workflows/notebooks-ci.yml +++ b/.github/workflows/notebooks-ci.yml @@ -67,10 +67,28 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: + # Validate the dispatched ref before it reaches actions/checkout's `ref:` + # input. Reading via env (NOT direct ${{ ... }} interpolation in the + # regex test) closes the GitHub-Actions-injection class where a + # client_payload.ref like `main"; rm -rf / #` would be embedded into the + # shell command. NOTEBOOKS_REF defaults to 'main' on non-dispatch + # events, but only repository_dispatch can supply attacker-controlled + # values, so we gate this check on that event type. + - name: Validate client_payload.ref shape + if: github.event_name == 'repository_dispatch' + env: + NOTEBOOKS_REF: ${{ github.event.client_payload.ref }} + run: | + if ! printf '%s' "$NOTEBOOKS_REF" | grep -Eq '^[A-Za-z0-9._/-]+$'; then + echo "::error::client_payload.ref contains disallowed characters" >&2 + exit 1 + fi + - name: Checkout unsloth (this PR) uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: unsloth + persist-credentials: false - name: Checkout unslothai/notebooks @ ${{ env.NOTEBOOKS_REF }} uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -79,6 +97,7 @@ jobs: ref: ${{ env.NOTEBOOKS_REF }} path: notebooks fetch-depth: 0 # drift check needs git status / diff + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: @@ -166,13 +185,28 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: + # See `static.Validate client_payload.ref shape` for rationale. This + # job's `if:` excludes repository_dispatch today, so the validation + # step is a defence-in-depth no-op until that gate ever relaxes. + - name: Validate client_payload.ref shape + if: github.event_name == 'repository_dispatch' + env: + NOTEBOOKS_REF: ${{ github.event.client_payload.ref }} + run: | + if ! printf '%s' "$NOTEBOOKS_REF" | grep -Eq '^[A-Za-z0-9._/-]+$'; then + echo "::error::client_payload.ref contains disallowed characters" >&2 + exit 1 + fi - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false with: { path: unsloth } - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: unslothai/notebooks ref: ${{ env.NOTEBOOKS_REF }} path: notebooks + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: { python-version: '3.12', cache: 'pip' } - name: Install @@ -200,13 +234,25 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 12 steps: + - name: Validate client_payload.ref shape + if: github.event_name == 'repository_dispatch' + env: + NOTEBOOKS_REF: ${{ github.event.client_payload.ref }} + run: | + if ! printf '%s' "$NOTEBOOKS_REF" | grep -Eq '^[A-Za-z0-9._/-]+$'; then + echo "::error::client_payload.ref contains disallowed characters" >&2 + exit 1 + fi - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false with: { path: unsloth } - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: unslothai/notebooks ref: ${{ env.NOTEBOOKS_REF }} path: notebooks + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: { python-version: '3.12', cache: 'pip' } @@ -294,13 +340,25 @@ jobs: - 'nb/Whisper.ipynb' # installation_whisper_content - 'nb/Synthetic_Data_Hackathon.ipynb' # installation_synthetic_data_content steps: + - name: Validate client_payload.ref shape + if: github.event_name == 'repository_dispatch' + env: + NOTEBOOKS_REF: ${{ github.event.client_payload.ref }} + run: | + if ! printf '%s' "$NOTEBOOKS_REF" | grep -Eq '^[A-Za-z0-9._/-]+$'; then + echo "::error::client_payload.ref contains disallowed characters" >&2 + exit 1 + fi - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false with: { path: unsloth } - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: unslothai/notebooks ref: ${{ env.NOTEBOOKS_REF }} path: notebooks + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: { python-version: '3.12' } diff --git a/.github/workflows/release-desktop.yml b/.github/workflows/release-desktop.yml index ea82739968..810bb644ba 100644 --- a/.github/workflows/release-desktop.yml +++ b/.github/workflows/release-desktop.yml @@ -3,16 +3,306 @@ name: Release Desktop App on: workflow_dispatch: inputs: + studio_version: + description: 'Studio version tag to release (for example, v0.1.39-beta)' + type: string + required: true + pypi_version: + description: 'Exact PyPI unsloth version just published/stamped (for example, 2026.5.3); leave blank to use MIN_DESKTOP_BACKEND_VERSION' + type: string + required: false draft: - description: 'Create as draft release' + description: 'Create as draft release; draft runs do not advance desktop-latest updater channel' type: boolean default: true permissions: - contents: write + contents: read + +concurrency: + group: release-desktop-${{ github.repository }} + cancel-in-progress: false jobs: + prepare-version: + name: Prepare release versions + runs-on: ubuntu-latest + outputs: + studio_version: ${{ steps.prepare.outputs.studio_version }} + app_version: ${{ steps.prepare.outputs.app_version }} + desktop_release_tag: ${{ steps.prepare.outputs.desktop_release_tag }} + prerelease: ${{ steps.prepare.outputs.prerelease }} + pypi_version: ${{ steps.prepare.outputs.pypi_version }} + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + persist-credentials: false + + - name: Validate release versions + id: prepare + shell: bash + env: + INPUT_STUDIO_VERSION: ${{ inputs.studio_version }} + INPUT_PYPI_VERSION: ${{ inputs.pypi_version }} + run: | + python3 <<'PY' + import os + import pathlib + import re + import sys + + studio_version = os.environ['INPUT_STUDIO_VERSION'].strip() + if not studio_version: + sys.exit('studio_version is required, for example v0.1.39-beta') + if re.fullmatch(r'v?20\d{2}\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?', studio_version): + sys.exit(f'studio_version must be a Studio SemVer tag, not a date-style backend version: {studio_version}') + + semver_tag = re.compile( + r'^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)' + r'(?:-[0-9A-Za-z.][0-9A-Za-z.-]*)?$' + ) + if not semver_tag.fullmatch(studio_version): + sys.exit(f'studio_version must be a SemVer tag with leading v, for example v0.1.39-beta: {studio_version}') + + app_version = studio_version.removeprefix('v') + desktop_release_tag = f'desktop-v{app_version}' + prerelease = 'true' if '-' in app_version.split('+', 1)[0] else 'false' + + def parse_backend_version(version): + match = re.fullmatch( + r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)' + r'(?:([a-zA-Z]|\.dev|dev|\.rc|rc|\.post|post)(\d*))?' + r'(?:[-+]([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?', + version, + ) + if not match: + return None + major, minor, patch, suffix_name, suffix_number, suffix_text = match.groups() + if suffix_name: + normalized = suffix_name.lower().lstrip('.') + order = {'dev': 0, 'a': 1, 'b': 2, 'rc': 3, 'post': 5}.get(normalized) + if order is None: + return None + number = int(suffix_number or '0') + elif suffix_text: + order = 3 if version[version.find(suffix_text) - 1] == '-' else 4 + number = 0 + else: + order = 4 + number = 0 + return (int(major), int(minor), int(patch), order, number) + + preflight = pathlib.Path('studio/src-tauri/src/preflight/version.rs').read_text() + match = re.search(r'MIN_DESKTOP_BACKEND_VERSION:\s*&str\s*=\s*"([^"]+)"', preflight) + if not match: + sys.exit('Could not read MIN_DESKTOP_BACKEND_VERSION') + min_backend_version = match.group(1) + + input_pypi_version = os.environ.get('INPUT_PYPI_VERSION', '').strip() + parsed_min_backend = parse_backend_version(min_backend_version) + if parsed_min_backend is None: + sys.exit(f'MIN_DESKTOP_BACKEND_VERSION is not a supported backend package version: {min_backend_version}') + + pypi_version = input_pypi_version or min_backend_version + parsed_pypi = parse_backend_version(pypi_version) + if parsed_pypi is None: + sys.exit(f'pypi_version is not a supported backend package version: {pypi_version}') + if parsed_pypi < parsed_min_backend: + sys.exit( + f'pypi_version {pypi_version} is lower than desktop minimum ' + f'MIN_DESKTOP_BACKEND_VERSION {min_backend_version}' + ) + + if input_pypi_version: + print( + 'Using exact PyPI unsloth version from pypi_version input: ' + f'{pypi_version} (desktop minimum: {min_backend_version})' + ) + else: + print( + 'Using exact PyPI unsloth version from MIN_DESKTOP_BACKEND_VERSION: ' + f'{pypi_version}' + ) + + with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as output: + print(f'studio_version={studio_version}', file=output) + print(f'app_version={app_version}', file=output) + print(f'desktop_release_tag={desktop_release_tag}', file=output) + print(f'prerelease={prerelease}', file=output) + print(f'pypi_version={pypi_version}', file=output) + PY + + - name: Verify PyPI package and Studio stamp + shell: bash + env: + STUDIO_VERSION: ${{ steps.prepare.outputs.studio_version }} + PYPI_VERSION: ${{ steps.prepare.outputs.pypi_version }} + run: | + set -euo pipefail + python3 <<'PY' + import json + import os + import pathlib + import sys + import time + import urllib.error + import urllib.request + + pypi_version = os.environ['PYPI_VERSION'] + dist_dir = pathlib.Path(os.environ['RUNNER_TEMP'], 'pypi-unsloth-dist') + dist_dir.mkdir(parents=True, exist_ok=True) + metadata_url = f'https://pypi.org/pypi/unsloth/{pypi_version}/json' + + last_error = None + for attempt in range(1, 6): + try: + with urllib.request.urlopen(metadata_url, timeout=30) as response: + metadata = json.load(response) + break + except Exception as exc: + last_error = exc + if attempt < 5: + time.sleep(10 * attempt) + else: + sys.exit(f'Publish unsloth=={pypi_version} to PyPI before the desktop release ({last_error})') + + files = metadata.get('urls') or [] + if not files: + sys.exit(f'PyPI returned no distribution files for unsloth=={pypi_version}') + + for file_info in files: + filename = file_info.get('filename') + url = file_info.get('url') + if not filename or '/' in filename or not url: + sys.exit(f'Unexpected PyPI file entry for unsloth=={pypi_version}: {file_info!r}') + target = dist_dir / filename + for attempt in range(1, 4): + try: + with urllib.request.urlopen(url, timeout=60) as response: + target.write_bytes(response.read()) + break + except Exception as exc: + last_error = exc + if attempt < 3: + time.sleep(5 * attempt) + else: + sys.exit(f'Could not download {filename} from PyPI ({last_error})') + PY + + if [ -f scripts/stamp_studio_release.py ]; then + mapfile -t dists < <(find "$RUNNER_TEMP/pypi-unsloth-dist" -type f \( -name '*.whl' -o -name '*.tar.gz' \) | sort) + if [ "${#dists[@]}" -eq 0 ]; then + echo "No PyPI wheel/sdist artifacts downloaded for unsloth==$PYPI_VERSION" >&2 + exit 1 + fi + python3 scripts/stamp_studio_release.py --verify-dist "$RUNNER_TEMP/pypi-unsloth-dist" --expected "$STUDIO_VERSION" + else + echo "scripts/stamp_studio_release.py not found; release-desktop requires #5308 to verify the PyPI Studio stamp." >&2 + exit 1 + fi + + - name: Guard public updater channel version + if: ${{ !inputs.draft }} + shell: bash + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} + APP_VERSION: ${{ steps.prepare.outputs.app_version }} + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/desktop-current" + if ! gh release download desktop-latest --pattern latest.json --dir "$RUNNER_TEMP/desktop-current" --clobber 2>/dev/null; then + echo "No existing desktop-latest latest.json found; allowing first channel publish." + exit 0 + fi + python3 <<'PY' + import json + import os + import pathlib + import re + import sys + + def parse(value: str): + value = value.removeprefix('v') + match = re.fullmatch( + r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)' + r'(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?' + r'(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?', + value, + ) + if not match: + sys.exit(f'desktop-latest latest.json has invalid version: {value}') + major, minor, patch, prerelease = match.groups() + return (int(major), int(minor), int(patch), prerelease) + + def numeric_tail(identifier: str) -> tuple[str, int] | None: + match = re.fullmatch(r'([A-Za-z-]+)(\d+)', identifier) + if not match: + return None + return (match.group(1).lower(), int(match.group(2))) + + def compare_identifier(left: str, right: str) -> int: + left_num = left.isdigit() + right_num = right.isdigit() + if left_num and right_num: + return (int(left) > int(right)) - (int(left) < int(right)) + if left_num: + return -1 + if right_num: + return 1 + + left_tail = numeric_tail(left) + right_tail = numeric_tail(right) + if left_tail and right_tail and left_tail[0] == right_tail[0]: + return (left_tail[1] > right_tail[1]) - (left_tail[1] < right_tail[1]) + + return (left > right) - (left < right) + + def compare_prerelease(left: str | None, right: str | None) -> int: + if left == right: + return 0 + if left is None: + return 1 + if right is None: + return -1 + left_parts = left.split('.') + right_parts = right.split('.') + for left_part, right_part in zip(left_parts, right_parts): + order = compare_identifier(left_part, right_part) + if order: + return order + return (len(left_parts) > len(right_parts)) - (len(left_parts) < len(right_parts)) + + def compare(left: str, right: str) -> int: + left_major, left_minor, left_patch, left_pre = parse(left) + right_major, right_minor, right_patch, right_pre = parse(right) + left_core = (left_major, left_minor, left_patch) + right_core = (right_major, right_minor, right_patch) + if left_core != right_core: + return (left_core > right_core) - (left_core < right_core) + return compare_prerelease(left_pre, right_pre) + + current_path = pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-current', 'latest.json') + current = json.loads(current_path.read_text()).get('version') + next_version = os.environ['APP_VERSION'] + if not isinstance(current, str): + sys.exit('desktop-latest latest.json has missing version') + if compare(next_version, current) < 0: + sys.exit( + f'Refusing to publish {next_version}; desktop-latest currently points at newer version {current}.' + ) + PY + build: + # TODO: split into a "build (no secrets)" + "publish (secrets)" job pair + # with actions/upload-artifact handoff so the matrix build cannot + # publish a Release on its own. The current matrix runs across + # Linux/macOS/Windows in a single job, so the split needs artefact + # collection across the OS matrix and is out of scope for this + # hardening pass. + permissions: + contents: write # tauri-apps/tauri-action creates / uploads a GitHub Release strategy: fail-fast: false max-parallel: 1 @@ -32,14 +322,31 @@ jobs: label: Windows (x64) name: Build ${{ matrix.label }} + needs: prepare-version runs-on: ${{ matrix.platform }} env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true - + APP_VERSION: ${{ needs.prepare-version.outputs.app_version }} + STUDIO_VERSION: ${{ needs.prepare-version.outputs.studio_version }} + DESKTOP_RELEASE_TAG: ${{ needs.prepare-version.outputs.desktop_release_tag }} + DESKTOP_PRERELEASE: ${{ needs.prepare-version.outputs.prerelease }} steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + # harden-runner in audit mode: surfaces every egress destination in + # the runner log so the allowlist for a future `egress-policy: block` + # promotion can be derived from observed traffic. Audit mode is + # cross-platform (Linux / macOS / Windows runners); blocking mode is + # currently Linux-only, so we deliberately stay in audit until the + # macOS + Windows codesign paths have been observed. + - name: Harden runner (audit) + uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1 + with: + egress-policy: audit + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + persist-credentials: false # ── Linux dependencies ── - name: Install Linux dependencies @@ -50,12 +357,18 @@ jobs: # ── Node.js ── - name: Setup Node.js - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e with: node-version: 24 - name: Install pinned Tauri CLI - run: npm install --save-dev --prefix studio @tauri-apps/cli@2.10.1 + # Lifecycle scripts (esbuild native-binary postinstall, etc.) are + # required for `vite build`. The pre-install lockfile structural + # audit (lockfile_supply_chain_audit.py) is the practical defence + # against the npm postinstall-dropper class -- it fires BEFORE any + # tarball runs, on the injection pattern itself rather than an + # advisory-DB lookup. + run: npm install --save-dev --prefix studio @tauri-apps/cli@2.10.1 --no-fund --no-audit - name: Verify pinned Tauri CLI shell: bash @@ -67,38 +380,152 @@ jobs: exit 1 fi - - name: Install frontend dependencies - working-directory: studio/frontend - run: npm install - - - name: Verify backend package is published + - name: Verify desktop updater and Linux package config shell: bash run: | node <<'JS' const { readFileSync } = require('node:fs'); - (async () => { - const cargo = readFileSync('studio/src-tauri/Cargo.toml', 'utf8'); - const match = cargo.match(/^version\s*=\s*"([^"]+)"/m); - if (!match) throw new Error('Could not read desktop app version'); + const expected = 'https://github.com/unslothai/unsloth/releases/download/desktop-latest/latest.json'; + const config = JSON.parse(readFileSync('studio/src-tauri/tauri.conf.json', 'utf8')); + const endpoints = config.plugins?.updater?.endpoints; + if (!Array.isArray(endpoints) || endpoints.length !== 1) { + throw new Error('Expected exactly one desktop updater endpoint'); + } + if (endpoints[0] !== expected) { + throw new Error('Desktop updater endpoint must be ' + expected + ', got ' + endpoints[0]); + } + if (endpoints.some((endpoint) => endpoint.includes('/releases/latest/'))) { + throw new Error('Desktop updater endpoint must not use repo-wide /releases/latest/'); + } - const appVersion = match[1]; - const response = await fetch(`https://pypi.org/pypi/unsloth/${appVersion}/json`); - if (!response.ok) { - const message = 'Publish unsloth=={app_version} to PyPI before the desktop release'; - throw new Error(`${message.replace('{app_version}', appVersion)} (HTTP ${response.status})`); + const targets = config.bundle?.targets; + if (Array.isArray(targets) && targets.some((target) => String(target).toLowerCase() === 'rpm')) { + throw new Error('Desktop release must not target RPM packages'); + } + if (config.bundle?.linux?.rpm) { + throw new Error('bundle.linux.rpm must not be configured'); + } + + const workflow = readFileSync('.github/workflows/release-desktop.yml', 'utf8'); + const lines = workflow.split(/\r?\n/); + const releaseBodies = []; + for (let i = 0; i < lines.length; i += 1) { + const match = lines[i].match(/^(\s*)releaseBody:\s*\|\s*$/); + if (!match) continue; + const baseIndent = match[1].length; + const bodyLines = []; + i += 1; + for (; i < lines.length; i += 1) { + const line = lines[i]; + if (line.trim() === '') { + bodyLines.push(''); + continue; + } + const indent = line.match(/^\s*/)[0].length; + if (indent <= baseIndent) { + i -= 1; + break; + } + bodyLines.push(line.slice(baseIndent + 2)); } - })(); + releaseBodies.push(bodyLines.join('\n')); + } + if (releaseBodies.length === 0) { + throw new Error('Expected at least one desktop release body'); + } + for (const body of releaseBodies) { + if (/\brpm\b|\.rpm/i.test(body)) { + throw new Error('Desktop release body must not advertise RPM packages'); + } + } JS + - name: Install frontend dependencies + working-directory: studio/frontend + # Lifecycle scripts (esbuild native-binary postinstall, etc.) are + # required for `vite build`. The pre-install lockfile structural + # audit (lockfile_supply_chain_audit.py) is the practical defence + # against the npm postinstall-dropper class -- it fires BEFORE any + # tarball runs, on the injection pattern itself rather than an + # advisory-DB lookup. + run: npm install --no-fund --no-audit + # ── Rust ── - name: Install Rust stable - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @ 2026-03-27 with: targets: ${{ matrix.platform == 'macos-latest' && 'aarch64-apple-darwin,x86_64-apple-darwin' || '' }} + - name: Patch desktop app version + shell: bash + working-directory: studio/src-tauri + run: | + set -euo pipefail + if command -v python3 >/dev/null 2>&1; then + PYTHON=python3 + else + PYTHON=python + fi + "$PYTHON" <<'PY' + import os + import pathlib + import re + import sys + + app_version = os.environ['APP_VERSION'] + if not app_version: + sys.exit('APP_VERSION is required') + + cargo_toml = pathlib.Path('Cargo.toml') + lines = cargo_toml.read_text().splitlines(keepends=True) + in_package = False + patched = False + for index, line in enumerate(lines): + stripped = line.strip() + if stripped == '[package]': + in_package = True + continue + if stripped.startswith('[') and stripped.endswith(']'): + in_package = False + if in_package and re.fullmatch(r'version\s*=\s*"[^"]+"\s*', stripped): + lines[index] = f'version = "{app_version}"\n' + patched = True + break + if not patched: + sys.exit('Could not patch [package] version in Cargo.toml') + cargo_toml.write_text(''.join(lines)) + + cargo_lock = pathlib.Path('Cargo.lock') + lock_text = cargo_lock.read_text() + lock_text, count = re.subn( + r'(?m)(^\[\[package\]\]\nname = "unsloth-studio"\nversion = ")[^"]+(")', + lambda match: f'{match.group(1)}{app_version}{match.group(2)}', + lock_text, + ) + if count != 1: + sys.exit(f'Could not patch unsloth-studio version in Cargo.lock (matches={count})') + cargo_lock.write_text(lock_text) + PY + + cargo metadata --locked --no-deps --format-version 1 > "$RUNNER_TEMP/cargo-metadata.json" + "$PYTHON" <<'PY' + import json + import os + import pathlib + import sys + + app_version = os.environ['APP_VERSION'] + metadata = json.loads(pathlib.Path(os.environ['RUNNER_TEMP'], 'cargo-metadata.json').read_text()) + versions = [package['version'] for package in metadata.get('packages', []) if package.get('name') == 'unsloth-studio'] + if versions != [app_version]: + sys.exit(f'cargo metadata unsloth-studio version mismatch: expected {app_version}, got {versions}') + PY + + git diff -- Cargo.toml Cargo.lock + - name: Rust cache - uses: swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae + uses: swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 with: workspaces: 'studio/src-tauri -> target' @@ -146,8 +573,8 @@ jobs: with: projectPath: studio tauriScript: npx --prefix . tauri - tagName: desktop-v__VERSION__ - releaseName: 'Unsloth Studio (Desktop) v__VERSION__' + tagName: ${{ needs.prepare-version.outputs.desktop_release_tag }} + releaseName: 'Unsloth Studio (Desktop) ${{ needs.prepare-version.outputs.studio_version }}' releaseBody: | Desktop app for Unsloth Studio. @@ -159,7 +586,7 @@ jobs: > Linux AppImage on Ubuntu 24.04+ may require: `sudo apt install libfuse2t64` > First-run system dependency elevation is supported on Ubuntu/Debian. Other Linux distributions should install system packages manually. releaseDraft: ${{ inputs.draft }} - prerelease: false + prerelease: ${{ needs.prepare-version.outputs.prerelease }} args: -v ${{ matrix.args }} # ── macOS: build + sign + notarize + upload ── @@ -177,8 +604,8 @@ jobs: with: projectPath: studio tauriScript: npx --prefix . tauri - tagName: desktop-v__VERSION__ - releaseName: 'Unsloth Studio (Desktop) v__VERSION__' + tagName: ${{ needs.prepare-version.outputs.desktop_release_tag }} + releaseName: 'Unsloth Studio (Desktop) ${{ needs.prepare-version.outputs.studio_version }}' releaseBody: | Desktop app for Unsloth Studio. @@ -190,7 +617,7 @@ jobs: > Linux AppImage on Ubuntu 24.04+ may require: `sudo apt install libfuse2t64` > First-run system dependency elevation is supported on Ubuntu/Debian. Other Linux distributions should install system packages manually. releaseDraft: ${{ inputs.draft }} - prerelease: false + prerelease: ${{ needs.prepare-version.outputs.prerelease }} args: -v ${{ matrix.args }} # ── Windows: build + sign + upload ── @@ -209,8 +636,8 @@ jobs: with: projectPath: studio tauriScript: npx --prefix . tauri - tagName: desktop-v__VERSION__ - releaseName: 'Unsloth Studio (Desktop) v__VERSION__' + tagName: ${{ needs.prepare-version.outputs.desktop_release_tag }} + releaseName: 'Unsloth Studio (Desktop) ${{ needs.prepare-version.outputs.studio_version }}' releaseBody: | Desktop app for Unsloth Studio. @@ -222,5 +649,254 @@ jobs: > Linux AppImage on Ubuntu 24.04+ may require: `sudo apt install libfuse2t64` > First-run system dependency elevation is supported on Ubuntu/Debian. Other Linux distributions should install system packages manually. releaseDraft: ${{ inputs.draft }} - prerelease: false + prerelease: ${{ needs.prepare-version.outputs.prerelease }} args: -v ${{ matrix.args }} + + # Release process note: only non-draft workflow runs advance the public + # desktop-latest updater channel. Draft builds are for private review; if a + # draft is manually published later, this channel intentionally remains + # unchanged until a narrow manual channel-publish flow is added or a public + # desktop release is created by running this workflow with draft=false. + publish-updater-channel: + name: Publish desktop updater channel + needs: [prepare-version, build] + if: ${{ !inputs.draft }} + runs-on: ubuntu-latest + permissions: + contents: write + env: + GH_REPO: ${{ github.repository }} + APP_VERSION: ${{ needs.prepare-version.outputs.app_version }} + STUDIO_VERSION: ${{ needs.prepare-version.outputs.studio_version }} + DESKTOP_RELEASE_TAG: ${{ needs.prepare-version.outputs.desktop_release_tag }} + DESKTOP_PRERELEASE: ${{ needs.prepare-version.outputs.prerelease }} + + steps: + - name: Download versioned updater metadata + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/desktop-updater" + gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${DESKTOP_RELEASE_TAG}" > "$RUNNER_TEMP/source-release.json" + python3 <<'PY' + import json + import os + import pathlib + import sys + + source = json.loads(pathlib.Path(os.environ['RUNNER_TEMP'], 'source-release.json').read_text()) + expected_tag = os.environ['DESKTOP_RELEASE_TAG'] + if source.get('tag_name') != expected_tag: + sys.exit(f'Expected source release {expected_tag}, got {source.get("tag_name")}') + if source.get('draft'): + sys.exit(f'Source desktop release {expected_tag} is draft; refusing to publish public updater channel') + PY + gh release download "$DESKTOP_RELEASE_TAG" --pattern latest.json --dir "$RUNNER_TEMP/desktop-updater" --clobber + test -s "$RUNNER_TEMP/desktop-updater/latest.json" + + - name: Validate versioned updater metadata + shell: bash + run: | + python3 <<'PY' + import json + import os + import pathlib + import re + import sys + + app_version = os.environ['APP_VERSION'] + release_tag = os.environ['DESKTOP_RELEASE_TAG'] + latest_path = pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-updater', 'latest.json') + data = json.loads(latest_path.read_text()) + if not isinstance(data, dict): + sys.exit('latest.json must be a JSON object') + + version = data.get('version') + if not isinstance(version, str) or not version: + sys.exit('latest.json missing version') + if not re.fullmatch(r'v?\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?', version): + sys.exit(f'latest.json version is not SemVer-like: {version}') + if version.removeprefix('v') != app_version: + sys.exit(f'latest.json version {version} does not match desktop app version {app_version}') + + platforms = data.get('platforms') + if not isinstance(platforms, dict) or not platforms: + sys.exit('latest.json missing platforms') + + required_families = { + 'darwin-aarch64': False, + 'linux-x86_64': False, + 'windows-x86_64': False, + } + expected_prefix = f'https://github.com/unslothai/unsloth/releases/download/{release_tag}/' + forbidden_fragments = ('/releases/latest/', '/releases/download/desktop-latest/') + + for platform, entry in platforms.items(): + if not isinstance(entry, dict): + sys.exit(f'Platform {platform} must be an object') + url = entry.get('url') + signature = entry.get('signature') + if not isinstance(url, str) or not url.strip(): + sys.exit(f'Platform {platform} missing url') + if not isinstance(signature, str) or not signature.strip(): + sys.exit(f'Platform {platform} missing signature') + if any(fragment in url for fragment in forbidden_fragments): + sys.exit(f'Platform {platform} points at a moving updater channel: {url}') + if not url.startswith(expected_prefix): + sys.exit(f'Platform {platform} URL must point at {release_tag}: {url}') + for family in required_families: + if platform == family or platform.startswith(family + '-'): + required_families[family] = True + + missing = [family for family, found in required_families.items() if not found] + if missing: + sys.exit('latest.json missing required platform families: ' + ', '.join(missing)) + PY + + - name: Ensure desktop updater channel release + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + channel_json="$RUNNER_TEMP/desktop-latest-release.json" + if ! gh api "repos/${GITHUB_REPOSITORY}/releases/tags/desktop-latest" > "$channel_json" 2>/dev/null; then + gh release create desktop-latest \ + --title "Unsloth Studio Desktop updater channel" \ + --notes "Machine-managed desktop updater channel; latest.json is replaced by release-desktop.yml." \ + --prerelease \ + --latest=false \ + --target "$GITHUB_SHA" + gh api "repos/${GITHUB_REPOSITORY}/releases/tags/desktop-latest" > "$channel_json" + fi + + python3 <<'PY' + import json + import os + import pathlib + import sys + + channel = json.loads(pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-latest-release.json').read_text()) + if channel.get('draft'): + sys.exit('desktop-latest release is draft; refusing to publish updater channel') + if channel.get('immutable'): + sys.exit('desktop-latest release is immutable; cannot replace latest.json') + if not channel.get('prerelease'): + sys.exit('desktop-latest release must be a prerelease so it cannot compete with repo-wide latest') + PY + + - name: Prevent updater channel downgrade + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/desktop-current" + if ! gh release download desktop-latest --pattern latest.json --dir "$RUNNER_TEMP/desktop-current" --clobber 2>/dev/null; then + echo "No existing desktop-latest latest.json found; allowing first channel publish." + exit 0 + fi + python3 <<'PY' + import json + import os + import pathlib + import re + import sys + + def parse(value: str): + value = value.removeprefix('v') + match = re.fullmatch( + r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)' + r'(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?' + r'(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?', + value, + ) + if not match: + sys.exit(f'desktop-latest latest.json has invalid version: {value}') + major, minor, patch, prerelease = match.groups() + return (int(major), int(minor), int(patch), prerelease) + + def numeric_tail(identifier: str) -> tuple[str, int] | None: + match = re.fullmatch(r'([A-Za-z-]+)(\d+)', identifier) + if not match: + return None + return (match.group(1).lower(), int(match.group(2))) + + def compare_identifier(left: str, right: str) -> int: + left_num = left.isdigit() + right_num = right.isdigit() + if left_num and right_num: + return (int(left) > int(right)) - (int(left) < int(right)) + if left_num: + return -1 + if right_num: + return 1 + + left_tail = numeric_tail(left) + right_tail = numeric_tail(right) + if left_tail and right_tail and left_tail[0] == right_tail[0]: + return (left_tail[1] > right_tail[1]) - (left_tail[1] < right_tail[1]) + + return (left > right) - (left < right) + + def compare_prerelease(left: str | None, right: str | None) -> int: + if left == right: + return 0 + if left is None: + return 1 + if right is None: + return -1 + left_parts = left.split('.') + right_parts = right.split('.') + for left_part, right_part in zip(left_parts, right_parts): + order = compare_identifier(left_part, right_part) + if order: + return order + return (len(left_parts) > len(right_parts)) - (len(left_parts) < len(right_parts)) + + def compare(left: str, right: str) -> int: + left_major, left_minor, left_patch, left_pre = parse(left) + right_major, right_minor, right_patch, right_pre = parse(right) + left_core = (left_major, left_minor, left_patch) + right_core = (right_major, right_minor, right_patch) + if left_core != right_core: + return (left_core > right_core) - (left_core < right_core) + return compare_prerelease(left_pre, right_pre) + + current_path = pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-current', 'latest.json') + next_path = pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-updater', 'latest.json') + current = json.loads(current_path.read_text()).get('version') + next_version = json.loads(next_path.read_text()).get('version') + if not isinstance(current, str) or not isinstance(next_version, str): + sys.exit('Could not compare desktop-latest channel versions') + if compare(next_version, current) < 0: + sys.exit( + f'Refusing to move desktop-latest from {current} to older version {next_version}.' + ) + PY + + - name: Publish desktop updater channel metadata + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + gh release upload desktop-latest "$RUNNER_TEMP/desktop-updater/latest.json" --clobber + gh api "repos/${GITHUB_REPOSITORY}/releases/tags/desktop-latest" > "$RUNNER_TEMP/desktop-latest-release.json" + python3 <<'PY' + import json + import os + import pathlib + import sys + + channel = json.loads(pathlib.Path(os.environ['RUNNER_TEMP'], 'desktop-latest-release.json').read_text()) + assets = [asset for asset in channel.get('assets', []) if asset.get('name') == 'latest.json'] + if len(assets) != 1: + sys.exit(f'Expected exactly one desktop-latest latest.json asset, found {len(assets)}') + expected_url = f'https://github.com/{os.environ["GITHUB_REPOSITORY"]}/releases/download/desktop-latest/latest.json' + actual_url = assets[0].get('browser_download_url') + if actual_url != expected_url: + sys.exit(f'desktop-latest latest.json URL mismatch: expected {expected_url}, got {actual_url}') + PY diff --git a/.github/workflows/security-audit.yml b/.github/workflows/security-audit.yml index 0fc8073e75..235fde5253 100644 --- a/.github/workflows/security-audit.yml +++ b/.github/workflows/security-audit.yml @@ -57,6 +57,7 @@ on: - 'studio/src-tauri/Cargo.lock' - 'pyproject.toml' - 'scripts/scan_packages.py' + - 'scripts/scan_npm_packages.py' - '.github/workflows/security-audit.yml' push: branches: [main, pip] @@ -97,17 +98,36 @@ jobs: # char SHA freezes this action at known-good code; Dependabot's # github-actions ecosystem will auto-bump the SHA. # v2.19.1 commit: - - name: Harden runner (egress audit) + # Per-job allowlist: advisory-audit hits PyPI, npm registry, + # crates.io advisories, GitHub release artefacts (osv-scanner + # binary), Semgrep registry, and TruffleHog's own GitHub action. + - name: Harden runner (egress block) uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1 with: - egress-policy: audit + egress-policy: block disable-sudo: true + allowed-endpoints: > + api.github.com:443 + github.com:443 + codeload.github.com:443 + objects.githubusercontent.com:443 + raw.githubusercontent.com:443 + release-assets.githubusercontent.com:443 + registry.npmjs.org:443 + pypi.org:443 + files.pythonhosted.org:443 + static.rust-lang.org:443 + index.crates.io:443 + static.crates.io:443 + crates.io:443 + semgrep.dev:443 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: # Full history so TruffleHog can diff base..head; without # this it sees only the latest commit and reports nothing. fetch-depth: 0 + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: @@ -122,7 +142,7 @@ jobs: - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @ 2026-03-27 - - uses: swatinem/rust-cache@23869a5bd66c73db3c0ac40331f3206eb23791dc # v2.9.1 + - uses: swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2.9.1 with: workspaces: studio/src-tauri -> target @@ -244,6 +264,27 @@ jobs: echo '```' } >> "$GITHUB_STEP_SUMMARY" + # ───────────────────────────────────────────────────────────── + # Pre-install lockfile supply-chain audit (npm + cargo). + # Catches structural anomalies (non-registry resolved URLs, + # missing integrity hashes, known IOC strings) BEFORE `npm + # audit` or OSV-Scanner consult the advisory DB. The advisory + # path is reactive -- there is a window between a malicious + # publication and the GHSA landing. This step fires on the + # injection pattern itself so it catches the same class of + # attack the moment the lockfile shape becomes wrong. + # ───────────────────────────────────────────────────────────── + - name: Lockfile supply-chain audit (pre-install scan) + run: | + python3 scripts/lockfile_supply_chain_audit.py + { + echo "## Lockfile supply-chain audit" + echo + echo "Scanned: studio/frontend/package-lock.json + studio/src-tauri/Cargo.lock" + echo + echo "No structural anomalies or known IOC strings." + } >> "$GITHUB_STEP_SUMMARY" + # ───────────────────────────────────────────────────────────── # npm: Studio frontend # ───────────────────────────────────────────────────────────── @@ -437,7 +478,7 @@ jobs: # ───────────────────────────────────────────────────────────── - name: TruffleHog (secrets in diff) continue-on-error: true - uses: trufflesecurity/trufflehog@17456f8c7d042d8c82c9a8ca9e937231f9f42e26 # v3.95.2 + uses: trufflesecurity/trufflehog@37b77001d0174ebec2fcca2bd83ff83a6d45a3ab # v3.95.3 with: path: ./ base: ${{ github.event.pull_request.base.sha || '' }} @@ -662,17 +703,28 @@ jobs: id: extras files: 'extras' steps: - # Egress audit on every shard. Each shard pulls hundreds of + # Egress block on every shard. Each shard pulls hundreds of # PyPI archives -- if a malicious wheel ever phones home from # within the scanner sandbox (it shouldn't; we never execute - # the archive), harden-runner's audit log records the host. - - name: Harden runner (egress audit) + # the archive), harden-runner now rejects the connect outright. + # Per-job allowlist: pip-scan-packages only fetches PyPI archives + # via scan_packages.py + pip download. No npm or cargo traffic. + - name: Harden runner (egress block) uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1 with: - egress-policy: audit + egress-policy: block disable-sudo: true + allowed-endpoints: > + api.github.com:443 + github.com:443 + codeload.github.com:443 + objects.githubusercontent.com:443 + pypi.org:443 + files.pythonhosted.org:443 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: @@ -749,7 +801,6 @@ jobs: # transitive set (no point fetching the same transformers # wheel five times). Across shards we accept some redundant # downloads in exchange for wall-clock parallelism. - continue-on-error: true env: SHARD_FILES: ${{ matrix.shard.files }} run: | @@ -794,3 +845,286 @@ jobs: logs-scan-packages-${{ matrix.shard.id }}.txt audit-reqs/ retention-days: 30 + + # ───────────────────────────────────────────────────────────────────── + # npm: pre-install tarball content scan. + # ───────────────────────────────────────────────────────────────────── + npm-scan-packages: + # Counterpart to pip-scan-packages for the npm side. Reads + # studio/frontend/package-lock.json, downloads each resolved + # tarball DIRECTLY from registry.npmjs.org (never via `npm + # install` -- no lifecycle scripts ever run), verifies the + # lockfile integrity hash, unpacks each tarball into a sandboxed + # temp dir behind size / count / path-escape / symlink guards, + # and pattern-scans the extracted file contents for the + # signatures common to npm supply-chain attacks: + # + # - lifecycle (preinstall / install / postinstall / prepare) + # scripts in any package.json that fetch + execute external + # code, + # - C2 / exfiltration hosts (getsession.org, AWS IMDS, + # Kubernetes ServiceAccount token paths, GitHub Actions OIDC, + # HashiCorp Vault endpoints), + # - credential-stealing references (.npmrc, .aws/credentials, + # GITHUB_TOKEN / NPM_TOKEN in JS sources), + # - known IOC filenames (router_init.js, tanstack_runner.js, + # router_runtime.js), + # - obfuscation shapes (Function/eval against base64 blobs). + # + # Threat model: every tarball is hostile. Safety guarantees are + # documented at scripts/scan_npm_packages.py top-of-file. The + # script is stdlib-only so adding it does not increase the + # transitive supply-chain surface. + name: npm scan-packages (Studio frontend tarballs) + runs-on: ubuntu-latest + timeout-minutes: 30 + needs: [] + steps: + # Per-job allowlist: npm-scan-packages only fetches tarballs from + # registry.npmjs.org. GitHub endpoints retained for checkout + + # setup-python action machinery. + - name: Harden runner (egress block) + uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1 + with: + egress-policy: block + disable-sudo: true + allowed-endpoints: > + api.github.com:443 + github.com:443 + codeload.github.com:443 + objects.githubusercontent.com:443 + registry.npmjs.org:443 + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + + - name: Sanity-check scan_npm_packages.py + run: | + test -f scripts/scan_npm_packages.py + python3 -c "import ast; ast.parse(open('scripts/scan_npm_packages.py').read())" + + - name: Scan npm tarballs (declared + transitive, no install) + # The script exits 1 on HIGH/CRITICAL findings; we capture the + # full log and surface it in the step summary either way. It + # never runs `npm install`, never executes anything from a + # downloaded tarball, and only fetches from registry.npmjs.org. + # Initially non-blocking so the baseline can settle; drop + # continue-on-error once the baseline is clean for a week. + run: | + set -o pipefail + LOG=logs-scan-npm.txt + python3 scripts/scan_npm_packages.py 2>&1 | tee "$LOG" + { + echo "## scan_npm_packages" + echo + echo '### Findings (tail)' + echo '```' + tail -300 "$LOG" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: scan-npm-packages-log + path: logs-scan-npm.txt + retention-days: 30 + + # ───────────────────────────────────────────────────────────────────── + # Workflow-trigger lint. Refuses two patterns that together powered the + # TanStack GHSA-g7cv-rxg3-hmpx supply-chain compromise: + # + # 1. `pull_request_target` -- runs a fork's workflow YAML against + # the base repository's secrets. There is no safe use of this + # trigger for a public open-source project. + # + # 2. Shared cache keys between PR-triggered workflows and the + # publish workflow. A fork PR can poison the cache; the publish + # workflow then restores the poisoned cache on next run. + # + # Cheap pure-Python lint, runs in seconds. Fail-closed. + # ───────────────────────────────────────────────────────────────────── + workflow-trigger-lint: + name: workflow-trigger lint (pull_request_target / cache-poisoning) + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Harden runner (egress block) + uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1 + with: + egress-policy: block + disable-sudo: true + allowed-endpoints: > + api.github.com:443 + github.com:443 + codeload.github.com:443 + objects.githubusercontent.com:443 + pypi.org:443 + files.pythonhosted.org:443 + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + + - name: Install PyYAML + run: pip install pyyaml + + - name: Lint workflow triggers + cache keys + run: python3 scripts/lint_workflow_triggers.py + + # ───────────────────────────────────────────────────────────────────── + # Regression tests: pin scanner IOC tables and pre-install fixtures. + # Hard gate (no continue-on-error) so future drift in the IOC tables + # or scanner exit semantics fails this PR at review time. + # ───────────────────────────────────────────────────────────────────── + tests-security: + name: pytest tests/security + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Harden runner (egress block) + uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1 + with: + egress-policy: block + disable-sudo: true + allowed-endpoints: > + api.github.com:443 + github.com:443 + codeload.github.com:443 + objects.githubusercontent.com:443 + pypi.org:443 + files.pythonhosted.org:443 + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + + - name: Install pytest + PyYAML + # PyYAML is imported by scripts/lint_workflow_triggers.py, which the + # `tests/security/test_lint_workflow_triggers.py` regression suite + # exercises as a subprocess. Without it the lint script bails with + # `ERROR: PyYAML is required` (exit 2) and the 5 lint regression + # tests fail. Pinned the same way pytest is pinned. + run: pip install pytest==9.0.3 pyyaml==6.0.2 + + - name: Run security regression tests + run: python3 -m pytest tests/security -v + + # ───────────────────────────────────────────────────────────────────── + # npm provenance + new install-script diff. Catches the two npm + # supply-chain levers we don't yet gate on: + # + # 1. `npm audit signatures` validates the registry-signed + # provenance of every tarball laid down in node_modules. Pulled + # from the public npm transparency log; surfaces unsigned or + # mis-signed deps. Informational for now (continue-on-error) + # while the baseline settles. + # + # 2. `check_new_install_scripts.py` diffs the PR's lockfile + # against the base ref and refuses any newly-added dep that + # ships a postinstall hook. Every recent npm supply-chain + # compromise leveraged a postinstall as the execution lever, so + # blocking new ones at PR time is a small, high-signal gate. + # ───────────────────────────────────────────────────────────────────── + npm-provenance-and-install-scripts: + name: npm provenance + new install-script diff + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Harden runner (egress block) + uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1 + with: + egress-policy: audit + disable-sudo: true + allowed-endpoints: > + api.github.com:443 + github.com:443 + codeload.github.com:443 + objects.githubusercontent.com:443 + registry.npmjs.org:443 + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # Need the base commit accessible for `git show + # :studio/frontend/package-lock.json` below. + fetch-depth: 0 + persist-credentials: false + + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '22' + cache: 'npm' + cache-dependency-path: studio/frontend/package-lock.json + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + + - name: Install Studio frontend deps (--ignore-scripts) + # `npm audit signatures` requires node_modules to be populated. + # `--ignore-scripts` is mandatory: this is exactly the lever the + # new-install-script gate below protects against, and we must + # not run any third-party hook to set up the audit. + working-directory: studio/frontend + run: npm ci --ignore-scripts + + - name: npm audit signatures (informational) + # Surfaces unsigned / mis-signed packages from the npm + # transparency log. continue-on-error during baseline-build + # phase; promote to hard gate once the lockfile is fully + # signed (most major maintainers signed by mid-2025). + working-directory: studio/frontend + continue-on-error: true + run: | + set -o pipefail + LOG=logs-audit-signatures.txt + npm audit signatures 2>&1 | tee "$LOG" + { + echo "## npm audit signatures" + echo + echo '```' + tail -200 "$LOG" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Extract base-ref lockfile (PR triggers only) + if: github.event_name == 'pull_request' + run: | + set -e + BASE_SHA="${{ github.event.pull_request.base.sha }}" + git show "$BASE_SHA:studio/frontend/package-lock.json" \ + > /tmp/base-package-lock.json + + - name: Diff for newly-added install-script deps + if: github.event_name == 'pull_request' + run: | + python3 scripts/check_new_install_scripts.py \ + --base /tmp/base-package-lock.json \ + --head studio/frontend/package-lock.json + + - name: Skip install-script diff (non-PR trigger) + if: github.event_name != 'pull_request' + run: | + echo "Not a pull_request event; install-script diff requires a base ref." + echo "This step is intentionally a no-op outside PR triggers." + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: npm-audit-signatures-log + path: studio/frontend/logs-audit-signatures.txt + if-no-files-found: ignore + retention-days: 30 diff --git a/.github/workflows/studio-api-smoke.yml b/.github/workflows/studio-api-smoke.yml index 742cba9ed9..29f056eca4 100644 --- a/.github/workflows/studio-api-smoke.yml +++ b/.github/workflows/studio-api-smoke.yml @@ -51,6 +51,8 @@ jobs: HF_HOME: ${{ github.workspace }}/hf-cache steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Linux deps run: | @@ -69,9 +71,10 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + - name: Restore HF_HOME for ${{ env.GGUF_REPO }} id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache # Same key as studio-ui-smoke.yml so the two jobs share a @@ -79,7 +82,8 @@ jobs: key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 - name: Prime HF_HOME with the GGUF - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -88,6 +92,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" + - name: Save HF_HOME for ${{ env.GGUF_REPO }} + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + - name: Install Studio (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/studio-backend-ci.yml b/.github/workflows/studio-backend-ci.yml index 59cd3a5685..63eb70f7f1 100644 --- a/.github/workflows/studio-backend-ci.yml +++ b/.github/workflows/studio-backend-ci.yml @@ -53,6 +53,8 @@ jobs: python: ['3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: @@ -106,6 +108,8 @@ jobs: timeout-minutes: 15 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: diff --git a/.github/workflows/studio-frontend-ci.yml b/.github/workflows/studio-frontend-ci.yml index eb00e297a7..a93cdb8661 100644 --- a/.github/workflows/studio-frontend-ci.yml +++ b/.github/workflows/studio-frontend-ci.yml @@ -36,6 +36,8 @@ jobs: working-directory: studio/frontend steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false # FIXME: drop this step once @assistant-ui/* and assistant-stream # leave 0.x -- on 1.x, caret ranges are conventional. Until then, @@ -58,7 +60,21 @@ jobs: cache: 'npm' cache-dependency-path: studio/frontend/package-lock.json + # Run the structural lockfile scan BEFORE npm ci. A compromised + # tarball runs its `prepare` / `postinstall` during `npm ci`, + # so any catch has to fire upstream of that. The scanner is + # pure-Python read-only; safe to call ahead of every install. + - name: Lockfile supply-chain audit (pre-install scan) + working-directory: ${{ github.workspace }} + run: python3 scripts/lockfile_supply_chain_audit.py + - name: Lockfile must agree with package.json (npm ci is strict) + # Lifecycle scripts (esbuild native-binary postinstall, etc.) are + # required for `vite build`. The pre-install lockfile structural + # audit (lockfile_supply_chain_audit.py) is the practical defence + # against the npm postinstall-dropper class -- it fires BEFORE any + # tarball runs, on the injection pattern itself rather than an + # advisory-DB lookup. run: npm ci --no-fund --no-audit - name: npm ci must not have modified the working tree diff --git a/.github/workflows/studio-inference-smoke.yml b/.github/workflows/studio-inference-smoke.yml index 19e1ab9cc5..ea14e4f5d5 100644 --- a/.github/workflows/studio-inference-smoke.yml +++ b/.github/workflows/studio-inference-smoke.yml @@ -67,6 +67,8 @@ jobs: HF_HOME: ${{ github.workspace }}/hf-cache steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Linux deps for llama.cpp prebuilt run: | @@ -85,15 +87,17 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + - name: Restore HF_HOME for ${{ env.GGUF_REPO }} id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 - name: Prime HF_HOME with the GGUF - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -102,6 +106,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" + - name: Save HF_HOME for ${{ env.GGUF_REPO }} + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + - name: Install Studio (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -306,6 +317,8 @@ jobs: STUDIO_PORT: '18889' steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Linux deps for llama.cpp prebuilt run: | @@ -324,15 +337,17 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Cache GGUF model file + - name: Restore GGUF model file id: cache-gguf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: gguf-cache key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 - name: Download GGUF if cache miss - if: steps.cache-gguf.outputs.cache-hit != 'true' + id: download-gguf + if: steps.cache-gguf.outputs.cache-hit != 'true' || steps.cache-gguf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -341,6 +356,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" --local-dir gguf-cache + - name: Save GGUF model file + if: always() && steps.download-gguf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: gguf-cache + key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 + - name: Install Studio (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -614,6 +636,8 @@ jobs: HF_HOME: ${{ github.workspace }}/hf-cache steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Linux deps for llama.cpp prebuilt run: | @@ -632,15 +656,17 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} (model + mmproj) + - name: Restore HF_HOME for ${{ env.GGUF_REPO }} (model + mmproj) id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-${{ env.MMPROJ_FILE }}-v1 - name: Prime HF_HOME with the GGUF + mmproj - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -651,6 +677,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$MMPROJ_FILE" + - name: Save HF_HOME for ${{ env.GGUF_REPO }} (model + mmproj) + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-${{ env.MMPROJ_FILE }}-v1 + - name: Install Studio (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/studio-mac-api-smoke.yml b/.github/workflows/studio-mac-api-smoke.yml index 98596f374a..aa7a616413 100644 --- a/.github/workflows/studio-mac-api-smoke.yml +++ b/.github/workflows/studio-mac-api-smoke.yml @@ -44,6 +44,8 @@ jobs: HF_HOME: ${{ github.workspace }}/hf-cache steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: @@ -56,15 +58,17 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + - name: Restore HF_HOME for ${{ env.GGUF_REPO }} id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 - name: Prime HF_HOME with the GGUF - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -73,6 +77,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" + - name: Save HF_HOME for ${{ env.GGUF_REPO }} + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + - name: Install Studio (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/studio-mac-inference-smoke.yml b/.github/workflows/studio-mac-inference-smoke.yml index 97efe3e74d..4e8456a297 100644 --- a/.github/workflows/studio-mac-inference-smoke.yml +++ b/.github/workflows/studio-mac-inference-smoke.yml @@ -4,8 +4,8 @@ # Three end-to-end smoke jobs that boot a freshly-installed Studio and # exercise the surfaces real users hit through the OpenAI / Anthropic # SDKs and curl. Each job picks the smallest model that exercises the -# behaviour under test, primes HF_HOME via actions/cache, and shares -# the install.sh --local --no-torch bootstrap. +# behaviour under test, primes a model cache via actions/cache, and +# shares the install.sh --local --no-torch bootstrap. # # 1. OpenAI, Anthropic API tests # gemma-3-270m-it UD-Q4_K_XL (~254 MiB). @@ -40,7 +40,7 @@ on: - '.github/workflows/studio-mac-inference-smoke.yml' push: branches: [main, pip] - # Manual trigger for pre-warming HF_HOME caches on main, or re-running + # Manual trigger for pre-warming model caches on main, or re-running # against an arbitrary branch without pushing a no-op commit. workflow_dispatch: @@ -67,6 +67,8 @@ jobs: HF_HOME: ${{ github.workspace }}/hf-cache steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: @@ -79,15 +81,17 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + - name: Restore HF_HOME for ${{ env.GGUF_REPO }} id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 - name: Prime HF_HOME with the GGUF - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -96,6 +100,15 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" + # Save partial caches on cancel/timeout -- hf download resumes by + # content hash. `outcome != skipped` keeps cache-hit a no-op. + - name: Save HF_HOME for ${{ env.GGUF_REPO }} + if: always() && steps.prime-hf.outcome != 'skipped' && hashFiles('hf-cache/**/*.gguf') != '' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + - name: Install Studio (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -306,6 +319,8 @@ jobs: STUDIO_PORT: '18898' steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: @@ -318,15 +333,17 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Cache GGUF model file + - name: Restore GGUF model file id: cache-gguf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: gguf-cache key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 - name: Download GGUF if cache miss - if: steps.cache-gguf.outputs.cache-hit != 'true' + id: download-gguf + if: steps.cache-gguf.outputs.cache-hit != 'true' || steps.cache-gguf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -335,6 +352,14 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" --local-dir gguf-cache + # Save partial caches on cancel; next run resumes via content hash. + - name: Save GGUF model file + if: always() && steps.download-gguf.outcome != 'skipped' && hashFiles('gguf-cache/**/*.gguf') != '' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: gguf-cache + key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 + - name: Install Studio (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -659,9 +684,10 @@ jobs: GGUF_FILE: gemma-4-E2B-it-UD-Q4_K_XL.gguf MMPROJ_FILE: mmproj-F16.gguf STUDIO_PORT: '18899' - HF_HOME: ${{ github.workspace }}/hf-cache steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: @@ -674,33 +700,47 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} (model + mmproj) - id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + # Cache flat .gguf + mmproj (Job 2's pattern). HF_HOME inflates + # ~3.6x via xet/blobs/snapshots, which made macOS saves never land. + # mmproj is auto-detected as a sibling via detect_mmproj_file + # (studio/backend/utils/models/model_config.py). + - name: Restore GGUF + mmproj files + id: cache-gguf + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: - path: hf-cache - key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-${{ env.MMPROJ_FILE }}-v1 + path: gguf-cache + key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-${{ env.MMPROJ_FILE }}-v1 - - name: Prime HF_HOME with the GGUF + mmproj - if: steps.cache-hf.outputs.cache-hit != 'true' + - name: Download GGUF + mmproj if cache miss + id: download-gguf + if: steps.cache-gguf.outputs.cache-hit != 'true' || steps.cache-gguf.outcome != 'success' # Authenticated + parallel: shared macos-14 NAT egress stalls # multi-GB anonymous downloads. env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | python -m pip install --upgrade huggingface_hub hf_transfer - mkdir -p hf-cache + mkdir -p gguf-cache HF_HUB_ENABLE_HF_TRANSFER=1 \ - hf download "$GGUF_REPO" "$GGUF_FILE" & + hf download "$GGUF_REPO" "$GGUF_FILE" --local-dir gguf-cache & MODEL_PID=$! HF_HUB_ENABLE_HF_TRANSFER=1 \ - hf download "$GGUF_REPO" "$MMPROJ_FILE" & + hf download "$GGUF_REPO" "$MMPROJ_FILE" --local-dir gguf-cache & MMPROJ_PID=$! wait "$MODEL_PID" wait "$MMPROJ_PID" # Fail loud on a partial download instead of in the next step. - find hf-cache -name "$GGUF_FILE" -o -name "$MMPROJ_FILE" \ - | xargs -I{} ls -lhL {} + ls -lh "gguf-cache/$GGUF_FILE" "gguf-cache/$MMPROJ_FILE" + + # Save partial caches on cancel. hashFiles guard avoids a hard + # save failure when the download step exits with no files. + - name: Save GGUF + mmproj files + if: always() && steps.download-gguf.outcome != 'skipped' && hashFiles('gguf-cache/**/*.gguf') != '' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: gguf-cache + key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-${{ env.MMPROJ_FILE }}-v1 - name: Install Studio (--local, --no-torch) env: @@ -755,12 +795,17 @@ jobs: -H 'content-type: application/json' \ -d "{\"username\":\"unsloth\",\"password\":\"$NEW\"}" | jq -r .access_token) echo "API_KEY=$TOKEN" >> "$GITHUB_ENV" - # Load the GGUF (mmproj is auto-detected via the HF repo - # lookup, the cached file is pulled out of HF_HOME). + # Load via local file path; mmproj sibling auto-detected by + # detect_mmproj_file (model_config.py). gguf_variant omitted + # -- it routes through _find_local_gguf_by_variant which + # expects a directory, not a file path. + GGUF_PATH="$GITHUB_WORKSPACE/gguf-cache/${GGUF_FILE}" + MMPROJ_PATH="$GITHUB_WORKSPACE/gguf-cache/${MMPROJ_FILE}" + ls -lh "$GGUF_PATH" "$MMPROJ_PATH" curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \ -H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \ --max-time 900 \ - -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048}" \ + -d "{\"model_path\":\"$GGUF_PATH\",\"is_lora\":false,\"max_seq_length\":2048}" \ | jq '{status, display_name, is_vision}' - name: JSON schema decoding + image input diff --git a/.github/workflows/studio-mac-ui-smoke.yml b/.github/workflows/studio-mac-ui-smoke.yml index c921ddf63e..28a9fc6d1d 100644 --- a/.github/workflows/studio-mac-ui-smoke.yml +++ b/.github/workflows/studio-mac-ui-smoke.yml @@ -44,6 +44,8 @@ jobs: HF_HOME: ${{ github.workspace }}/hf-cache steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: @@ -56,15 +58,17 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + - name: Restore HF_HOME for ${{ env.GGUF_REPO }} id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 - name: Prime HF_HOME with the GGUF - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -73,6 +77,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" + - name: Save HF_HOME for ${{ env.GGUF_REPO }} + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + - name: Install Studio (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/studio-mac-update-smoke.yml b/.github/workflows/studio-mac-update-smoke.yml index 2733fef1d1..dd2333251a 100644 --- a/.github/workflows/studio-mac-update-smoke.yml +++ b/.github/workflows/studio-mac-update-smoke.yml @@ -46,6 +46,8 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: diff --git a/.github/workflows/studio-tauri-smoke.yml b/.github/workflows/studio-tauri-smoke.yml index d517a5f454..159d5dbbe6 100644 --- a/.github/workflows/studio-tauri-smoke.yml +++ b/.github/workflows/studio-tauri-smoke.yml @@ -40,6 +40,8 @@ jobs: timeout-minutes: 25 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Linux native deps for Tauri / WebKit2GTK run: | @@ -56,12 +58,18 @@ jobs: - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @ 2026-03-27 - - uses: swatinem/rust-cache@23869a5bd66c73db3c0ac40331f3206eb23791dc # v2.9.1 + - uses: swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2.9.1 with: workspaces: studio/src-tauri -> target - name: Install pinned Tauri CLI (matches release-desktop.yml) - run: npm install --save-dev --prefix studio @tauri-apps/cli@2.10.1 + # Lifecycle scripts (esbuild native-binary postinstall, etc.) are + # required for `vite build`. The pre-install lockfile structural + # audit (lockfile_supply_chain_audit.py) is the practical defence + # against the npm postinstall-dropper class -- it fires BEFORE any + # tarball runs, on the injection pattern itself rather than an + # advisory-DB lookup. + run: npm install --save-dev --prefix studio @tauri-apps/cli@2.10.1 --no-fund --no-audit - name: Verify pinned Tauri CLI version run: | @@ -69,8 +77,17 @@ jobs: echo "$out" [ "$out" = "tauri-cli 2.10.1" ] || { echo "::error::expected tauri-cli 2.10.1, got $out"; exit 1; } + - name: Lockfile supply-chain audit (pre-install scan) + run: python3 scripts/lockfile_supply_chain_audit.py + - name: Frontend build (npm ci, vite) working-directory: studio/frontend + # Lifecycle scripts (esbuild native-binary postinstall, etc.) are + # required for `vite build`. The pre-install lockfile structural + # audit (lockfile_supply_chain_audit.py) is the practical defence + # against the npm postinstall-dropper class -- it fires BEFORE any + # tarball runs, on the injection pattern itself rather than an + # advisory-DB lookup. run: | npm ci --no-fund --no-audit npm run build diff --git a/.github/workflows/studio-ui-smoke.yml b/.github/workflows/studio-ui-smoke.yml index 756eea64b2..1f3a5a8594 100644 --- a/.github/workflows/studio-ui-smoke.yml +++ b/.github/workflows/studio-ui-smoke.yml @@ -52,6 +52,8 @@ jobs: HF_HOME: ${{ github.workspace }}/hf-cache steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Linux deps run: | @@ -70,15 +72,17 @@ jobs: python-version: '3.12' cache: 'pip' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + - name: Restore HF_HOME for ${{ env.GGUF_REPO }} id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 - name: Prime HF_HOME with the GGUF - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -87,6 +91,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" + - name: Save HF_HOME for ${{ env.GGUF_REPO }} + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + - name: Install Studio (--local, --no-torch) env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/studio-update-smoke.yml b/.github/workflows/studio-update-smoke.yml index 574b447a94..624001142a 100644 --- a/.github/workflows/studio-update-smoke.yml +++ b/.github/workflows/studio-update-smoke.yml @@ -40,6 +40,8 @@ jobs: timeout-minutes: 15 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Linux deps for llama.cpp prebuilt run: | diff --git a/.github/workflows/studio-windows-api-smoke.yml b/.github/workflows/studio-windows-api-smoke.yml index d9ed5d5594..86a07b41e5 100644 --- a/.github/workflows/studio-windows-api-smoke.yml +++ b/.github/workflows/studio-windows-api-smoke.yml @@ -52,6 +52,8 @@ jobs: PYTHONUTF8: '1' steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: @@ -63,15 +65,17 @@ jobs: with: python-version: '3.12' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + - name: Restore HF_HOME for ${{ env.GGUF_REPO }} id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 - name: Prime HF_HOME with the GGUF - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -80,6 +84,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" + - name: Save HF_HOME for ${{ env.GGUF_REPO }} + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + - name: Pre-install Windows tweaks (npm 11 + Defender exclusions) shell: pwsh # See studio-windows-update-smoke.yml for the full rationale. diff --git a/.github/workflows/studio-windows-inference-smoke.yml b/.github/workflows/studio-windows-inference-smoke.yml index b33b6f9563..bc13ec8199 100644 --- a/.github/workflows/studio-windows-inference-smoke.yml +++ b/.github/workflows/studio-windows-inference-smoke.yml @@ -62,6 +62,8 @@ jobs: PYTHONUTF8: '1' steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: @@ -73,15 +75,29 @@ jobs: with: python-version: '3.12' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + # Split restore + save (rather than the one-step actions/cache) so a + # transient restore-side failure does not kill the whole job. v5 has a + # known flake where it logs "Cache hit for: " and then exits + # non-zero without actually extracting the archive (see + # actions/cache#1621 and github community discussion #163260). + # continue-on-error on restore masks that failure so the Prime step + # below can re-download from HF and the job keeps running. Save then + # populates the cache key on a real miss only; cache keys are + # immutable, so a corrupted cached entry persists until the -v1 + # suffix below is bumped. + - name: Restore HF_HOME cache for ${{ env.GGUF_REPO }} id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 - name: Prime HF_HOME with the GGUF - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + # Run on a real cache miss AND on the silent-restore-failure mode + # described above (outcome != success). + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -90,6 +106,16 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" + - name: Save HF_HOME cache for ${{ env.GGUF_REPO }} + # Only write a fresh cache entry when we actually rebuilt the + # directory (Prime ran and succeeded). Skipping when Prime is + # skipped avoids "already exists" save warnings on the happy path. + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + - name: Pre-install Windows tweaks (npm 11 + Defender exclusions) shell: pwsh # See studio-windows-update-smoke.yml for the full rationale. @@ -364,6 +390,8 @@ jobs: PYTHONUTF8: '1' steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: @@ -375,15 +403,20 @@ jobs: with: python-version: '3.12' - - name: Cache GGUF model file + # Split restore + save so a transient restore-side failure does not + # kill the whole job. See the matching block in the tool-calling job + # above for the full rationale (actions/cache#1621). + - name: Restore GGUF model cache id: cache-gguf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: gguf-cache key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 - name: Download GGUF if cache miss - if: steps.cache-gguf.outputs.cache-hit != 'true' + id: download-gguf + if: steps.cache-gguf.outputs.cache-hit != 'true' || steps.cache-gguf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -392,6 +425,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" --local-dir gguf-cache + - name: Save GGUF model cache + if: always() && steps.download-gguf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: gguf-cache + key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 + - name: Pre-install Windows tweaks (npm 11 + Defender exclusions) shell: pwsh # See studio-windows-update-smoke.yml for the full rationale. @@ -760,6 +800,8 @@ jobs: PYTHONUTF8: '1' steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: @@ -771,15 +813,23 @@ jobs: with: python-version: '3.12' - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} (model + mmproj) + # Split restore + save so a transient restore-side failure does not + # kill the whole job. See the matching block in the tool-calling job + # for the full rationale (actions/cache#1621). This is the block that + # actually broke in run 25713577488: "Cache hit for: " was + # logged, the step exited non-zero in ~0.3 s without extracting the + # 3.4 GiB archive, and steps 6-15 were skipped. + - name: Restore HF_HOME cache for ${{ env.GGUF_REPO }} (model + mmproj) id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-${{ env.MMPROJ_FILE }}-v1 - name: Prime HF_HOME with the GGUF + mmproj - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -790,6 +840,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$MMPROJ_FILE" + - name: Save HF_HOME cache for ${{ env.GGUF_REPO }} (model + mmproj) + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-${{ env.MMPROJ_FILE }}-v1 + - name: Pre-install Windows tweaks (npm 11 + Defender exclusions) shell: pwsh # See studio-windows-update-smoke.yml for the full rationale. diff --git a/.github/workflows/studio-windows-ui-smoke.yml b/.github/workflows/studio-windows-ui-smoke.yml index a5a4753ba5..90fce0558b 100644 --- a/.github/workflows/studio-windows-ui-smoke.yml +++ b/.github/workflows/studio-windows-ui-smoke.yml @@ -57,6 +57,8 @@ jobs: PYTHONUTF8: '1' steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: @@ -72,15 +74,17 @@ jobs: # then fatal-errors with "Cache folder path is retrieved # for pip but doesn't exist on disk". - - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + - name: Restore HF_HOME for ${{ env.GGUF_REPO }} id: cache-hf - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true with: path: hf-cache key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 - name: Prime HF_HOME with the GGUF - if: steps.cache-hf.outputs.cache-hit != 'true' + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | @@ -89,6 +93,13 @@ jobs: HF_HUB_ENABLE_HF_TRANSFER=1 \ hf download "$GGUF_REPO" "$GGUF_FILE" + - name: Save HF_HOME for ${{ env.GGUF_REPO }} + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + - name: Pre-install Windows tweaks (npm 11 + Defender exclusions) shell: pwsh # See studio-windows-update-smoke.yml for the full rationale. diff --git a/.github/workflows/studio-windows-update-smoke.yml b/.github/workflows/studio-windows-update-smoke.yml index c16edc5aff..0303bc746d 100644 --- a/.github/workflows/studio-windows-update-smoke.yml +++ b/.github/workflows/studio-windows-update-smoke.yml @@ -58,6 +58,8 @@ jobs: PYTHONUTF8: '1' steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: diff --git a/.github/workflows/version-compat-ci.yml b/.github/workflows/version-compat-ci.yml index ff3218bba0..1ebea81066 100644 --- a/.github/workflows/version-compat-ci.yml +++ b/.github/workflows/version-compat-ci.yml @@ -58,6 +58,8 @@ jobs: timeout-minutes: 12 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.12' @@ -83,6 +85,8 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.12' @@ -107,6 +111,8 @@ jobs: timeout-minutes: 8 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.12' @@ -129,6 +135,8 @@ jobs: timeout-minutes: 8 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.12' @@ -151,6 +159,8 @@ jobs: timeout-minutes: 8 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.12' @@ -173,6 +183,8 @@ jobs: timeout-minutes: 12 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.12' @@ -200,11 +212,27 @@ jobs: timeout-minutes: 15 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false with: { path: unsloth } - name: Clone unsloth-zoo @ main run: | - git clone --depth=1 https://github.com/unslothai/unsloth-zoo \ - "$RUNNER_TEMP/unsloth-zoo" + # github.com occasionally 500s on the git fetch; retry so a + # single upstream blip does not fail CI. + for attempt in 1 2 3; do + rm -rf "$RUNNER_TEMP/unsloth-zoo" + if git clone --depth=1 https://github.com/unslothai/unsloth-zoo \ + "$RUNNER_TEMP/unsloth-zoo"; then + break + fi + if [ "$attempt" -eq 3 ]; then + echo "::error::git clone unsloth-zoo failed after 3 attempts" + exit 1 + fi + delay=$((5 * attempt)) + echo "::warning::clone failed (attempt $attempt/3), retrying in ${delay}s..." + sleep "$delay" + done - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.12' @@ -265,6 +293,8 @@ jobs: timeout-minutes: 20 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.12' diff --git a/.github/workflows/wheel-smoke.yml b/.github/workflows/wheel-smoke.yml index 983070ae13..464a8e324a 100644 --- a/.github/workflows/wheel-smoke.yml +++ b/.github/workflows/wheel-smoke.yml @@ -42,6 +42,8 @@ jobs: timeout-minutes: 15 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: @@ -53,7 +55,16 @@ jobs: with: python-version: '3.12' + - name: Lockfile supply-chain audit (pre-install scan) + run: python3 scripts/lockfile_supply_chain_audit.py + - name: Build frontend + # Lifecycle scripts (esbuild native-binary postinstall, etc.) are + # required for `vite build`. The pre-install lockfile structural + # audit (lockfile_supply_chain_audit.py) is the practical defence + # against the npm postinstall-dropper class -- it fires BEFORE any + # tarball runs, on the injection pattern itself rather than an + # advisory-DB lookup. run: | cd studio/frontend npm ci --no-fund --no-audit diff --git a/.semgrep/unsloth-rules.yml b/.semgrep/unsloth-rules.yml deleted file mode 100644 index 654ff9a490..0000000000 --- a/.semgrep/unsloth-rules.yml +++ /dev/null @@ -1,183 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -# -# Custom Semgrep rules for unsloth + studio backend. The off-the-shelf -# rule packs (p/python, p/javascript, p/supply-chain, p/security-audit) -# wired into the security-audit workflow already cover the common -# patterns. These rules add catches for the *specific* shape of recent -# CVEs in the broader Python ML / dev-tools stack -- so if we ever -# introduce a similar bug ourselves, CI lights up. -# -# Run locally: -# pip install 'semgrep>=1.95' -# semgrep --config .semgrep/unsloth-rules.yml studio/backend unsloth scripts -# -# Wired into CI via .github/workflows/security-audit.yml's Semgrep step. - -rules: - # ───────────────────────────────────────────────────────────────── - # langchain-core CVE-2025-68664 shape: - # `dumps()` / `dumpd()` over a user-controlled dict that may carry - # the `lc` marker key -> deserialization injection on the round - # trip. Catch any json.dumps / pickle.dumps / yaml.dump on data - # that flowed through a Request/WebSocket payload. - # ───────────────────────────────────────────────────────────────── - - id: unsloth-deserialize-roundtrip - message: >- - Serializing user-controlled data with langchain-style `dumps` - can re-instantiate arbitrary classes when deserialized. See - langchain-core CVE-2025-68664. Sanitize / strip `lc` marker keys - before dumping, or use a strict schema (Pydantic) instead. - severity: WARNING - languages: [python] - patterns: - - pattern-either: - - pattern: langchain_core.load.dumps($DATA, ...) - - pattern: langchain_core.load.dumpd($DATA, ...) - - pattern: dumps($DATA) - - pattern: dumpd($DATA) - - metavariable-pattern: - metavariable: $DATA - patterns: - - pattern-either: - - pattern: request.$F - - pattern: payload - - pattern: body - - pattern: data - - pattern: input - - # ───────────────────────────────────────────────────────────────── - # n8n CVE-2025-68668 shape: - # `_pyodide._base.eval_code(...)` or any private/underscore call - # into pyodide internals that escapes the public sandbox API. - # ───────────────────────────────────────────────────────────────── - - id: unsloth-pyodide-private-eval - message: >- - Calling `_pyodide._base.eval_code` (or any `_pyodide.`) - bypasses the public Pyodide sandbox -- this is how n8n - CVE-2025-68668 (CVSS 9.9) escaped the Code Node's blocklist. - Use the documented sandbox API (`pyodide.runPython`) and rely - on web-worker isolation for untrusted input. - severity: ERROR - languages: [python, javascript, typescript] - patterns: - - pattern-either: - - pattern: _pyodide._base.eval_code(...) - - pattern: $X._pyodide.$Y(...) - - # ───────────────────────────────────────────────────────────────── - # marimo CVE-2026-39987 shape: - # FastAPI / Starlette WebSocket route that accepts connections - # without checking auth -- in marimo this dropped a PTY shell to - # any unauthenticated attacker. - # ───────────────────────────────────────────────────────────────── - - id: unsloth-websocket-no-auth - message: >- - WebSocket route accepts connections without an auth check. - marimo CVE-2026-39987 was a pre-auth WebSocket on - `/terminal/ws` that handed a full PTY shell to any - unauthenticated peer. Add a Depends(get_current_user) / - `await websocket.headers.get("authorization")` gate before - `await websocket.accept()`. - severity: WARNING - languages: [python] - patterns: - - pattern: | - @$APP.websocket("...") - async def $F(websocket: WebSocket, ...): - ... - await websocket.accept() - ... - - pattern-not-inside: | - @$APP.websocket("...") - async def $F(websocket: WebSocket, ..., $USER = Depends(...)): - ... - - pattern-not-inside: | - @$APP.websocket("...") - async def $F(websocket: WebSocket, ...): - ... - if not $AUTH: - ... - await websocket.accept() - - # ───────────────────────────────────────────────────────────────── - # litellm 1.82.7 shape: - # `subprocess.Popen` of a child Python interpreter that reads - # stdin from a network response (the C2-fetch-then-exec dropper - # pattern). Catches both `Popen([sys.executable, ...], stdin=...)` - # and `Popen("python ...", stdin=...)` variants. - # ───────────────────────────────────────────────────────────────── - - id: unsloth-popen-network-stdin - message: >- - Spawning a Python interpreter that reads its program from a - network call is the canonical fetch-and-exec dropper (litellm - 1.82.7 used this exact shape). Almost never legitimate inside a - package's import path. - severity: ERROR - languages: [python] - pattern-either: - - pattern: | - subprocess.Popen([..., $PY, ...], stdin=$NET, ...) - - pattern: | - subprocess.run([..., $PY, ...], input=$NET, ...) - - # ───────────────────────────────────────────────────────────────── - # Shai-Hulud / ForceMemo shape: - # programmatic write of a `.github/workflows/*.yml` file from - # inside our own Python source. We never write workflows - # programmatically; if a contributor ever does, they're probably - # re-implementing the worm pattern. - # ───────────────────────────────────────────────────────────────── - - id: unsloth-write-github-workflow - message: >- - Code that programmatically writes into `.github/workflows/` - from within unsloth itself is the Shai-Hulud / ForceMemo - self-propagation pattern. If you legitimately need a workflow - template, ship it under examples/ or templates/ instead. - severity: ERROR - languages: [python] - patterns: - - pattern-either: - - pattern: open("$P", ...) - - pattern: Path("$P").write_text(...) - - pattern: open("$P", "w", ...) - - metavariable-regex: - metavariable: $P - regex: \.github/workflows/.*\.ya?ml - - # ───────────────────────────────────────────────────────────────── - # Pickle-from-network shape: classic deserialization sink that - # several recent ML pipeline CVEs hit (mlflow, pyzmq, ray serve). - # ───────────────────────────────────────────────────────────────── - - id: unsloth-pickle-from-network - message: >- - `pickle.loads` on bytes that flowed from a network response is - arbitrary code execution. Use `safetensors` or a strict - schema (Pydantic / msgspec) instead. ML frameworks have shipped - multiple CVEs of this exact shape (mlflow, ray serve, pyzmq). - severity: ERROR - languages: [python] - pattern-either: - - pattern: pickle.loads($X.content) - - pattern: pickle.loads($X.text.encode(...)) - - pattern: pickle.loads(requests.get(...).content) - - pattern: pickle.load(urllib.request.urlopen(...)) - - # ───────────────────────────────────────────────────────────────── - # Subprocess shell=True with f-string / format / concat -- command - # injection if any interpolated value comes from user input. - # ───────────────────────────────────────────────────────────────── - - id: unsloth-shell-true-interpolation - message: >- - `subprocess` call with `shell=True` and an interpolated command - string is command injection if any input is user-controlled. - Pass argv list instead, or use shlex.quote on each part. - severity: WARNING - languages: [python] - pattern-either: - - pattern: subprocess.run(f"...", shell=True, ...) - - pattern: subprocess.Popen(f"...", shell=True, ...) - - pattern: subprocess.call(f"...", shell=True, ...) - - pattern: os.system(f"...") - - pattern: subprocess.run("..." + $X, shell=True, ...) - - pattern: subprocess.run("...{}...".format(...), shell=True, ...) diff --git a/build.sh b/build.sh index cf8aa02910..1558dca240 100644 --- a/build.sh +++ b/build.sh @@ -2,6 +2,10 @@ set -euo pipefail +# PyPI/Studio release publishing must use `./build.sh publish` (or an +# equivalent stamp -> build -> verify-dist -> upload flow) so packaged Studio +# artifacts include the display-only Studio release version. + # 1. Build frontend (Vite outputs to dist/) cd studio/frontend @@ -70,10 +74,33 @@ cd ../.. # 2. Clean old artifacts rm -rf build dist *.egg-info -# 3. Build wheel +# 3. Stamp display-only Studio release metadata for packaged builds. +_STUDIO_BUILD_INFO="studio/backend/utils/_studio_release_build.py" +_STUDIO_BUILD_INFO_BACKUP="$(mktemp)" +cp "$_STUDIO_BUILD_INFO" "$_STUDIO_BUILD_INFO_BACKUP" +_restore_studio_build_info() { + cp "$_STUDIO_BUILD_INFO_BACKUP" "$_STUDIO_BUILD_INFO" 2>/dev/null || true + rm -f "$_STUDIO_BUILD_INFO_BACKUP" +} +trap _restore_studio_build_info EXIT + +if [ "${1:-}" = "publish" ]; then + STUDIO_STAMPED_VERSION="$(python scripts/stamp_studio_release.py --require-release)" +else + STUDIO_STAMPED_VERSION="$(python scripts/stamp_studio_release.py)" +fi + +# 4. Build wheel/sdist python -m build -# 4. Optionally publish +if [ "${1:-}" = "publish" ]; then + python scripts/stamp_studio_release.py --verify-dist dist --expected "$STUDIO_STAMPED_VERSION" +fi + +_restore_studio_build_info +trap - EXIT + +# 5. Optionally publish if [ "${1:-}" = "publish" ]; then python -m twine upload dist/* fi diff --git a/pyproject.toml b/pyproject.toml index 5687ea12f8..c66cb870eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1185,3 +1185,10 @@ ignore = [ ] [tool.ruff.format] + +[tool.pytest.ini_options] +# Narrow the default test discovery so `pytest` from the repo root +# does NOT pick up the GPU-heavy tests under tests/python, tests/qlora, +# etc. The CI security job runs `pytest tests/security` explicitly. +testpaths = ["tests/security"] +pythonpath = ["."] diff --git a/scripts/check_new_install_scripts.py b/scripts/check_new_install_scripts.py new file mode 100644 index 0000000000..af3c84f96d --- /dev/null +++ b/scripts/check_new_install_scripts.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Diff two `package-lock.json` files and flag NEW install-script deps. + +A package with `"hasInstallScript": true` runs `preinstall` / `install` / +`postinstall` lifecycle hooks every time `npm ci` lays it down. Every +npm supply-chain compromise of the last 18 months (Shai-Hulud, +TanStack, axios-style, ArmorCode hijacks) leveraged exactly this lever: +the attacker publishes a new malicious version of a dep we already +trust, and the post-install hook runs the next time CI installs. + +This scanner refuses to allow a newly-introduced install-script dep to +land without a maintainer eyeball on the lifecycle script body. +Existing install-script deps are NOT re-flagged -- if `node-gyp` has +been in the lockfile since day one, it's not part of this PR's threat +model. Only new entries are surfaced. + +Supports lockfileVersion 1 (`dependencies` key, recursive), 2 and 3 +(flat `packages` key with `node_modules//node_modules/` nesting +for transitive entries). For each NEW install-script package we +attempt a stdlib-only fetch of +`https://registry.npmjs.org//` to recover the actual +postinstall command body. If the network is blocked we still emit the +finding -- the lifecycle command body is informational, not +load-bearing. + +Exit codes +========== + 0 no newly-added install-script deps + 1 one or more newly-added install-script deps; listed on stderr + 2 internal error (missing lockfile, malformed JSON, etc.) +""" + +from __future__ import annotations + +import argparse +import json +import sys +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + +REGISTRY_BASE = "https://registry.npmjs.org/" +REGISTRY_TIMEOUT_SECS = 5 + +CRITICAL = "CRITICAL" +HIGH = "HIGH" + + +class Finding: + __slots__ = ("severity", "name", "version", "kind", "detail") + + def __init__( + self, severity: str, name: str, version: str, kind: str, detail: str + ) -> None: + self.severity = severity + self.name = name + self.version = version + self.kind = kind + self.detail = detail + + def __str__(self) -> str: + return ( + f" [{self.severity}] {self.name}@{self.version}\n" + f" kind: {self.kind}\n" + f" detail: {self.detail}" + ) + + +# ───────────────────────────────────────────────────────────────────── +# Lockfile parsing. +# ───────────────────────────────────────────────────────────────────── + + +def _strip_nm_prefix(key: str) -> str: + """Convert a v2/v3 `packages` key into a bare package name. + + `node_modules/foo` -> `foo`; `node_modules/foo/node_modules/bar` -> + `bar`. The empty key (`""`) is the project root and returns "". + """ + if not key: + return "" + # Use the LAST `node_modules/` segment so transitives map to their + # leaf name, matching how npm install resolves a postinstall. + marker = "node_modules/" + idx = key.rfind(marker) + if idx == -1: + return key + return key[idx + len(marker) :] + + +def _collect_install_script_entries(lock: dict) -> dict[str, str]: + """Walk a parsed lockfile and return {package_name: version} for + every entry with `hasInstallScript: true` (v2/v3) OR a + non-empty `scripts.preinstall|install|postinstall` (v1). + + The same package may appear at multiple versions in a single + lockfile (de-duplicated copies under different parents); we key by + `name@version` so we don't lose either copy. Returns a dict keyed + by `name@version` -> the same string for convenience. + """ + seen: dict[str, str] = {} + version = lock.get("lockfileVersion") + + # v2 / v3: flat `packages` map. + packages = lock.get("packages") or {} + for key, entry in packages.items(): + if key == "" or not isinstance(entry, dict): + continue + if entry.get("link"): + continue + if not entry.get("hasInstallScript"): + continue + name = _strip_nm_prefix(key) + if not name: + continue + ver = entry.get("version") or "" + seen[f"{name}@{ver}"] = name + + # v1 also embeds a `dependencies` tree; v2/v3 carry both for + # backwards-compat but `packages` is canonical for them. For v1 + # there is no `hasInstallScript` flag, so look for a non-empty + # `scripts.preinstall|install|postinstall` directly. + def _walk_v1(deps: dict, depth: int = 0) -> None: + if depth > 64 or not isinstance(deps, dict): + return + for name, entry in deps.items(): + if not isinstance(entry, dict): + continue + scripts = entry.get("scripts") or {} + lifecycle = any( + isinstance(scripts, dict) and scripts.get(hook) + for hook in ("preinstall", "install", "postinstall") + ) + # v1 also sets `requires` only on the parent, no flag, so + # the lifecycle-script presence is the only signal. + if lifecycle: + ver = entry.get("version") or "" + seen[f"{name}@{ver}"] = name + _walk_v1(entry.get("dependencies"), depth = depth + 1) + + if version == 1 or "dependencies" in lock: + _walk_v1(lock.get("dependencies") or {}) + + return seen + + +def _load_lockfile(path: Path) -> dict: + if not path.exists(): + raise FileNotFoundError(f"lockfile not found: {path}") + try: + return json.loads(path.read_text(encoding = "utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"{path}: not valid JSON: {exc}") from exc + + +# ───────────────────────────────────────────────────────────────────── +# Registry lookup for the postinstall command body (best-effort). +# ───────────────────────────────────────────────────────────────────── + + +def _fetch_registry_scripts(name: str, version: str) -> dict[str, str] | None: + """Return {hook: command} for any of preinstall / install / + postinstall published in the registry metadata for this name@ver. + + Returns None on any error (network blocked, 404, malformed JSON). + Never raises; the caller treats absence as "could not enrich, emit + finding anyway". + """ + safe_name = urllib.parse.quote(name, safe = "@/") + url = f"{REGISTRY_BASE}{safe_name}/{urllib.parse.quote(version)}" + try: + with urllib.request.urlopen(url, timeout = REGISTRY_TIMEOUT_SECS) as resp: + body = resp.read() + except (urllib.error.URLError, OSError, ValueError, TimeoutError): + return None + try: + meta = json.loads(body) + except json.JSONDecodeError: + return None + scripts = meta.get("scripts") or {} + if not isinstance(scripts, dict): + return None + keep = {} + for hook in ("preinstall", "install", "postinstall"): + cmd = scripts.get(hook) + if isinstance(cmd, str) and cmd.strip(): + keep[hook] = cmd + return keep or None + + +# ───────────────────────────────────────────────────────────────────── +# Diff. +# ───────────────────────────────────────────────────────────────────── + + +def diff_new_install_scripts(base_lock: dict, head_lock: dict) -> list[Finding]: + base = _collect_install_script_entries(base_lock) + head = _collect_install_script_entries(head_lock) + findings: list[Finding] = [] + for key in sorted(head): + if key in base: + continue # pre-existing install-script dep; not in scope + name = head[key] + # key is "name@version"; rsplit("@", 1) handles scoped names. + version = ( + key[len(name) + 1 :] if key.startswith(name + "@") else "" + ) + scripts = _fetch_registry_scripts(name, version) + if scripts: + detail = "; ".join(f"{h}={cmd!r}" for h, cmd in scripts.items()) + else: + detail = ( + "newly added with hasInstallScript=true; registry " + "metadata unreachable -- inspect the package's " + "scripts.{preinstall,install,postinstall} manually" + ) + findings.append( + Finding( + severity = CRITICAL, + name = name, + version = version, + kind = "new-install-script", + detail = detail, + ) + ) + return findings + + +# ───────────────────────────────────────────────────────────────────── +# CLI. +# ───────────────────────────────────────────────────────────────────── + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description = ( + "Diff two package-lock.json files and refuse any newly-" + "added install-script dep." + ), + ) + parser.add_argument( + "--base", + required = True, + help = "Path to the BASE package-lock.json (e.g. main branch).", + ) + parser.add_argument( + "--head", + required = True, + help = "Path to the HEAD package-lock.json (this PR).", + ) + args = parser.parse_args(argv) + + try: + base_lock = _load_lockfile(Path(args.base)) + head_lock = _load_lockfile(Path(args.head)) + except (FileNotFoundError, ValueError) as exc: + print(f"[install-script-diff] ERROR: {exc}", file = sys.stderr) + return 2 + + findings = diff_new_install_scripts(base_lock, head_lock) + if not findings: + print( + "[install-script-diff] OK: no newly-added install-script " + "dependencies between base and head", + flush = True, + ) + return 0 + + print( + f"\n[install-script-diff] FAIL: {len(findings)} newly-added " + f"install-script dependency(ies):\n", + file = sys.stderr, + ) + for f in findings: + print(str(f), file = sys.stderr) + print(file = sys.stderr) + print( + "[install-script-diff] Refusing to proceed. Every new " + "install-script dep is a postinstall lifecycle hook that " + "would run on the next `npm ci`. Review each finding above, " + "confirm the maintainer + version, and re-run.", + file = sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/enforce_kwargs_spacing.py b/scripts/enforce_kwargs_spacing.py index ca2ff343a0..6b36231610 100755 --- a/scripts/enforce_kwargs_spacing.py +++ b/scripts/enforce_kwargs_spacing.py @@ -6,12 +6,38 @@ from __future__ import annotations import ast import argparse import io +import os import sys +import tempfile import tokenize from collections import defaultdict from pathlib import Path +def _atomic_write_text(path: Path, data: str, encoding: str) -> None: + """Write ``data`` to ``path`` atomically. + + Stages a tmp file in the same directory (so it's on the same + filesystem as the destination), fsyncs, then `os.replace`s into + place. A crash mid-write therefore leaves either the previous + content or the fully new content -- never a truncated source file. + """ + dirpath = str(path.parent) or "." + fd, tmp_path = tempfile.mkstemp(prefix=".kwargs_fix.", dir=dirpath) + try: + with os.fdopen(fd, "w", encoding=encoding) as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp_path, path) + except Exception: + try: + os.unlink(tmp_path) + except OSError: + pass + raise + + def enforce_spacing(text: str) -> tuple[str, bool]: """Return updated text with keyword '=' padded by spaces, plus change flag.""" lines = text.splitlines(keepends=True) @@ -146,7 +172,7 @@ def process_file(path: Path) -> bool: updated, changed = enforce_spacing(original) updated, removed = remove_redundant_passes(updated) if changed or removed: - path.write_text(updated, encoding=encoding) + _atomic_write_text(path, updated, encoding) return True return False diff --git a/scripts/install_gemma4_mlx.sh b/scripts/install_gemma4_mlx.sh index 26415735b8..e1f43b827c 100755 --- a/scripts/install_gemma4_mlx.sh +++ b/scripts/install_gemma4_mlx.sh @@ -1,9 +1,15 @@ #!/bin/bash -set -e +set -euo pipefail # ============================================================ # Gemma 4 MLX — One-command setup + inference # +# Supply-chain hardening: the uv installer payload is pinned by +# SHA-256. Rotate by running: +# curl -sSLf https://astral.sh/uv/install.sh | shasum -a 256 +# and updating _UV_INSTALLER_SHA256 below. +# ============================================================ +# # Usage: # bash install_gemma4_mlx.sh [--venv-dir DIR] # @@ -104,10 +110,17 @@ else fi # ── Install uv ─────────────────────────────────────────────── +_UV_INSTALLER_SHA256="48cd5aca5d5671a3b3d5f61538cc8622e4434af63319115159990d8b0dd02416" + if ! command -v uv >/dev/null 2>&1; then step "uv" "installing uv package manager..." _uv_tmp=$(mktemp) curl -LsSf "https://astral.sh/uv/install.sh" -o "$_uv_tmp" + _uv_actual=$(shasum -a 256 "$_uv_tmp" | awk '{print $1}') + if [ "$_uv_actual" != "$_UV_INSTALLER_SHA256" ]; then + rm -f "$_uv_tmp" + fail "uv installer SHA-256 mismatch: got $_uv_actual expected $_UV_INSTALLER_SHA256 (refusing to execute)" + fi sh "$_uv_tmp" /dev/null 2>&1 rm -f "$_uv_tmp" if [ -f "$HOME/.local/bin/env" ]; then diff --git a/scripts/install_qwen3_6_mlx.sh b/scripts/install_qwen3_6_mlx.sh index 5ce66d29a6..38fe1bea05 100644 --- a/scripts/install_qwen3_6_mlx.sh +++ b/scripts/install_qwen3_6_mlx.sh @@ -1,9 +1,18 @@ #!/bin/bash -set -e +set -euo pipefail # ============================================================ # Qwen3.6 MLX — One-command setup + inference # +# Supply-chain hardening: +# - All third-party downloads (uv installer, mlx_vlm qwen3_5 +# patches) are pinned to an immutable git commit SHA and verified +# against a hardcoded SHA-256. Any mismatch aborts the install +# before the bytes are copied into site-packages. +# - To rotate any pin, fetch the new file with `curl`, run +# `shasum -a 256`, and update the corresponding constant below. +# ============================================================ +# # Usage: # bash install_qwen3_6_mlx.sh [--venv-dir DIR] # @@ -104,10 +113,21 @@ else fi # ── Install uv ─────────────────────────────────────────────── +# Pin the uv installer payload by SHA-256. Rotate by running: +# curl -sSLf https://astral.sh/uv/install.sh | shasum -a 256 +# and updating the constant below. We fetch into a temp file, verify +# the digest, and only then execute. Mismatch aborts. +_UV_INSTALLER_SHA256="48cd5aca5d5671a3b3d5f61538cc8622e4434af63319115159990d8b0dd02416" + if ! command -v uv >/dev/null 2>&1; then step "uv" "installing uv package manager..." _uv_tmp=$(mktemp) curl -LsSf "https://astral.sh/uv/install.sh" -o "$_uv_tmp" + _uv_actual=$(shasum -a 256 "$_uv_tmp" | awk '{print $1}') + if [ "$_uv_actual" != "$_UV_INSTALLER_SHA256" ]; then + rm -f "$_uv_tmp" + fail "uv installer SHA-256 mismatch: got $_uv_actual expected $_UV_INSTALLER_SHA256 (refusing to execute)" + fi sh "$_uv_tmp" +# curl -sSLf "https://raw.githubusercontent.com/unslothai/unsloth/$_PATCH_COMMIT/unsloth/models/patches/mlx_vlm_qwen3_5/qwen3_5.py" | shasum -a 256 +# curl -sSLf "https://raw.githubusercontent.com/unslothai/unsloth/$_PATCH_COMMIT/unsloth/models/patches/mlx_vlm_qwen3_5/generate.py" | shasum -a 256 +_PATCH_COMMIT="013c99e51bbb8c4b83d88f3b150a1e53251a19d2" +_PATCH_BASE="https://raw.githubusercontent.com/unslothai/unsloth/${_PATCH_COMMIT}/unsloth/models/patches/mlx_vlm_qwen3_5" +_PATCH_SHA_QWEN35="4b6fbbcc59b1d6b935e7204351aae1476836d25542a11c7885402b672d2efa64" +_PATCH_SHA_GENERATE="50c4cbb8c3d94c0c74a4d209db6d2b23b102944c147c6421f2eded427b8edaf7" + _SITE_PKGS=$("$_VENV_PY" -c "import site; print(site.getsitepackages()[0])") step "patch" "fixing multi-turn image chat..." -if curl -sSLf "${_PATCH_BASE}/qwen3_5.py" -o "${_SITE_PKGS}/mlx_vlm/models/qwen3_5/qwen3_5.py"; then +# Stage all downloads in an isolated tmpdir; we only copy into +# site-packages after every checksum has matched. +_PATCH_TMP=$(mktemp -d) +trap 'rm -rf "$_PATCH_TMP"' EXIT + +apply_pinned_patch() { + # apply_pinned_patch + _name="$1"; _expected="$2"; _dest="$3" + _staged="$_PATCH_TMP/$_name" + if ! curl -sSLf "${_PATCH_BASE}/${_name}" -o "$_staged"; then + step "warning" "failed to download ${_name} patch — multi-turn image chat may not work" "$C_WARN" + return 1 + fi + _actual=$(shasum -a 256 "$_staged" | awk '{print $1}') + if [ "$_actual" != "$_expected" ]; then + step "warning" "${_name} SHA-256 mismatch (got $_actual expected $_expected) — refusing to install patch" "$C_WARN" + return 1 + fi + mkdir -p "$(dirname "$_dest")" + cp "$_staged" "$_dest" + return 0 +} + +if apply_pinned_patch "qwen3_5.py" "$_PATCH_SHA_QWEN35" "${_SITE_PKGS}/mlx_vlm/models/qwen3_5/qwen3_5.py"; then substep "patched qwen3_5.py (MRoPE position reset)" -else - step "warning" "failed to download qwen3_5.py patch — multi-turn image chat may not work" "$C_WARN" fi -if curl -sSLf "${_PATCH_BASE}/generate.py" -o "${_SITE_PKGS}/mlx_vlm/generate.py"; then +if apply_pinned_patch "generate.py" "$_PATCH_SHA_GENERATE" "${_SITE_PKGS}/mlx_vlm/generate.py"; then substep "patched generate.py (mask trim on cache reuse)" -else - step "warning" "failed to download generate.py patch — multi-turn image chat may not work" "$C_WARN" fi # Clear pycache so patches take effect diff --git a/scripts/lint_workflow_triggers.py b/scripts/lint_workflow_triggers.py new file mode 100644 index 0000000000..d8e7356fd1 --- /dev/null +++ b/scripts/lint_workflow_triggers.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Refuse dangerous GitHub Actions trigger patterns at PR time. + +Two patterns are banned outright, both of which powered the TanStack +GHSA-g7cv-rxg3-hmpx supply-chain compromise: + +1. `pull_request_target` -- runs a fork's workflow YAML against the + BASE repository's secrets and permissions. The fork can inject + arbitrary code into the base context. The TanStack worm used this + to land base-context execution from a fork PR. There is essentially + no safe use of this trigger for a public open-source project; + `pull_request` is the safe alternative. + +2. `workflow_run` chained to a PR-triggered workflow -- carries the + same trust boundary problem one hop later. If a PR-triggered + workflow can poison artifacts/caches and a `workflow_run` trigger + fires off the result with elevated permissions, the attacker still + reaches the trusted context. + +3. Shared cache keys between PR-triggered workflows and publish / + release / push-triggered workflows. The TanStack worm poisoned the + Actions cache from a fork PR and the legitimate release workflow + then restored the poisoned cache. Cache keys must be partitioned + so that nothing a PR can write is ever read by a workflow that + holds secrets. + +Exit codes +========== + + 0 no findings + 1 one or more findings; stderr lists each with file path + +Run from repo root: + python3 scripts/lint_workflow_triggers.py +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +try: + import yaml +except ImportError: + print( + "ERROR: PyYAML is required. Install with 'pip install pyyaml'", file = sys.stderr + ) + sys.exit(2) + +REPO_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_WORKFLOWS_DIR = REPO_ROOT / ".github" / "workflows" + +BANNED_TRIGGERS: tuple[str, ...] = ("pull_request_target",) +RESTRICTED_TRIGGERS: tuple[str, ...] = ("workflow_run",) +PUBLISH_WORKFLOW_NAMES: tuple[str, ...] = ("release-desktop.yml",) + + +def _normalise_on(on_field): + if isinstance(on_field, str): + return {on_field} + if isinstance(on_field, list): + return set(on_field) + if isinstance(on_field, dict): + return set(on_field.keys()) + return set() + + +def _load_workflow(path: Path): + try: + return yaml.safe_load(path.read_text()) + except Exception as exc: + print(f"ERROR: failed to parse {path}: {exc}", file = sys.stderr) + sys.exit(2) + + +def _extract_cache_keys(path: Path) -> list[str]: + text = path.read_text() + keys: list[str] = [] + for m in re.finditer(r"(?:^|\n)\s*key:\s*([^\n]+)", text): + keys.append(m.group(1).strip()) + return keys + + +def _trigger_set(yaml_doc) -> set[str]: + on = yaml_doc.get(True) + if on is None: + on = yaml_doc.get("on") + return _normalise_on(on) + + +def main() -> int: + parser = argparse.ArgumentParser(description = __doc__) + parser.add_argument( + "--workflows-dir", + type = Path, + default = DEFAULT_WORKFLOWS_DIR, + help = "Override the workflows directory (used by tests).", + ) + args = parser.parse_args() + workflows_dir = args.workflows_dir + + findings: list[str] = [] + workflows = sorted(workflows_dir.glob("*.yml")) + pr_triggered: list[tuple[Path, list[str]]] = [] + publish_triggered: list[tuple[Path, list[str]]] = [] + + for path in workflows: + doc = _load_workflow(path) + triggers = _trigger_set(doc) + + for t in BANNED_TRIGGERS: + if t in triggers: + findings.append( + f"{path.name}: BANNED trigger '{t}' (GHSA-g7cv-rxg3-hmpx " + "pattern: fork PRs run in base-repo context). Switch to " + "'pull_request' and use a deploy-on-merge workflow for " + "any privileged step." + ) + + for t in RESTRICTED_TRIGGERS: + if t in triggers: + text = path.read_text() + if "lint:workflow_triggers-allow-workflow_run" not in text: + findings.append( + f"{path.name}: RESTRICTED trigger '{t}' requires an " + "explicit `# lint:workflow_triggers-allow-workflow_run` " + "comment somewhere in the file, with a justification." + ) + + if "pull_request" in triggers: + pr_triggered.append((path, _extract_cache_keys(path))) + is_dispatch_only = "workflow_dispatch" in triggers and not ( + "push" in triggers or "pull_request" in triggers + ) + if path.name in PUBLISH_WORKFLOW_NAMES or is_dispatch_only: + publish_triggered.append((path, _extract_cache_keys(path))) + + pr_keys = {key for _, keys in pr_triggered for key in keys} + for pub_path, pub_keys in publish_triggered: + for k in pub_keys: + if k in pr_keys: + findings.append( + f"{pub_path.name}: cache key {k!r} is also declared in a " + "PR-triggered workflow. A fork PR could poison this cache " + "and the publish workflow would restore it on next run. " + "Add a unique suffix (e.g. '-publish-only') to partition " + "the namespaces." + ) + + if findings: + print( + "Workflow trigger lint failed with the following issues:", file = sys.stderr + ) + for f in findings: + print(f" - {f}", file = sys.stderr) + return 1 + + print( + f"OK: scanned {len(workflows)} workflow file(s); " + f"no pull_request_target, no unjustified workflow_run, " + f"no PR/publish cache-key collision." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/lockfile_supply_chain_audit.py b/scripts/lockfile_supply_chain_audit.py new file mode 100644 index 0000000000..ae215bf344 --- /dev/null +++ b/scripts/lockfile_supply_chain_audit.py @@ -0,0 +1,754 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Lockfile supply-chain audit for the Studio frontend and Tauri shell. + +Runs BEFORE `npm ci` / `cargo fetch` in CI. Refuses to proceed when a +lockfile contains patterns that indicate the kind of supply-chain +injection seen in the npm Shai-Hulud waves and the cargo +crates.io brand-squat attempts. + +What it checks +============== + +studio/frontend/package-lock.json (lockfileVersion 2 or 3): + + 1. `resolved` URL origin. Every entry must resolve through + `https://registry.npmjs.org/`. Direct GitHub-hosted dependencies + (`git+ssh://`, `git+https://`, `github:owner/repo#sha`, + `file:`, `http://`) are refused -- npm's TanStack incident used + exactly this vector to land an unaudited GitHub commit hash as + an optional dependency. + + 2. `integrity` field presence. Every non-workspace entry must carry + an `integrity` SHA. A missing integrity means the registry can + swap the tarball after lockfile generation and CI will not + notice. + + 3. Known IOC strings. A hardcoded set of indicator-of-compromise + substrings is grepped across the entire lockfile body (file + names, dependency keys, URLs). The list is updated as new + campaigns surface. Catching one means the local install was + about to pull a publicly-known malicious release. + +studio/src-tauri/Cargo.lock: + + 4. `source` field origin. Every entry with a `source` must point at + `registry+https://github.com/rust-lang/crates.io-index`. Direct + git sources (`git+https://...`) and `path+...` for cross-crate + paths warrant manual review and are flagged. + + 5. Known cargo IOC strings. Same idea as (3), separate list. + +Exit codes +========== + + 0 no findings, or an opt-out env var (UNSLOTH_LOCKFILE_AUDIT_SKIP) + is set to a justification string (>=5 chars, not '1'/'true'/etc). + A value like '1' or 'true' is now REJECTED loudly and the audit + runs normally + 1 one or more findings; stderr lists them with file path and line + number where derivable + 2 internal error (missing dependency, malformed JSON, etc.) + +Operational stance +================== + +This scanner only PARSES the lockfiles -- it never executes anything +in them, never resolves anything against the network. Safe to run +ahead of every `npm ci`. The IOC list is short by design; this +complements (not replaces) `npm audit`, OSV-Scanner, and the +advisory-DB pipeline in `.github/workflows/security-audit.yml`. The +shape of the catch is "we refuse to proceed because the lockfile +itself is shaped wrong", which fires before any third-party install +script gets a chance to run on the runner. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +# ───────────────────────────────────────────────────────────────────── +# Known IOC strings (case-sensitive substring match). +# ───────────────────────────────────────────────────────────────────── +# +# Keep these short and FACTUAL. Each entry is tied to a public advisory +# and is the literal string an attacker would have to embed for the +# attack to work. Adding speculative or generic patterns here would +# generate false positives on dependency upgrades. +NPM_IOC_STRINGS: tuple[str, ...] = ( + # Shai-Hulud TanStack wave -- May 11, 2026 (GHSA-g7cv-rxg3-hmpx). + "router_init.js", + "tanstack_runner.js", + "router_runtime.js", + "@tanstack/setup", + "github:tanstack/router#79ac49eedf774dd4b0cfa308722bc463cfe5885c", + # Exfiltration endpoints observed across both Shai-Hulud waves. + "filev2.getsession.org", + "getsession.org/file/", + # Campaign markers; the worm tarballs print this to stdout on run. + "A Mini Shai-Hulud has Appeared", + # Mini Shai-Hulud May-12 2026 wave. + "git-tanstack.com", + "transformers.pyz", + "/tmp/transformers.pyz", + "With Love TeamPCP", + # Aikido (May-12 wave): payload SHA-256 hashes + Bun marker. + "ab4fcadaec49c03278063dd269ea5eef82d24f2124a8e15d7b90f2fa8601266c", + "2ec78d556d696e208927cc503d48e4b5eb56b31abc2870c2ed2e98d6be27fc96", + "bun run tanstack_runner.js", + "We've been online over 2 hours", +) + +# Hard pin-blocks for publicly confirmed malicious versions. +# keep in sync with scripts/scan_npm_packages.py +BLOCKED_NPM_VERSIONS: dict[str, set[str]] = { + # GHSA-g7cv-rxg3-hmpx -- TanStack May-11 2026 (84 versions). + "@tanstack/arktype-adapter": {"1.166.12", "1.166.15"}, + "@tanstack/eslint-plugin-router": {"1.161.9", "1.161.12"}, + "@tanstack/eslint-plugin-start": {"0.0.4", "0.0.7"}, + "@tanstack/history": {"1.161.9", "1.161.12"}, + "@tanstack/nitro-v2-vite-plugin": {"1.154.12", "1.154.15"}, + "@tanstack/react-router": {"1.169.5", "1.169.8"}, + "@tanstack/react-router-devtools": {"1.166.16", "1.166.19"}, + "@tanstack/react-router-ssr-query": {"1.166.15", "1.166.18"}, + "@tanstack/react-start": {"1.167.68", "1.167.71"}, + "@tanstack/react-start-client": {"1.166.51", "1.166.54"}, + "@tanstack/react-start-rsc": {"0.0.47", "0.0.50"}, + "@tanstack/react-start-server": {"1.166.55", "1.166.58"}, + "@tanstack/router-cli": {"1.166.46", "1.166.49"}, + "@tanstack/router-core": {"1.169.5", "1.169.8"}, + "@tanstack/router-devtools": {"1.166.16", "1.166.19"}, + "@tanstack/router-devtools-core": {"1.167.6", "1.167.9"}, + "@tanstack/router-generator": {"1.166.45", "1.166.48"}, + "@tanstack/router-plugin": {"1.167.38", "1.167.41"}, + "@tanstack/router-ssr-query-core": {"1.168.3", "1.168.6"}, + "@tanstack/router-utils": {"1.161.11", "1.161.14"}, + "@tanstack/router-vite-plugin": {"1.166.53", "1.166.56"}, + "@tanstack/solid-router": {"1.169.5", "1.169.8"}, + "@tanstack/solid-router-devtools": {"1.166.16", "1.166.19"}, + "@tanstack/solid-router-ssr-query": {"1.166.15", "1.166.18"}, + "@tanstack/solid-start": {"1.167.65", "1.167.68"}, + "@tanstack/solid-start-client": {"1.166.50", "1.166.53"}, + "@tanstack/solid-start-server": {"1.166.54", "1.166.57"}, + "@tanstack/start-client-core": {"1.168.5", "1.168.8"}, + "@tanstack/start-fn-stubs": {"1.161.9", "1.161.12"}, + "@tanstack/start-plugin-core": {"1.169.23", "1.169.26"}, + "@tanstack/start-server-core": {"1.167.33", "1.167.36"}, + "@tanstack/start-static-server-functions": {"1.166.44", "1.166.47"}, + "@tanstack/start-storage-context": {"1.166.38", "1.166.41"}, + "@tanstack/valibot-adapter": {"1.166.12", "1.166.15"}, + "@tanstack/virtual-file-routes": {"1.161.10", "1.161.13"}, + "@tanstack/vue-router": {"1.169.5", "1.169.8"}, + "@tanstack/vue-router-devtools": {"1.166.16", "1.166.19"}, + "@tanstack/vue-router-ssr-query": {"1.166.15", "1.166.18"}, + "@tanstack/vue-start": {"1.167.61", "1.167.64"}, + "@tanstack/vue-start-client": {"1.166.46", "1.166.49"}, + "@tanstack/vue-start-server": {"1.166.50", "1.166.53"}, + "@tanstack/zod-adapter": {"1.166.12", "1.166.15"}, + # Mini Shai-Hulud May-12 wave: OpenSearch JS client. + "@opensearch-project/opensearch": {"3.5.3", "3.6.2", "3.7.0", "3.8.0"}, + # Mini Shai-Hulud May-12 wave: @squawk/* (22 packages, 5 versions each; + # https://safedep.io/mass-npm-supply-chain-attack-tanstack-mistral/). + "@squawk/airport-data": {"0.7.4", "0.7.5", "0.7.6", "0.7.7", "0.7.8"}, + "@squawk/airports": {"0.6.2", "0.6.3", "0.6.4", "0.6.5", "0.6.6"}, + "@squawk/airspace": {"0.8.1", "0.8.2", "0.8.3", "0.8.4", "0.8.5"}, + "@squawk/airspace-data": {"0.5.3", "0.5.4", "0.5.5", "0.5.6", "0.5.7"}, + "@squawk/airway-data": {"0.5.4", "0.5.5", "0.5.6", "0.5.7", "0.5.8"}, + "@squawk/airways": {"0.4.2", "0.4.3", "0.4.4", "0.4.5", "0.4.6"}, + "@squawk/fix-data": {"0.6.4", "0.6.5", "0.6.6", "0.6.7", "0.6.8"}, + "@squawk/fixes": {"0.3.2", "0.3.3", "0.3.4", "0.3.5", "0.3.6"}, + "@squawk/flight-math": {"0.5.4", "0.5.5", "0.5.6", "0.5.7", "0.5.8"}, + "@squawk/flightplan": {"0.5.2", "0.5.3", "0.5.4", "0.5.5", "0.5.6"}, + "@squawk/geo": {"0.4.4", "0.4.5", "0.4.6", "0.4.7", "0.4.8"}, + "@squawk/icao-registry": {"0.5.2", "0.5.3", "0.5.4", "0.5.5", "0.5.6"}, + "@squawk/icao-registry-data": {"0.8.4", "0.8.5", "0.8.6", "0.8.7", "0.8.8"}, + "@squawk/mcp": {"0.9.1", "0.9.2", "0.9.3", "0.9.4", "0.9.5"}, + "@squawk/navaid-data": {"0.6.4", "0.6.5", "0.6.6", "0.6.7", "0.6.8"}, + "@squawk/navaids": {"0.4.2", "0.4.3", "0.4.4", "0.4.5", "0.4.6"}, + "@squawk/notams": {"0.3.6", "0.3.7", "0.3.8", "0.3.9", "0.3.10"}, + "@squawk/procedure-data": {"0.7.3", "0.7.4", "0.7.5", "0.7.6", "0.7.7"}, + "@squawk/procedures": {"0.5.2", "0.5.3", "0.5.4", "0.5.5", "0.5.6"}, + "@squawk/types": {"0.8.1", "0.8.2", "0.8.3", "0.8.4", "0.8.5"}, + "@squawk/units": {"0.4.3", "0.4.4", "0.4.5", "0.4.6", "0.4.7"}, + "@squawk/weather": {"0.5.6", "0.5.7", "0.5.8", "0.5.9", "0.5.10"}, + # Mini Shai-Hulud May-12 wave: @uipath/* (64 packages, single version each; + # https://www.aikido.dev/blog/mini-shai-hulud-is-back-tanstack-compromised). + "@uipath/apollo-react": {"4.24.5"}, + "@uipath/apollo-wind": {"2.16.2"}, + "@uipath/cli": {"1.0.1"}, + "@uipath/rpa-tool": {"0.9.5"}, + "@uipath/apollo-core": {"5.9.2"}, + "@uipath/filesystem": {"1.0.1"}, + "@uipath/solutionpackager-tool-core": {"0.0.34"}, + "@uipath/solution-tool": {"1.0.1"}, + "@uipath/maestro-tool": {"1.0.1"}, + "@uipath/codedapp-tool": {"1.0.1"}, + "@uipath/agent-tool": {"1.0.1"}, + "@uipath/orchestrator-tool": {"1.0.1"}, + "@uipath/integrationservice-tool": {"1.0.2"}, + "@uipath/rpa-legacy-tool": {"1.0.1"}, + "@uipath/vertical-solutions-tool": {"1.0.1"}, + "@uipath/flow-tool": {"1.0.2"}, + "@uipath/codedagent-tool": {"1.0.1"}, + "@uipath/common": {"1.0.1"}, + "@uipath/resource-tool": {"1.0.1"}, + "@uipath/auth": {"1.0.1"}, + "@uipath/docsai-tool": {"1.0.1"}, + "@uipath/case-tool": {"1.0.1"}, + "@uipath/api-workflow-tool": {"1.0.1"}, + "@uipath/test-manager-tool": {"1.0.2"}, + "@uipath/robot": {"1.3.4"}, + "@uipath/traces-tool": {"1.0.1"}, + "@uipath/agent-sdk": {"1.0.2"}, + "@uipath/integrationservice-sdk": {"1.0.2"}, + "@uipath/maestro-sdk": {"1.0.1"}, + "@uipath/data-fabric-tool": {"1.0.2"}, + "@uipath/tasks-tool": {"1.0.1"}, + "@uipath/insights-tool": {"1.0.1"}, + "@uipath/insights-sdk": {"1.0.1"}, + "@uipath/uipath-python-bridge": {"1.0.1"}, + "@uipath/ap-chat": {"1.5.7"}, + "@uipath/project-packager": {"1.1.16"}, + "@uipath/packager-tool-case": {"0.0.9"}, + "@uipath/packager-tool-workflowcompiler-browser": {"0.0.34"}, + "@uipath/packager-tool-connector": {"0.0.19"}, + "@uipath/packager-tool-workflowcompiler": {"0.0.16"}, + "@uipath/packager-tool-webapp": {"1.0.6"}, + "@uipath/packager-tool-apiworkflow": {"0.0.19"}, + "@uipath/packager-tool-functions": {"0.1.1"}, + "@uipath/widget.sdk": {"1.2.3"}, + "@uipath/resources-tool": {"0.1.11"}, + "@uipath/agent.sdk": {"0.0.18"}, + "@uipath/codedagents-tool": {"0.1.12"}, + "@uipath/aops-policy-tool": {"0.3.1"}, + "@uipath/solution-packager": {"0.0.35"}, + "@uipath/packager-tool-bpmn": {"0.0.9"}, + "@uipath/packager-tool-flow": {"0.0.19"}, + "@uipath/telemetry": {"0.0.7"}, + "@uipath/tool-workflowcompiler": {"0.0.12"}, + "@uipath/vss": {"0.1.6"}, + "@uipath/solutionpackager-sdk": {"1.0.11"}, + "@uipath/ui-widgets-multi-file-upload": {"1.0.1"}, + "@uipath/access-policy-tool": {"0.3.1"}, + "@uipath/context-grounding-tool": {"0.1.1"}, + "@uipath/gov-tool": {"0.3.1"}, + "@uipath/admin-tool": {"0.1.1"}, + "@uipath/identity-tool": {"0.1.1"}, + "@uipath/llmgw-tool": {"1.0.1"}, + "@uipath/resourcecatalog-tool": {"0.1.1"}, + "@uipath/functions-tool": {"1.0.1"}, + "@uipath/access-policy-sdk": {"0.3.1"}, + "@uipath/platform-tool": {"1.0.1"}, + # Mini Shai-Hulud May-12 wave: @mistralai/* (npm) — separate from PyPI mistralai + # (https://www.aikido.dev/blog/mini-shai-hulud-is-back-tanstack-compromised). + "@mistralai/mistralai": {"2.2.2", "2.2.3", "2.2.4"}, + "@mistralai/mistralai-gcp": {"1.7.1", "1.7.2", "1.7.3"}, + "@mistralai/mistralai-azure": {"1.7.1", "1.7.2", "1.7.3"}, + # Mini Shai-Hulud May-12 wave: @tallyui/* (30 entries, 10 packages) + # (Aikido enumeration). + "@tallyui/components": {"1.0.1", "1.0.2", "1.0.3"}, + "@tallyui/connector-medusa": {"1.0.1", "1.0.2", "1.0.3"}, + "@tallyui/connector-shopify": {"1.0.1", "1.0.2", "1.0.3"}, + "@tallyui/connector-vendure": {"1.0.1", "1.0.2", "1.0.3"}, + "@tallyui/connector-woocommerce": {"1.0.1", "1.0.2", "1.0.3"}, + "@tallyui/core": {"0.2.1", "0.2.2", "0.2.3"}, + "@tallyui/database": {"1.0.1", "1.0.2", "1.0.3"}, + "@tallyui/pos": {"0.1.1", "0.1.2", "0.1.3"}, + "@tallyui/storage-sqlite": {"0.2.1", "0.2.2", "0.2.3"}, + "@tallyui/theme": {"0.2.1", "0.2.2", "0.2.3"}, + # Mini Shai-Hulud May-12 wave: @beproduct/nestjs-auth (18 versions) + # (Aikido enumeration). + "@beproduct/nestjs-auth": { + "0.1.2", + "0.1.3", + "0.1.4", + "0.1.5", + "0.1.6", + "0.1.7", + "0.1.8", + "0.1.9", + "0.1.10", + "0.1.11", + "0.1.12", + "0.1.13", + "0.1.14", + "0.1.15", + "0.1.16", + "0.1.17", + "0.1.18", + "0.1.19", + }, + # Mini Shai-Hulud May-12 wave: @draftlab/* + @draftauth/* + # (Aikido enumeration). + "@draftauth/client": {"0.2.1", "0.2.2"}, + "@draftauth/core": {"0.13.1", "0.13.2"}, + "@draftlab/auth": {"0.24.1", "0.24.2"}, + "@draftlab/auth-router": {"0.5.1", "0.5.2"}, + "@draftlab/db": {"0.16.1"}, + # Mini Shai-Hulud May-12 wave: @taskflow-corp/cli + @tolka/cli + # (Aikido enumeration). + "@taskflow-corp/cli": {"0.1.24", "0.1.25", "0.1.26", "0.1.27", "0.1.28", "0.1.29"}, + "@tolka/cli": {"1.0.2", "1.0.3", "1.0.4", "1.0.5", "1.0.6"}, + # Mini Shai-Hulud May-12 wave: @ml-toolkit-ts/* + @mesadev/* + @dirigible-ai/sdk + @supersurkhet/* + # (Aikido enumeration). + "@dirigible-ai/sdk": {"0.6.2", "0.6.3"}, + "@mesadev/rest": {"0.28.3"}, + "@mesadev/saguaro": {"0.4.22"}, + "@mesadev/sdk": {"0.28.3"}, + "@ml-toolkit-ts/preprocessing": {"1.0.2", "1.0.3"}, + "@ml-toolkit-ts/xgboost": {"1.0.3", "1.0.4"}, + "@supersurkhet/cli": {"0.0.2", "0.0.3", "0.0.4", "0.0.5", "0.0.6", "0.0.7"}, + "@supersurkhet/sdk": {"0.0.2", "0.0.3", "0.0.4", "0.0.5", "0.0.6", "0.0.7"}, + # Mini Shai-Hulud May-12 wave: Unscoped packages (10 entries) + # (Aikido enumeration). + "safe-action": {"0.8.3", "0.8.4"}, + "ts-dna": {"3.0.1", "3.0.2", "3.0.3", "3.0.4"}, + "cross-stitch": {"1.1.3", "1.1.4", "1.1.5", "1.1.6"}, + "cmux-agent-mcp": {"0.1.3", "0.1.4", "0.1.5", "0.1.6", "0.1.7", "0.1.8"}, + "agentwork-cli": {"0.1.4", "0.1.5"}, + "git-branch-selector": {"1.3.3", "1.3.4", "1.3.5", "1.3.6", "1.3.7"}, + "wot-api": {"0.8.1", "0.8.2", "0.8.3", "0.8.4"}, + "git-git-git": {"1.0.8", "1.0.9", "1.0.10", "1.0.11", "1.0.12"}, + "nextmove-mcp": {"0.1.3", "0.1.4", "0.1.5", "0.1.6", "0.1.7"}, + "ml-toolkit-ts": {"1.0.4", "1.0.5"}, + # Cross-ecosystem Mini Shai-Hulud (Apr-30 wave): npm counterpart of + # PyPI lightning 2.6.2/2.6.3. Same threat actor (TeamPCP) per Semgrep, + # Aikido, OX Security, Resecurity. Safe version: 7.0.3 and earlier. + "intercom-client": {"7.0.4"}, +} + +CARGO_IOC_STRINGS: tuple[str, ...] = ( + # Reserved for future cargo-side incidents. Empty by default -- + # `source` origin check below catches the structural pattern. +) + + +# ───────────────────────────────────────────────────────────────────── +# Allowed lockfile origins. +# ───────────────────────────────────────────────────────────────────── +NPM_REGISTRY_PREFIX = "https://registry.npmjs.org/" + +# Tarballs are also fetched from this mirror on some GH Actions cached +# runs (npm rewrites the resolved URL on cache hit). Allow either. +NPM_REGISTRY_PREFIXES_ALLOWED: tuple[str, ...] = (NPM_REGISTRY_PREFIX,) + +CARGO_REGISTRY_SOURCE = "registry+https://github.com/rust-lang/crates.io-index" + + +# ───────────────────────────────────────────────────────────────────── +# Cargo non-registry source allowlist. +# ───────────────────────────────────────────────────────────────────── +# +# Each entry is `(crate_name, exact_source_string)`. The crate must +# match by name AND the source must match the full pinned-SHA string +# verbatim. Bumping the commit SHA forces a re-review here: the +# scanner fires until the new SHA is appended. +# +# Studio's Tauri shell pulls `fix-path-env` directly from +# tauri-apps/fix-path-env-rs because the crate is not published to +# crates.io. The pinned commit (c4c45d5) was reviewed at the time it +# landed; future bumps need explicit approval. +CARGO_SOURCE_ALLOWLIST: tuple[tuple[str, str], ...] = ( + ( + "fix-path-env", + "git+https://github.com/tauri-apps/fix-path-env-rs#" + "c4c45d503ea115a839aae718d02f79e7c7f0f673", + ), +) + + +# ───────────────────────────────────────────────────────────────────── +# Finding container. +# ───────────────────────────────────────────────────────────────────── + + +class Finding: + __slots__ = ("path", "package", "kind", "detail") + + def __init__(self, path: str, package: str, kind: str, detail: str) -> None: + self.path = path + self.package = package + self.kind = kind + self.detail = detail + + def __str__(self) -> str: + return ( + f" [{self.kind}] {self.path}\n" + f" package: {self.package}\n" + f" detail: {self.detail}" + ) + + +# ───────────────────────────────────────────────────────────────────── +# package-lock.json audit. +# ───────────────────────────────────────────────────────────────────── + + +def audit_npm_lockfile(path: Path) -> list[Finding]: + findings: list[Finding] = [] + if not path.exists(): + return findings + + raw = path.read_text(encoding = "utf-8") + try: + lock = json.loads(raw) + except json.JSONDecodeError as exc: + findings.append( + Finding( + path = str(path), + package = "", + kind = "malformed-lockfile", + detail = f"could not parse as JSON: {exc}", + ) + ) + return findings + + lockfile_version = lock.get("lockfileVersion") + if lockfile_version not in (2, 3): + findings.append( + Finding( + path = str(path), + package = "", + kind = "unsupported-lockfile-version", + detail = (f"only lockfileVersion 2 or 3 audited; got {lockfile_version}"), + ) + ) + + packages = lock.get("packages") or {} + for key, entry in packages.items(): + # The empty key "" is the project root; workspace entries use + # keys like "node_modules/foo" or "studio/frontend/sub-pkg". + # Skip the project root (it has no `resolved`). + if key == "": + continue + if entry.get("link"): + # Workspace symlink; no tarball to resolve. + continue + + resolved = entry.get("resolved") + # Entries living inside another package's `node_modules/` + # tree are bundled fold-ins -- the parent's tarball ships + # their source verbatim and the parent's `integrity` covers + # the whole subtree. npm represents them in lockfileVersion 3 + # as nested entries with no `resolved` and no `integrity` of + # their own. Treat them as transparent to this audit. + nested = key.count("/node_modules/") >= 1 + + # 1. resolved-URL origin. + if resolved is None: + if nested or entry.get("bundled"): + # Bundled / fold-in entry; covered by parent integrity. + pass + elif entry.get("version"): + # Top-level entry without a resolved URL is suspicious. + findings.append( + Finding( + path = str(path), + package = key, + kind = "missing-resolved-url", + detail = ( + f"version={entry['version']!r} but no `resolved` " + "field; lockfile is incomplete" + ), + ) + ) + else: + if not any(resolved.startswith(p) for p in NPM_REGISTRY_PREFIXES_ALLOWED): + findings.append( + Finding( + path = str(path), + package = key, + kind = "non-registry-resolved-url", + detail = ( + f"resolved={resolved!r}; only " + f"{NPM_REGISTRY_PREFIX} is permitted. Direct " + "GitHub / git / file references are the " + "Shai-Hulud injection vector." + ), + ) + ) + + # 2. integrity-hash presence. + if resolved is not None and not entry.get("integrity"): + findings.append( + Finding( + path = str(path), + package = key, + kind = "missing-integrity-hash", + detail = ( + "no `integrity` field; npm cannot verify the " + "tarball SHA against the registry-published hash" + ), + ) + ) + + # 3. Blocked malicious version list. + nm_prefix = "node_modules/" + pkg_name = key[len(nm_prefix) :] if key.startswith(nm_prefix) else key + version = entry.get("version") + blocked = BLOCKED_NPM_VERSIONS.get(pkg_name, set()) + if version and version in blocked: + findings.append( + Finding( + path = str(path), + package = key, + kind = "blocked-known-malicious", + detail = ( + f"{pkg_name}@{version} is on the " "BLOCKED_NPM_VERSIONS list" + ), + ) + ) + + # 4. Known IOC strings: scan the raw file body so we hit fields the + # structural pass above doesn't enumerate (scripts, optional + # dependencies, etc.). Cheap and complete. + for ioc in NPM_IOC_STRINGS: + if ioc in raw: + # Best-effort line number lookup. + line_no = _first_line_containing(raw, ioc) + findings.append( + Finding( + path = f"{path}:{line_no}" if line_no else str(path), + package = "", + kind = "known-ioc-string", + detail = ( + f"matched known IOC substring {ioc!r}; this is " + "a public indicator of a recent supply-chain " + "compromise. Refuse to install." + ), + ) + ) + + return findings + + +def _first_line_containing(text: str, needle: str) -> int | None: + for i, line in enumerate(text.splitlines(), start = 1): + if needle in line: + return i + return None + + +# ───────────────────────────────────────────────────────────────────── +# Cargo.lock audit. +# ───────────────────────────────────────────────────────────────────── + + +# Cargo.lock is TOML; parse with stdlib tomllib (Python 3.11+). The +# studio's Tauri shell already requires a modern toolchain so this is +# always available where CI runs. +_PACKAGE_HEADER = re.compile(r"^\[\[package\]\]\s*$") + + +def audit_cargo_lockfile(path: Path) -> list[Finding]: + findings: list[Finding] = [] + if not path.exists(): + return findings + + raw = path.read_text(encoding = "utf-8") + try: + import tomllib # type: ignore[import-not-found] + except ImportError: + # Python <3.11; fall back to a tomli shim if importable. + try: + import tomli as tomllib # type: ignore[no-redef] + except ImportError: + findings.append( + Finding( + path = str(path), + package = "", + kind = "missing-toml-parser", + detail = ( + "Python 3.11+ tomllib or tomli is required to " + "parse Cargo.lock; install tomli or upgrade " + "Python before re-running this audit" + ), + ) + ) + return findings + + try: + lock = tomllib.loads(raw) + except Exception as exc: + findings.append( + Finding( + path = str(path), + package = "", + kind = "malformed-lockfile", + detail = f"could not parse as TOML: {exc}", + ) + ) + return findings + + for entry in lock.get("package", []): + name = entry.get("name") or "" + version = entry.get("version") or "" + source = entry.get("source") + # Workspace-local crates have no `source` field; skip them. + if source is None: + continue + if source != CARGO_REGISTRY_SOURCE: + if (name, source) in CARGO_SOURCE_ALLOWLIST: + # Pre-approved non-registry source pinned by SHA. + pass + else: + findings.append( + Finding( + path = str(path), + package = f"{name}@{version}", + kind = "non-registry-cargo-source", + detail = ( + f"source={source!r}; only " + f"{CARGO_REGISTRY_SOURCE!r} is permitted " + "by default, and no allowlist entry covers " + "this crate. If the source is legitimate, " + "add `(name, source)` to " + "CARGO_SOURCE_ALLOWLIST after reviewing the " + "pinned commit." + ), + ) + ) + if not entry.get("checksum") and source == CARGO_REGISTRY_SOURCE: + findings.append( + Finding( + path = str(path), + package = f"{name}@{version}", + kind = "missing-cargo-checksum", + detail = ( + "registry crate without checksum; cargo cannot " + "verify the downloaded source against the " + "registry-published SHA" + ), + ) + ) + + for ioc in CARGO_IOC_STRINGS: + if ioc in raw: + line_no = _first_line_containing(raw, ioc) + findings.append( + Finding( + path = f"{path}:{line_no}" if line_no else str(path), + package = "", + kind = "known-ioc-string", + detail = f"matched known IOC substring {ioc!r}", + ) + ) + + return findings + + +# ───────────────────────────────────────────────────────────────────── +# CLI. +# ───────────────────────────────────────────────────────────────────── + + +DEFAULT_NPM_LOCKFILES = ("studio/frontend/package-lock.json",) +DEFAULT_CARGO_LOCKFILES = ("studio/src-tauri/Cargo.lock",) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description = "Pre-install lockfile supply-chain audit.", + ) + parser.add_argument( + "--root", + default = str(REPO_ROOT), + help = "Repo root (default: parent of this script).", + ) + parser.add_argument( + "--npm-lockfile", + action = "append", + default = None, + help = ( + "Path to a package-lock.json (repeatable). " + "Default: studio/frontend/package-lock.json." + ), + ) + parser.add_argument( + "--cargo-lockfile", + action = "append", + default = None, + help = ( + "Path to a Cargo.lock (repeatable). " + "Default: studio/src-tauri/Cargo.lock." + ), + ) + args = parser.parse_args(argv) + + # SF4: require a real justification (e.g. JIRA ticket id) for the + # skip env var. Treat the trivially-set values ("1", "true", "yes", + # "on", empty) as INVALID -- they look like accidental flips and + # silently bypassed the supply-chain audit. A valid value is a + # non-empty string >=5 chars after stripping that does not match + # any of the boolean-shaped tokens above. An invalid value emits a + # loud GitHub Actions warning to stderr and FALLS THROUGH to run + # the audit normally (fail-safe). A valid value emits a warning + # naming the reason and skips with rc=0 (compat). + _skip_raw = os.environ.get("UNSLOTH_LOCKFILE_AUDIT_SKIP") + if _skip_raw is not None: + _skip = _skip_raw.strip() + _invalid_tokens = {"", "1", "0", "true", "false", "yes", "no", "on", "off"} + if _skip.lower() in _invalid_tokens or len(_skip) < 5: + print( + "::warning::Lockfile audit skip REQUIRES a justification " + f"value (>=5 chars, not '{_skip_raw}'). Proceeding with " + "audit. Use e.g. UNSLOTH_LOCKFILE_AUDIT_SKIP=ticket-1234.", + file = sys.stderr, + flush = True, + ) + else: + print( + f"::warning::Lockfile audit skipped: reason='{_skip}'", + file = sys.stderr, + flush = True, + ) + return 0 + + root = Path(args.root).resolve() + npm_paths = [root / p for p in (args.npm_lockfile or DEFAULT_NPM_LOCKFILES)] + cargo_paths = [root / p for p in (args.cargo_lockfile or DEFAULT_CARGO_LOCKFILES)] + + all_findings: list[Finding] = [] + for p in npm_paths: + print(f"[lockfile-audit] npm: {p}", flush = True) + all_findings.extend(audit_npm_lockfile(p)) + for p in cargo_paths: + print(f"[lockfile-audit] cargo: {p}", flush = True) + all_findings.extend(audit_cargo_lockfile(p)) + + if not all_findings: + print( + f"[lockfile-audit] OK: 0 findings across " + f"{len(npm_paths)} npm + {len(cargo_paths)} cargo lockfile(s)", + flush = True, + ) + return 0 + + print( + f"\n[lockfile-audit] FAIL: {len(all_findings)} finding(s):\n", + file = sys.stderr, + ) + for f in all_findings: + print(str(f), file = sys.stderr) + print(file = sys.stderr) + print( + "[lockfile-audit] Refusing to proceed. Each finding above is " + "either a structural lockfile anomaly or a public indicator-of-" + "compromise. Investigate before running `npm ci` or `cargo fetch`.", + file = sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/notebook_to_python.py b/scripts/notebook_to_python.py index 86f3239b72..4e3046103e 100644 --- a/scripts/notebook_to_python.py +++ b/scripts/notebook_to_python.py @@ -14,6 +14,7 @@ Converts IPython magics to plain Python: import nbformat import re +import shlex import sys import os import urllib.request @@ -21,6 +22,22 @@ import urllib.parse from pathlib import Path +# Hosts we are willing to fetch raw notebook JSON from. Anything else +# is rejected before `urlopen` so a typoed / hostile URL cannot pull +# code from arbitrary infrastructure. +_ALLOWED_NOTEBOOK_HOSTS = { + "raw.githubusercontent.com", + "gist.githubusercontent.com", +} + + +# Shell metacharacters that imply the cell's `!cmd` line cannot be +# parsed as a flat argv. If any of these appears, `shlex.split` would +# either fail or, worse, silently strip the operator -- so we keep +# `shell=True` for that command and emit a review marker. +_SHELL_METACHARS_RE = re.compile(r"\$\(|`|\|\||\||&&|>>?|< bool: """Check if command has Python variable interpolation like {var_name}.""" pattern = r"(? tuple[str, str]: parsed = urllib.parse.urlparse(raw_url) filename = os.path.basename(urllib.parse.unquote(parsed.path)) + # Host allowlist. Refuse to fetch from anywhere the campaign IOC + # tables flag (or just anywhere we don't recognise). The blob->raw + # conversion above only emits `raw.githubusercontent.com`, so a + # rejection here means the caller hand-typed a URL pointing + # somewhere we don't trust. + host = parsed.hostname + if host not in _ALLOWED_NOTEBOOK_HOSTS: + raise ValueError( + f"Refused notebook fetch from {host!r}: not in allowlist " + f"{sorted(_ALLOWED_NOTEBOOK_HOSTS)}" + ) + # Download print(f"Downloading {url}...") with urllib.request.urlopen(raw_url, timeout = 60) as response: @@ -74,7 +103,52 @@ def replace_colab_paths(source: str) -> str: return source -def convert_cell_to_python(source: str) -> str: +def _emit_shell_command(indent: str, full_cmd: str, *, allow_shell: bool) -> list[str]: + """Render a `!cmd` notebook line as one or more Python statements. + + When the command body is f-string-interpolated, contains shell + metacharacters, or spans multiple lines, falling back to + `shell=True` is the only correct option -- `shlex.split` would + either drop operators or fail outright. We surface that with a + `# WARNING: shell=True; reviewed for hostile input` comment so a + reviewer cannot miss it. + + Otherwise we emit `subprocess.run(shlex.split(cmd), shell=False)` + so the converted script is not a re-injection vector if the + notebook ever interpolates user-controlled data. + + `allow_shell` defaults to True at the CLI for backwards + compatibility. Setting it to False makes `shell=True` emission a + hard error (no surprise behaviour). + """ + needs_f = needs_fstring(full_cmd) + has_meta = bool(_SHELL_METACHARS_RE.search(full_cmd)) + multiline = "\n" in full_cmd + + must_use_shell = needs_f or has_meta or multiline + + if must_use_shell: + if not allow_shell: + raise ValueError( + "Cell uses shell metacharacters / interpolation but " + "--no-allow-shell was set; refusing to emit shell=True" + ) + warn = f"{indent}# WARNING: shell=True; reviewed for hostile input" + f_prefix = "f" if needs_f else "" + if multiline: + escaped_cmd = full_cmd.replace('"""', r"\"\"\"") + if escaped_cmd.rstrip().endswith('"'): + escaped_cmd = escaped_cmd.rstrip() + " " + stmt = f'{indent}subprocess.run({f_prefix}"""{escaped_cmd}""", shell=True)' + else: + stmt = f"{indent}subprocess.run({f_prefix}{full_cmd!r}, shell=True)" + return [warn, stmt] + + # Shell-safe argv form. + return [f"{indent}subprocess.run(shlex.split({full_cmd!r}), shell=False)"] + + +def convert_cell_to_python(source: str, *, allow_shell: bool = True) -> str: """Convert a cell's IPython magics to plain Python.""" lines = source.split("\n") result = [] @@ -112,18 +186,9 @@ def convert_cell_to_python(source: str) -> str: cmd_lines.append(lines[i].strip()) full_cmd = "\n".join(cmd_lines) - f_prefix = "f" if needs_fstring(full_cmd) else "" - if "\n" in full_cmd: - escaped_cmd = full_cmd.replace('"""', r"\"\"\"") - if escaped_cmd.rstrip().endswith('"'): - escaped_cmd = escaped_cmd.rstrip() + " " - result.append( - f'{indent}subprocess.run({f_prefix}"""{escaped_cmd}""", shell=True)' - ) - else: - result.append( - f"{indent}subprocess.run({f_prefix}{full_cmd!r}, shell=True)" - ) + result.extend( + _emit_shell_command(indent, full_cmd, allow_shell = allow_shell) + ) # %cd path -> os.chdir(path) elif stripped.startswith("%cd "): @@ -154,7 +219,12 @@ def convert_cell_to_python(source: str) -> str: return "\n".join(result) -def convert_notebook(notebook_content: str, source_name: str = "notebook") -> str: +def convert_notebook( + notebook_content: str, + source_name: str = "notebook", + *, + allow_shell: bool = True, +) -> str: """Convert notebook JSON content to Python script.""" # Parse notebook if isinstance(notebook_content, str): @@ -167,6 +237,7 @@ def convert_notebook(notebook_content: str, source_name: str = "notebook") -> st "# coding: utf-8", f"# Converted from: {source_name}", "", + "import shlex", "import subprocess", "import os", "import sys", @@ -189,7 +260,7 @@ def convert_notebook(notebook_content: str, source_name: str = "notebook") -> st continue if cell.cell_type == "code": - converted = convert_cell_to_python(source) + converted = convert_cell_to_python(source, allow_shell = allow_shell) converted = replace_colab_paths(converted) lines.append(converted) lines.append("") @@ -215,13 +286,20 @@ def convert_notebook(notebook_content: str, source_name: str = "notebook") -> st return "\n".join(lines) -def convert_notebook_to_script(source: str, output_dir: str | None = None): +def convert_notebook_to_script( + source: str, + output_dir: str | None = None, + *, + allow_shell: bool = True, +): """ Convert a notebook to Python script. Args: source: Local file path or URL to notebook output_dir: Output directory (optional, defaults to current directory) + allow_shell: When False, refuse to emit `shell=True` for any + `!cmd` cell that uses metacharacters / interpolation. """ if is_url(source): content, filename = download_notebook(source) @@ -246,7 +324,7 @@ def convert_notebook_to_script(source: str, output_dir: str | None = None): output_path = output_filename # Convert - script = convert_notebook(content, source_name) + script = convert_notebook(content, source_name, allow_shell = allow_shell) # Write output with open(output_path, "w", encoding = "utf-8") as f: @@ -281,19 +359,56 @@ Examples: parser.add_argument( "-o", "--output", dest = "output_dir", default = ".", help = "Output directory." ) + # Default True for backwards compatibility: existing Colab notebooks + # routinely use pipes / redirection / interpolation in `!cmd` lines + # and the converted script needs to keep working. Operators who + # convert untrusted notebooks should pass --no-allow-shell to force + # a hard error on every metacharacter-bearing cell. + parser.add_argument( + "--allow-shell", + dest = "allow_shell", + action = "store_true", + default = True, + help = "Allow emitting subprocess.run(..., shell=True) for cells " + "that use shell metacharacters or interpolation (default).", + ) + parser.add_argument( + "--no-allow-shell", + dest = "allow_shell", + action = "store_false", + help = "Refuse to emit shell=True; cells with metacharacters error out.", + ) args = parser.parse_args() # Create output directory if needed os.makedirs(args.output_dir, exist_ok = True) + # SF2: track per-notebook failures so a CI invocation that converts + # 10 notebooks but silently fails on 3 is no longer reported as + # success. Each failure is collected and the loop continues so the + # caller sees the full set; final exit status is 1 if anything + # failed. + failures: list[tuple[str, str]] = [] + ok = 0 + total = len(args.notebooks) for source in args.notebooks: try: convert_notebook_to_script( - source, output_dir = args.output_dir if args.output_dir != "." else None + source, + output_dir = args.output_dir if args.output_dir != "." else None, + allow_shell = args.allow_shell, ) + ok += 1 except Exception as e: print(f"ERROR converting {source}: {e}") + failures.append((source, f"{type(e).__name__}: {e}")) + + print( + f"converted {ok}/{total}, {len(failures)} failed", + file = sys.stderr if failures else sys.stdout, + ) + sys.exit(1 if failures else 0) if __name__ == "__main__": diff --git a/scripts/notebook_validator.py b/scripts/notebook_validator.py index 79ceb48bf0..55a9203e0b 100644 --- a/scripts/notebook_validator.py +++ b/scripts/notebook_validator.py @@ -40,12 +40,38 @@ import re import shlex import subprocess import sys +import tempfile import textwrap import time import urllib.error import urllib.request from typing import Any, Iterable, Iterator + +def _atomic_write_bytes(path: pathlib.Path, data: bytes) -> None: + """Atomic write helper. See `scripts/scan_packages.py::update_req_file`. + + A crash between `mkstemp` and `os.replace` leaves the prior file + untouched, so a half-downloaded PyPI metadata cache file cannot + poison subsequent runs of the validator. + """ + path.parent.mkdir(parents = True, exist_ok = True) + dirpath = str(path.parent) or "." + fd, tmp_path = tempfile.mkstemp(prefix = ".nb_val.", dir = dirpath) + try: + with os.fdopen(fd, "wb") as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp_path, path) + except Exception: + try: + os.unlink(tmp_path) + except OSError: + pass + raise + + HERE = pathlib.Path(__file__).resolve().parent DATA_DIR = HERE / "data" PYPI_CACHE_DIR = DATA_DIR / "pypi_cache" @@ -388,7 +414,7 @@ def pypi_metadata(name: str, version: str) -> dict[str, Any] | None: data = json.loads(r.read()) except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError): return None - path.write_text(json.dumps(data)) + _atomic_write_bytes(path, json.dumps(data).encode("utf-8")) return data @@ -863,47 +889,72 @@ def cmd_drift(args: argparse.Namespace) -> int: check = False, capture_output = True, ) - try: - proc = subprocess.run( - [sys.executable, str(update_script)], - cwd = nbdir, - capture_output = True, - text = True, - timeout = 600, - ) - except subprocess.TimeoutExpired: - print("FAIL: update_all_notebooks.py timed out (>600s)", file = sys.stderr) - return 2 - if proc.returncode != 0: - print( - f"FAIL: update_all_notebooks.py exited {proc.returncode}", file = sys.stderr - ) - sys.stderr.write(proc.stderr[-2000:]) - return 2 - diff_proc = subprocess.run( - ["git", "-C", str(nbdir), "diff", "--stat"], capture_output = True, text = True - ) + # SF3: the restore MUST run even on SystemExit / KeyboardInterrupt / + # segfault-propagated exception, otherwise the user's working tree + # silently stays rolled back into the stash. A bare try/finally + # (NOT try/except/finally) preserves the original exception and + # still runs the cleanup. The pre-existing try/except around + # `subprocess.run` of the updater is folded inside the new outer + # try so its early returns still happen, but the stash pop is + # protected. findings: list[Finding] = [] - if diff_proc.stdout.strip(): - for line in diff_proc.stdout.splitlines(): - findings.append( - Finding( - rule = "R-DRIFT-001", - file = line.strip(), - severity = "error", - message = "generator-vs-checked-in drift", - hint = "run `python update_all_notebooks.py` and commit the diff", - ) + rc: int + try: + try: + proc = subprocess.run( + [sys.executable, str(update_script)], + cwd = nbdir, + capture_output = True, + text = True, + timeout = 600, ) - # Restore. - subprocess.run( - ["git", "-C", str(nbdir), "checkout", "."], check = False, capture_output = True - ) - subprocess.run( - ["git", "-C", str(nbdir), "stash", "pop"], check = False, capture_output = True - ) + except subprocess.TimeoutExpired: + print( + "FAIL: update_all_notebooks.py timed out (>600s)", + file = sys.stderr, + ) + rc = 2 + else: + if proc.returncode != 0: + print( + f"FAIL: update_all_notebooks.py exited {proc.returncode}", + file = sys.stderr, + ) + sys.stderr.write(proc.stderr[-2000:]) + rc = 2 + else: + diff_proc = subprocess.run( + ["git", "-C", str(nbdir), "diff", "--stat"], + capture_output = True, + text = True, + ) + if diff_proc.stdout.strip(): + for line in diff_proc.stdout.splitlines(): + findings.append( + Finding( + rule = "R-DRIFT-001", + file = line.strip(), + severity = "error", + message = "generator-vs-checked-in drift", + hint = "run `python update_all_notebooks.py` and commit the diff", + ) + ) + rc = 0 if not findings else 1 + finally: + # Restore the working tree. Both commands MUST run regardless of + # how the try block exited (including SystemExit/KeyboardInterrupt). + subprocess.run( + ["git", "-C", str(nbdir), "checkout", "."], + check = False, + capture_output = True, + ) + subprocess.run( + ["git", "-C", str(nbdir), "stash", "pop"], + check = False, + capture_output = True, + ) _emit(findings) - return 0 if not findings else 1 + return rc # ----- Convert ----- # @@ -1096,7 +1147,7 @@ def cmd_refresh_colab(args: argparse.Namespace) -> int: except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError) as e: print(f"FAIL: could not fetch {COLAB_PIP_FREEZE_URL}: {e}", file = sys.stderr) return 2 - out.write_bytes(data) + _atomic_write_bytes(out, data) print(f"wrote {len(data)} bytes to {out}") return 0 diff --git a/scripts/scan_npm_packages.py b/scripts/scan_npm_packages.py new file mode 100644 index 0000000000..07eccdd716 --- /dev/null +++ b/scripts/scan_npm_packages.py @@ -0,0 +1,1457 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +# +# .github/workflows/security-audit.yml's npm-scan-packages job depends +# on this file existing at scripts/scan_npm_packages.py. + +"""scan_npm_packages.py -- npm-side content scanner. + +Counterpart to scripts/scan_packages.py for the pip ecosystem. Reads +studio/frontend/package-lock.json, downloads each resolved tarball +DIRECTLY from registry.npmjs.org (never via `npm install` -- no +lifecycle scripts ever run), verifies the lockfile integrity hash, +unpacks each tarball into a sandboxed temp dir behind size / count / +path-escape / symlink guards, and pattern-scans the extracted file +contents for the signatures common to npm supply-chain attacks: + + - Lifecycle (preinstall / install / postinstall / prepare) scripts + in any package.json that fetch + execute external code. + - C2 / exfiltration hosts (getsession.org, AWS IMDS endpoints, + Kubernetes ServiceAccount token paths, GitHub Actions OIDC, + HashiCorp Vault endpoints). + - Credential-stealing references (~/.npmrc, ~/.aws/credentials, + GITHUB_TOKEN / NPM_TOKEN in JS sources). + - Known IOC filenames from public advisories + (router_init.js, tanstack_runner.js, router_runtime.js). + - Obfuscation shapes (large single JS in package root with a low + whitespace ratio + Function/eval against a base64-decoded blob). + +Safety stance +============= + +This script ingests attacker-controlled archives. Every parse path +assumes the worst: + + 1. Downloads ONLY from `registry.npmjs.org`. Any tarball URL with a + different hostname is refused without fetching. + 2. Tarball download is size-capped (HARD_MAX_TARBALL_BYTES default + 64 MiB). HEAD-style probe via the Content-Length response header + plus a chunked read that aborts on overflow. + 3. SHA-512 integrity verified against the lockfile entry BEFORE the + tarball is even opened. A mismatch aborts that package -- the + scanner does not "fall back" to the registry-published hash. + 4. tar extraction goes through `safe_extract`: + - rejects symbolic links (`SYMTYPE`, `LNKTYPE`) + - rejects absolute paths, `..` traversal, paths outside the + extract root after resolution + - rejects character / block / FIFO devices + - per-file uncompressed size cap (HARD_MAX_FILE_BYTES, default + 8 MiB) AND cumulative cap (HARD_MAX_TOTAL_BYTES, default + 128 MiB) AND member-count cap (HARD_MAX_MEMBERS, default + 50_000) + - tar reads happen via `tarfile.open(mode='r|gz')` streaming + so an oversized file is detected before write + 5. NOTHING from the extracted tree is ever executed. Files are read + as raw bytes, decoded with `errors='replace'`, and grepped. We + never call `node`, `eval`, `compile`, `subprocess.run`, + `os.system`, or anything that would touch the tarball's + declared scripts. + 6. Tempdir is created with `tempfile.mkdtemp(prefix='npm-scan-')`, + fully resolved with .resolve(), and registered with atexit to be + wiped on every termination path. + 7. Stdlib only. No third-party deps -- adding one would itself be a + supply-chain liability. + +Exit codes +========== + + 0 no findings of severity HIGH or higher + 1 one or more HIGH/CRITICAL findings (or pre-scan structural + anomalies -- non-registry resolved URL, missing integrity) + 2 internal error (lockfile missing, integrity mismatch on + download, malformed tarball, etc.) + +The script is meant to be run in CI on every PR that touches +package-lock.json and on a nightly schedule. +""" + +from __future__ import annotations + +import argparse +import atexit +import base64 as _b64 # imported only so the IOC string-scan can detect it +import hashlib +import io +import json +import os +import re +import shutil +import sys +import tarfile +import tempfile +import urllib.parse +import urllib.request +from dataclasses import dataclass, field +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] + +# ───────────────────────────────────────────────────────────────────── +# Hard caps (deliberately conservative; npm tarballs in this repo are +# all well under these limits, so a packaging spike is noticeable). +# ───────────────────────────────────────────────────────────────────── +# Caps calibrated against the real Studio frontend transitive closure: +# - typescript.js is 9.1 MB (TS compiler bundled into one file) +# - mermaid 11.x dist/mermaid.js.map is ~12 MB (sourcemap) +# - lightningcss-linux-x64-{gnu,musl}.node is 10 MB +# - rolldown bindings (.node) are 18-26 MB per platform +# - @next/swc-*.node is ~137 MB (rust-compiled SWC engine) +# - next.js cumulative bundle is ~134 MB (turbopack compiled) +# +# Native binaries (.node, .wasm, .so, .dll, .dylib) are GENUINELY +# huge and not amenable to text pattern scanning -- we extract them +# only to verify the tarball integrity over the full archive, then +# skip them in scan_extracted_tree. They get a much higher per-file +# cap. Text files (JS/TS/JSON/etc) keep the tight cap because the +# pattern scanner runs over them and a 9.1 MB typescript.js is the +# legitimate ceiling. +HARD_MAX_TARBALL_BYTES = 256 * 1024 * 1024 # 256 MiB compressed +HARD_MAX_TEXT_FILE_BYTES = 16 * 1024 * 1024 # 16 MiB per text file +HARD_MAX_BINARY_FILE_BYTES = 256 * 1024 * 1024 # 256 MiB per .node etc +HARD_MAX_TOTAL_BYTES = 512 * 1024 * 1024 # 512 MiB cumulative +HARD_MAX_MEMBERS = 50_000 # entries per tarball +HARD_HTTP_TIMEOUT_S = 60 # per request + +# Native-binary / compiled-asset suffixes that bypass the text cap. +# This is the SUFFIX shortlist; the content-magic check below covers +# extensionless executables (biome) and versioned shared libraries +# (libvips-cpp.so.8.17.3) that the suffix list misses. +_BINARY_SUFFIXES = ( + ".node", + ".wasm", + ".so", + ".dll", + ".dylib", + ".exe", + ".a", + ".lib", + ".o", + ".obj", + ".bin", + ".dat", + ".woff", + ".woff2", + ".ttf", + ".otf", + ".eot", + ".png", + ".jpg", + ".jpeg", + ".gif", + ".webp", + ".ico", + ".mp3", + ".mp4", + ".webm", + ".zip", + ".tar", + ".gz", + ".tgz", + ".xz", + ".bz2", +) + +# Versioned shared libraries: libfoo.so.1.2.3 / libfoo.dylib.1.2. +_VERSIONED_LIB = re.compile( + r"\.(?:so|dylib)(?:\.\d+)+$", + re.IGNORECASE, +) + +# Magic numbers at offset 0 that identify common executable formats. +# We sniff the first ~16 bytes of every member to catch extensionless +# binaries (eg `package/biome`, `package/bin/foo`). +_BINARY_MAGICS = ( + b"\x7fELF", # ELF (Linux executable / .so) + b"MZ", # PE / .exe / .dll (DOS header prefix) + b"\xfe\xed\xfa\xce", # Mach-O 32 BE + b"\xfe\xed\xfa\xcf", # Mach-O 64 BE + b"\xce\xfa\xed\xfe", # Mach-O 32 LE + b"\xcf\xfa\xed\xfe", # Mach-O 64 LE + b"\xca\xfe\xba\xbe", # Mach-O fat / Java class (also starts with this) + b"\x00asm", # WASM + b"PK\x03\x04", # ZIP / JAR / nupkg / xpi + b"PK\x05\x06", # ZIP (empty) + b"\x1f\x8b", # gzip + b"BZh", # bzip2 + b"\xfd7zXZ", # xz + b"7z\xbc\xaf\x27\x1c", # 7zip + b"\x89PNG", # PNG + b"\xff\xd8\xff", # JPEG + b"GIF8", # GIF + b"RIFF", # WAV / WEBP / AVI container + b"\x00\x00\x01\x00", # ICO + b"OggS", # Ogg + b"\x1aE\xdf\xa3", # Matroska / WebM +) + + +def _looks_binary(name: str, header: bytes) -> bool: + """True if `name` or first bytes suggest a non-text file.""" + lower = name.lower() + if lower.endswith(_BINARY_SUFFIXES): + return True + if _VERSIONED_LIB.search(lower): + return True + for magic in _BINARY_MAGICS: + if header.startswith(magic): + return True + # Null-byte density: real text files almost never carry NULs. + if header and (header.count(b"\x00") / len(header)) > 0.02: + return True + return False + + +ALLOWED_DOWNLOAD_HOST = "registry.npmjs.org" + +# ───────────────────────────────────────────────────────────────────── +# Severities + finding shape (mirrors scripts/scan_packages.py). +# ───────────────────────────────────────────────────────────────────── +CRITICAL = "CRITICAL" +HIGH = "HIGH" +MEDIUM = "MEDIUM" +INFO = "INFO" +_SEVERITY_RANK = {CRITICAL: 0, HIGH: 1, MEDIUM: 2, INFO: 3} + + +@dataclass +class Finding: + severity: str + package: str # name@version + filename: str # relative path inside the tarball + pattern: str # what matched + evidence: str = "" # short surrounding snippet + detail: str = "" # human-readable description + + def __str__(self) -> str: + head = f" [{self.severity}] {self.package} :: {self.filename}" + body = f" pattern: {self.pattern}" + if self.detail: + body += f"\n detail: {self.detail}" + if self.evidence: + ev = self.evidence + if len(ev) > 240: + ev = ev[:240] + "..." + body += f"\n evidence: {ev!r}" + return f"{head}\n{body}" + + +@dataclass +class PackageEntry: + name: str + version: str + resolved: str + integrity: str | None + lockfile_key: str + + @property + def display(self) -> str: + return f"{self.name}@{self.version}" + + +# ───────────────────────────────────────────────────────────────────── +# IOC patterns. Two flavours: +# - HOSTS / TOKEN_PATHS: high-confidence substrings; near-zero FP rate +# - JS_PATTERNS / SCRIPT_PATTERNS: regex; tuned to recent campaigns +# Keep this list short and factual. Speculative patterns spam the +# false-positive ledger and dull the signal. +# ───────────────────────────────────────────────────────────────────── + + +# Substring (case-sensitive) -> (severity, detail). +KNOWN_IOC_STRINGS: dict[str, tuple[str, str]] = { + # Shai-Hulud TanStack wave (2026-05-11, GHSA-g7cv-rxg3-hmpx). + "router_init.js": (HIGH, "filename associated with TanStack worm"), + "tanstack_runner.js": (HIGH, "filename associated with TanStack worm"), + "router_runtime.js": (HIGH, "filename associated with TanStack worm"), + "A Mini Shai-Hulud has Appeared": ( + CRITICAL, + "TanStack worm campaign stdout marker", + ), + "github:tanstack/router#79ac49eedf774dd4b0cfa308722bc463cfe5885c": ( + CRITICAL, + "TanStack worm dropper pinned commit", + ), + # Exfil hosts observed across both Shai-Hulud waves. + "filev2.getsession.org": (CRITICAL, "exfiltration C2 host"), + "getsession.org/file/": (CRITICAL, "exfiltration C2 endpoint"), + # Mini Shai-Hulud May-12 2026 wave additions. + "git-tanstack.com": (CRITICAL, "May-12 dropper host"), + "transformers.pyz": (HIGH, "May-12 PyPI dropper artifact"), + "/tmp/transformers.pyz": (CRITICAL, "May-12 dropper drop path"), + "With Love TeamPCP": (CRITICAL, "May-12 campaign signature"), + "We've been online over 2 hours": (CRITICAL, "May-12 campaign signature"), + # Aikido (May-12 wave): payload SHA-256 hashes published in IOCs. + "ab4fcadaec49c03278063dd269ea5eef82d24f2124a8e15d7b90f2fa8601266c": ( + HIGH, + "router_init.js payload SHA-256", + ), + "2ec78d556d696e208927cc503d48e4b5eb56b31abc2870c2ed2e98d6be27fc96": ( + HIGH, + "tanstack_runner.js payload SHA-256", + ), + # The new dependency vector: optional dep -> Bun-executed prepare script. + "bun run tanstack_runner.js": ( + CRITICAL, + "TanStack-wave Bun prepare-script dropper invocation", + ), + "@tanstack/setup": ( + CRITICAL, + "TanStack-wave optional-dep dropper carrier (no legit pkg of this name)", + ), +} + +# Hard pin-blocks for publicly confirmed malicious versions. +# name -> {malicious_versions...}. A match short-circuits the scan +# at the lockfile-walk stage; no tarball is fetched. +# keep in sync with scripts/lockfile_supply_chain_audit.py +BLOCKED_NPM_VERSIONS: dict[str, set[str]] = { + # GHSA-g7cv-rxg3-hmpx -- TanStack May-11 2026 (84 versions). + "@tanstack/arktype-adapter": {"1.166.12", "1.166.15"}, + "@tanstack/eslint-plugin-router": {"1.161.9", "1.161.12"}, + "@tanstack/eslint-plugin-start": {"0.0.4", "0.0.7"}, + "@tanstack/history": {"1.161.9", "1.161.12"}, + "@tanstack/nitro-v2-vite-plugin": {"1.154.12", "1.154.15"}, + "@tanstack/react-router": {"1.169.5", "1.169.8"}, + "@tanstack/react-router-devtools": {"1.166.16", "1.166.19"}, + "@tanstack/react-router-ssr-query": {"1.166.15", "1.166.18"}, + "@tanstack/react-start": {"1.167.68", "1.167.71"}, + "@tanstack/react-start-client": {"1.166.51", "1.166.54"}, + "@tanstack/react-start-rsc": {"0.0.47", "0.0.50"}, + "@tanstack/react-start-server": {"1.166.55", "1.166.58"}, + "@tanstack/router-cli": {"1.166.46", "1.166.49"}, + "@tanstack/router-core": {"1.169.5", "1.169.8"}, + "@tanstack/router-devtools": {"1.166.16", "1.166.19"}, + "@tanstack/router-devtools-core": {"1.167.6", "1.167.9"}, + "@tanstack/router-generator": {"1.166.45", "1.166.48"}, + "@tanstack/router-plugin": {"1.167.38", "1.167.41"}, + "@tanstack/router-ssr-query-core": {"1.168.3", "1.168.6"}, + "@tanstack/router-utils": {"1.161.11", "1.161.14"}, + "@tanstack/router-vite-plugin": {"1.166.53", "1.166.56"}, + "@tanstack/solid-router": {"1.169.5", "1.169.8"}, + "@tanstack/solid-router-devtools": {"1.166.16", "1.166.19"}, + "@tanstack/solid-router-ssr-query": {"1.166.15", "1.166.18"}, + "@tanstack/solid-start": {"1.167.65", "1.167.68"}, + "@tanstack/solid-start-client": {"1.166.50", "1.166.53"}, + "@tanstack/solid-start-server": {"1.166.54", "1.166.57"}, + "@tanstack/start-client-core": {"1.168.5", "1.168.8"}, + "@tanstack/start-fn-stubs": {"1.161.9", "1.161.12"}, + "@tanstack/start-plugin-core": {"1.169.23", "1.169.26"}, + "@tanstack/start-server-core": {"1.167.33", "1.167.36"}, + "@tanstack/start-static-server-functions": {"1.166.44", "1.166.47"}, + "@tanstack/start-storage-context": {"1.166.38", "1.166.41"}, + "@tanstack/valibot-adapter": {"1.166.12", "1.166.15"}, + "@tanstack/virtual-file-routes": {"1.161.10", "1.161.13"}, + "@tanstack/vue-router": {"1.169.5", "1.169.8"}, + "@tanstack/vue-router-devtools": {"1.166.16", "1.166.19"}, + "@tanstack/vue-router-ssr-query": {"1.166.15", "1.166.18"}, + "@tanstack/vue-start": {"1.167.61", "1.167.64"}, + "@tanstack/vue-start-client": {"1.166.46", "1.166.49"}, + "@tanstack/vue-start-server": {"1.166.50", "1.166.53"}, + "@tanstack/zod-adapter": {"1.166.12", "1.166.15"}, + # Mini Shai-Hulud May-12 wave: OpenSearch JS client. + "@opensearch-project/opensearch": {"3.5.3", "3.6.2", "3.7.0", "3.8.0"}, + # Mini Shai-Hulud May-12 wave: @squawk/* (22 packages, 5 versions each; + # https://safedep.io/mass-npm-supply-chain-attack-tanstack-mistral/). + "@squawk/airport-data": {"0.7.4", "0.7.5", "0.7.6", "0.7.7", "0.7.8"}, + "@squawk/airports": {"0.6.2", "0.6.3", "0.6.4", "0.6.5", "0.6.6"}, + "@squawk/airspace": {"0.8.1", "0.8.2", "0.8.3", "0.8.4", "0.8.5"}, + "@squawk/airspace-data": {"0.5.3", "0.5.4", "0.5.5", "0.5.6", "0.5.7"}, + "@squawk/airway-data": {"0.5.4", "0.5.5", "0.5.6", "0.5.7", "0.5.8"}, + "@squawk/airways": {"0.4.2", "0.4.3", "0.4.4", "0.4.5", "0.4.6"}, + "@squawk/fix-data": {"0.6.4", "0.6.5", "0.6.6", "0.6.7", "0.6.8"}, + "@squawk/fixes": {"0.3.2", "0.3.3", "0.3.4", "0.3.5", "0.3.6"}, + "@squawk/flight-math": {"0.5.4", "0.5.5", "0.5.6", "0.5.7", "0.5.8"}, + "@squawk/flightplan": {"0.5.2", "0.5.3", "0.5.4", "0.5.5", "0.5.6"}, + "@squawk/geo": {"0.4.4", "0.4.5", "0.4.6", "0.4.7", "0.4.8"}, + "@squawk/icao-registry": {"0.5.2", "0.5.3", "0.5.4", "0.5.5", "0.5.6"}, + "@squawk/icao-registry-data": {"0.8.4", "0.8.5", "0.8.6", "0.8.7", "0.8.8"}, + "@squawk/mcp": {"0.9.1", "0.9.2", "0.9.3", "0.9.4", "0.9.5"}, + "@squawk/navaid-data": {"0.6.4", "0.6.5", "0.6.6", "0.6.7", "0.6.8"}, + "@squawk/navaids": {"0.4.2", "0.4.3", "0.4.4", "0.4.5", "0.4.6"}, + "@squawk/notams": {"0.3.6", "0.3.7", "0.3.8", "0.3.9", "0.3.10"}, + "@squawk/procedure-data": {"0.7.3", "0.7.4", "0.7.5", "0.7.6", "0.7.7"}, + "@squawk/procedures": {"0.5.2", "0.5.3", "0.5.4", "0.5.5", "0.5.6"}, + "@squawk/types": {"0.8.1", "0.8.2", "0.8.3", "0.8.4", "0.8.5"}, + "@squawk/units": {"0.4.3", "0.4.4", "0.4.5", "0.4.6", "0.4.7"}, + "@squawk/weather": {"0.5.6", "0.5.7", "0.5.8", "0.5.9", "0.5.10"}, + # Mini Shai-Hulud May-12 wave: @uipath/* (64 packages, single version each; + # https://www.aikido.dev/blog/mini-shai-hulud-is-back-tanstack-compromised). + "@uipath/apollo-react": {"4.24.5"}, + "@uipath/apollo-wind": {"2.16.2"}, + "@uipath/cli": {"1.0.1"}, + "@uipath/rpa-tool": {"0.9.5"}, + "@uipath/apollo-core": {"5.9.2"}, + "@uipath/filesystem": {"1.0.1"}, + "@uipath/solutionpackager-tool-core": {"0.0.34"}, + "@uipath/solution-tool": {"1.0.1"}, + "@uipath/maestro-tool": {"1.0.1"}, + "@uipath/codedapp-tool": {"1.0.1"}, + "@uipath/agent-tool": {"1.0.1"}, + "@uipath/orchestrator-tool": {"1.0.1"}, + "@uipath/integrationservice-tool": {"1.0.2"}, + "@uipath/rpa-legacy-tool": {"1.0.1"}, + "@uipath/vertical-solutions-tool": {"1.0.1"}, + "@uipath/flow-tool": {"1.0.2"}, + "@uipath/codedagent-tool": {"1.0.1"}, + "@uipath/common": {"1.0.1"}, + "@uipath/resource-tool": {"1.0.1"}, + "@uipath/auth": {"1.0.1"}, + "@uipath/docsai-tool": {"1.0.1"}, + "@uipath/case-tool": {"1.0.1"}, + "@uipath/api-workflow-tool": {"1.0.1"}, + "@uipath/test-manager-tool": {"1.0.2"}, + "@uipath/robot": {"1.3.4"}, + "@uipath/traces-tool": {"1.0.1"}, + "@uipath/agent-sdk": {"1.0.2"}, + "@uipath/integrationservice-sdk": {"1.0.2"}, + "@uipath/maestro-sdk": {"1.0.1"}, + "@uipath/data-fabric-tool": {"1.0.2"}, + "@uipath/tasks-tool": {"1.0.1"}, + "@uipath/insights-tool": {"1.0.1"}, + "@uipath/insights-sdk": {"1.0.1"}, + "@uipath/uipath-python-bridge": {"1.0.1"}, + "@uipath/ap-chat": {"1.5.7"}, + "@uipath/project-packager": {"1.1.16"}, + "@uipath/packager-tool-case": {"0.0.9"}, + "@uipath/packager-tool-workflowcompiler-browser": {"0.0.34"}, + "@uipath/packager-tool-connector": {"0.0.19"}, + "@uipath/packager-tool-workflowcompiler": {"0.0.16"}, + "@uipath/packager-tool-webapp": {"1.0.6"}, + "@uipath/packager-tool-apiworkflow": {"0.0.19"}, + "@uipath/packager-tool-functions": {"0.1.1"}, + "@uipath/widget.sdk": {"1.2.3"}, + "@uipath/resources-tool": {"0.1.11"}, + "@uipath/agent.sdk": {"0.0.18"}, + "@uipath/codedagents-tool": {"0.1.12"}, + "@uipath/aops-policy-tool": {"0.3.1"}, + "@uipath/solution-packager": {"0.0.35"}, + "@uipath/packager-tool-bpmn": {"0.0.9"}, + "@uipath/packager-tool-flow": {"0.0.19"}, + "@uipath/telemetry": {"0.0.7"}, + "@uipath/tool-workflowcompiler": {"0.0.12"}, + "@uipath/vss": {"0.1.6"}, + "@uipath/solutionpackager-sdk": {"1.0.11"}, + "@uipath/ui-widgets-multi-file-upload": {"1.0.1"}, + "@uipath/access-policy-tool": {"0.3.1"}, + "@uipath/context-grounding-tool": {"0.1.1"}, + "@uipath/gov-tool": {"0.3.1"}, + "@uipath/admin-tool": {"0.1.1"}, + "@uipath/identity-tool": {"0.1.1"}, + "@uipath/llmgw-tool": {"1.0.1"}, + "@uipath/resourcecatalog-tool": {"0.1.1"}, + "@uipath/functions-tool": {"1.0.1"}, + "@uipath/access-policy-sdk": {"0.3.1"}, + "@uipath/platform-tool": {"1.0.1"}, + # Mini Shai-Hulud May-12 wave: @mistralai/* (npm) — separate from PyPI mistralai + # (https://www.aikido.dev/blog/mini-shai-hulud-is-back-tanstack-compromised). + "@mistralai/mistralai": {"2.2.2", "2.2.3", "2.2.4"}, + "@mistralai/mistralai-gcp": {"1.7.1", "1.7.2", "1.7.3"}, + "@mistralai/mistralai-azure": {"1.7.1", "1.7.2", "1.7.3"}, + # Mini Shai-Hulud May-12 wave: @tallyui/* (30 entries, 10 packages) + # (Aikido enumeration). + "@tallyui/components": {"1.0.1", "1.0.2", "1.0.3"}, + "@tallyui/connector-medusa": {"1.0.1", "1.0.2", "1.0.3"}, + "@tallyui/connector-shopify": {"1.0.1", "1.0.2", "1.0.3"}, + "@tallyui/connector-vendure": {"1.0.1", "1.0.2", "1.0.3"}, + "@tallyui/connector-woocommerce": {"1.0.1", "1.0.2", "1.0.3"}, + "@tallyui/core": {"0.2.1", "0.2.2", "0.2.3"}, + "@tallyui/database": {"1.0.1", "1.0.2", "1.0.3"}, + "@tallyui/pos": {"0.1.1", "0.1.2", "0.1.3"}, + "@tallyui/storage-sqlite": {"0.2.1", "0.2.2", "0.2.3"}, + "@tallyui/theme": {"0.2.1", "0.2.2", "0.2.3"}, + # Mini Shai-Hulud May-12 wave: @beproduct/nestjs-auth (18 versions) + # (Aikido enumeration). + "@beproduct/nestjs-auth": { + "0.1.2", + "0.1.3", + "0.1.4", + "0.1.5", + "0.1.6", + "0.1.7", + "0.1.8", + "0.1.9", + "0.1.10", + "0.1.11", + "0.1.12", + "0.1.13", + "0.1.14", + "0.1.15", + "0.1.16", + "0.1.17", + "0.1.18", + "0.1.19", + }, + # Mini Shai-Hulud May-12 wave: @draftlab/* + @draftauth/* + # (Aikido enumeration). + "@draftauth/client": {"0.2.1", "0.2.2"}, + "@draftauth/core": {"0.13.1", "0.13.2"}, + "@draftlab/auth": {"0.24.1", "0.24.2"}, + "@draftlab/auth-router": {"0.5.1", "0.5.2"}, + "@draftlab/db": {"0.16.1"}, + # Mini Shai-Hulud May-12 wave: @taskflow-corp/cli + @tolka/cli + # (Aikido enumeration). + "@taskflow-corp/cli": {"0.1.24", "0.1.25", "0.1.26", "0.1.27", "0.1.28", "0.1.29"}, + "@tolka/cli": {"1.0.2", "1.0.3", "1.0.4", "1.0.5", "1.0.6"}, + # Mini Shai-Hulud May-12 wave: @ml-toolkit-ts/* + @mesadev/* + @dirigible-ai/sdk + @supersurkhet/* + # (Aikido enumeration). + "@dirigible-ai/sdk": {"0.6.2", "0.6.3"}, + "@mesadev/rest": {"0.28.3"}, + "@mesadev/saguaro": {"0.4.22"}, + "@mesadev/sdk": {"0.28.3"}, + "@ml-toolkit-ts/preprocessing": {"1.0.2", "1.0.3"}, + "@ml-toolkit-ts/xgboost": {"1.0.3", "1.0.4"}, + "@supersurkhet/cli": {"0.0.2", "0.0.3", "0.0.4", "0.0.5", "0.0.6", "0.0.7"}, + "@supersurkhet/sdk": {"0.0.2", "0.0.3", "0.0.4", "0.0.5", "0.0.6", "0.0.7"}, + # Mini Shai-Hulud May-12 wave: Unscoped packages (10 entries) + # (Aikido enumeration). + "safe-action": {"0.8.3", "0.8.4"}, + "ts-dna": {"3.0.1", "3.0.2", "3.0.3", "3.0.4"}, + "cross-stitch": {"1.1.3", "1.1.4", "1.1.5", "1.1.6"}, + "cmux-agent-mcp": {"0.1.3", "0.1.4", "0.1.5", "0.1.6", "0.1.7", "0.1.8"}, + "agentwork-cli": {"0.1.4", "0.1.5"}, + "git-branch-selector": {"1.3.3", "1.3.4", "1.3.5", "1.3.6", "1.3.7"}, + "wot-api": {"0.8.1", "0.8.2", "0.8.3", "0.8.4"}, + "git-git-git": {"1.0.8", "1.0.9", "1.0.10", "1.0.11", "1.0.12"}, + "nextmove-mcp": {"0.1.3", "0.1.4", "0.1.5", "0.1.6", "0.1.7"}, + "ml-toolkit-ts": {"1.0.4", "1.0.5"}, + # Cross-ecosystem Mini Shai-Hulud (Apr-30 wave): npm counterpart of + # PyPI lightning 2.6.2/2.6.3. Same threat actor (TeamPCP) per Semgrep, + # Aikido, OX Security, Resecurity. Safe version: 7.0.3 and earlier. + "intercom-client": {"7.0.4"}, +} + +# Cloud / k8s / CI credential surfaces. A bare substring match here +# false-positives on DEFENSIVE code -- e.g. langchain ships an SSRF +# protection module with a literal blocklist of IMDS IPs. We split +# these into two tiers: +# +# ALWAYS_BAD: substrings with no legitimate use anywhere in a +# dependency. A bare match is enough. +# +# NEEDS_CONTEXT: hosts/paths that DO appear legitimately in +# defensive code. We only fire when they co-occur with a fetch +# verb or appear inside an http URL -- that is the structural +# difference between "blocked address constant" and "exfil +# target". +# +# The dispatch lives in `_scan_cred_surface` below. + +CRED_HOST_ALWAYS_BAD: tuple[tuple[str, str], ...] = ( + ("registry.npmjs.org/-/npm/v1/tokens", "npm publish-token enumeration endpoint"), + ("ACTIONS_ID_TOKEN_REQUEST_URL", "GitHub Actions OIDC token-exchange endpoint env"), + ("ACTIONS_ID_TOKEN_REQUEST_TOKEN", "GitHub Actions OIDC token-exchange token env"), +) + +# Hosts that need fetch-verb or URL-scheme context to be malicious. +CRED_HOST_NEEDS_CONTEXT: tuple[tuple[str, str], ...] = ( + ("169.254.169.254", "AWS / GCP / Azure instance metadata service (IMDS)"), + ("169.254.170.2", "ECS task metadata service"), + ("metadata.google.internal", "GCE metadata service"), + ("vault.svc.cluster.local", "in-cluster HashiCorp Vault endpoint"), + ( + "/var/run/secrets/kubernetes.io/serviceaccount", + "Kubernetes ServiceAccount token path", + ), +) + +# Credentials a frontend package should NEVER need to read. Bare +# substring match is too noisy (object-treeify ships a `docker` dev +# script that mounts ~/.npmrc -- legitimate dev tooling, never run +# at install time). We instead surface these only when they appear +# inside a LIFECYCLE script (preinstall / install / postinstall / +# prepare), which is the only path that runs automatically on +# `npm ci`. See `scan_package_json` below. +CRED_PATH_SUBSTRINGS: tuple[tuple[str, str], ...] = ( + ("/.npmrc", "npm credentials file"), + ("/.aws/credentials", "AWS shared credentials file"), + ("/.ssh/id_rsa", "SSH private key"), + ("/.ssh/id_ed25519", "SSH private key"), + ("/.docker/config.json", "Docker registry credentials"), + ("/.kube/config", "Kubernetes kubeconfig"), +) + +# Fetch verbs whose presence near a metadata host upgrades a bare +# substring hit into an actionable finding. +_FETCH_VERBS_PAT = ( + r"(?:fetch|axios|XMLHttpRequest|got\b|undici|" + r"http\.get|https\.get|http\.request|https\.request|" + r"new\s+URL|url\.parse|net\.connect|" + r"\.request\s*\(|\.get\s*\(\s*['\"]\s*https?://)" +) + +# JS regex patterns (compile lazily). +_JS_FETCH_EVAL = re.compile( + r"""(?xs) + (?: + Function\s*\(\s*['"`] # new Function("...") + | eval\s*\(\s*['"`] + | \(\s*0\s*,\s*eval\s*\)\s*\( + ) + .{0,200} + (?:atob\s*\(|Buffer\s*\.from\s*\([^)]+,\s*['"]base64) + """, +) + +# `process.env.GITHUB_TOKEN` / `NPM_TOKEN` / `AWS_*` access in +# top-level / install-time code is suspicious. We also catch +# `os.environ["GITHUB_TOKEN"]` for the rare Python-in-npm postinstall. +_JS_ENV_TOKEN = re.compile( + r"""(process\.env\.|os\.environ\[?['"])(?: + GITHUB_TOKEN | GH_TOKEN | NPM_TOKEN | NODE_AUTH_TOKEN + | AWS_ACCESS_KEY_ID | AWS_SECRET_ACCESS_KEY | AWS_SESSION_TOKEN + | GOOGLE_APPLICATION_CREDENTIALS + | DOCKER_AUTH_CONFIG | VAULT_TOKEN + )['"]?\]?""", + re.VERBOSE, +) + +# Suspicious lifecycle-script payloads. Anything in a package.json +# `scripts` field that wgets/curls an external resource and executes +# it. We do NOT block ALL curl/wget in scripts (some legit packages +# fetch test fixtures into devDependencies), but we DO block the +# fetch+exec chain. +_LIFECYCLE_FETCH_EXEC = re.compile( + r"""(?xs) + (?:curl|wget|fetch|http\.get|axios\.get)\s+ # fetch verb + .{0,200} + (?:\|\s*(?:sh|bash|node|python|eval)\b # pipe to interpreter + | \&\&\s*(?:sh|bash|node|python|eval)\b # &&-chain to interpreter + | -o\s+\S+\s*&&\s*(?:sh|bash|node|python) # download then run + | --post-file\s+ + | \$\(.*\) # command-sub of fetched content + ) + """, +) + +# Obfuscation: large JS file that is mostly one line of base64-ish +# blob with a Function() / eval() bookend. Tuned against the +# router_init.js shape (2.3 MB obfuscated single-blob). +_OBFUSC_BLOB = re.compile( + r"""(?xs) + (?:Function|eval)\s*\(\s*['"`]? + [A-Za-z0-9+/=_-]{2048,} # >=2 KiB of b64-ish + """, +) + + +# ───────────────────────────────────────────────────────────────────── +# Lockfile parsing. +# ───────────────────────────────────────────────────────────────────── + + +def parse_lockfile(path: Path) -> tuple[list[PackageEntry], list[Finding]]: + """Return (entries, structural_findings). + + Structural findings here are HIGH-severity refusals that should + short-circuit the scan -- a lockfile with non-registry resolved + URLs is itself a finding (covered by scripts/lockfile_supply_chain + _audit.py in detail; we surface a summary here so this scanner is + standalone-runnable). + """ + entries: list[PackageEntry] = [] + findings: list[Finding] = [] + + try: + lock = json.loads(path.read_text(encoding = "utf-8")) + except (OSError, json.JSONDecodeError) as exc: + findings.append( + Finding( + severity = CRITICAL, + package = "", + filename = str(path), + pattern = "lockfile-unreadable", + detail = f"could not parse: {exc}", + ) + ) + return entries, findings + + if lock.get("lockfileVersion") not in (2, 3): + findings.append( + Finding( + severity = HIGH, + package = "", + filename = str(path), + pattern = "unsupported-lockfile-version", + detail = ( + f"only lockfileVersion 2 or 3 supported; got " + f"{lock.get('lockfileVersion')!r}" + ), + ) + ) + return entries, findings + + for key, entry in (lock.get("packages") or {}).items(): + if key == "" or entry.get("link"): + continue + # Nested fold-ins (deps inside another package's node_modules/) + # are covered by the parent tarball's integrity. Skip. + if key.count("/node_modules/") >= 1: + continue + resolved = entry.get("resolved") + if not resolved: + continue + # Strict registry origin check. lockfile_supply_chain_audit + # already catches this; double-defend here so this scanner + # cannot be tricked into fetching from an attacker-chosen URL. + parsed = urllib.parse.urlparse(resolved) + if parsed.scheme != "https" or parsed.hostname != ALLOWED_DOWNLOAD_HOST: + findings.append( + Finding( + severity = CRITICAL, + package = key, + filename = str(path), + pattern = "non-registry-resolved-url", + detail = ( + f"resolved={resolved!r}; only " + f"https://{ALLOWED_DOWNLOAD_HOST}/ is " + "permitted. Refusing to download." + ), + ) + ) + continue + integrity = entry.get("integrity") + if not integrity: + findings.append( + Finding( + severity = HIGH, + package = key, + filename = str(path), + pattern = "missing-integrity-hash", + detail = "no `integrity` field; cannot verify download", + ) + ) + continue + # node_modules/@scope/name -> @scope/name; node_modules/name -> name + nm = "node_modules/" + name = key[len(nm) :] if key.startswith(nm) else key + version = entry.get("version") or "" + entries.append( + PackageEntry( + name = name, + version = version, + resolved = resolved, + integrity = integrity, + lockfile_key = key, + ) + ) + return entries, findings + + +# ───────────────────────────────────────────────────────────────────── +# Tarball download (registry-only, size-capped, integrity-verified). +# ───────────────────────────────────────────────────────────────────── + + +def _decode_integrity(integrity: str) -> tuple[str, bytes] | None: + """Parse SRI integrity 'sha512-' -> (algo, digest_bytes).""" + if "-" not in integrity: + return None + algo, b64 = integrity.split("-", 1) + algo = algo.strip().lower() + if algo not in ("sha256", "sha384", "sha512"): + return None + try: + digest = _b64.b64decode(b64, validate = True) + except Exception: + return None + return algo, digest + + +def download_tarball( + entry: PackageEntry, + dest: Path, + *, + timeout: float = HARD_HTTP_TIMEOUT_S, + max_bytes: int = HARD_MAX_TARBALL_BYTES, +) -> tuple[Path, str | None]: + """Stream-download entry.resolved to dest. Verify SRI integrity. + + Returns (downloaded_path, error_or_none). On any error the + returned path may not exist. Network access is restricted to + https://{ALLOWED_DOWNLOAD_HOST}/ -- the caller passes a Request + we already validated. + """ + # Re-assert hostname; the entry was validated at parse time but a + # defence-in-depth check here means a future refactor cannot + # accidentally bypass it. + parsed = urllib.parse.urlparse(entry.resolved) + if parsed.scheme != "https" or parsed.hostname != ALLOWED_DOWNLOAD_HOST: + return dest, (f"refused download from non-allowlisted URL {entry.resolved!r}") + + decoded = _decode_integrity(entry.integrity or "") + if decoded is None: + return dest, f"unparseable integrity field {entry.integrity!r}" + algo, expected_digest = decoded + h = hashlib.new(algo) + + req = urllib.request.Request( + entry.resolved, + headers = { + "User-Agent": "unsloth-scan-npm-packages/1.0 (+supply-chain audit)", + "Accept": "application/octet-stream", + }, + method = "GET", + ) + try: + with urllib.request.urlopen(req, timeout = timeout) as r: + # Advertised length, if any. + cl = r.headers.get("Content-Length") + if cl is not None: + try: + cl_int = int(cl) + if cl_int > max_bytes: + return dest, (f"Content-Length {cl_int} > cap {max_bytes}") + except ValueError: + pass + written = 0 + with open(dest, "wb") as out: + while True: + chunk = r.read(64 * 1024) + if not chunk: + break + written += len(chunk) + if written > max_bytes: + return dest, ( + f"download exceeded cap {max_bytes} bytes " + f"after {written} bytes" + ) + h.update(chunk) + out.write(chunk) + except Exception as exc: + return dest, f"download failed: {exc}" + + actual = h.digest() + if actual != expected_digest: + return dest, ( + f"integrity mismatch: expected {algo}={_b64.b64encode(expected_digest).decode()!r}, " + f"got {algo}={_b64.b64encode(actual).decode()!r}" + ) + return dest, None + + +# ───────────────────────────────────────────────────────────────────── +# Safe tar extraction. Every Tarfile member is policed before write. +# ───────────────────────────────────────────────────────────────────── + + +def _is_within(root: Path, candidate: Path) -> bool: + try: + return candidate.resolve().is_relative_to(root.resolve()) + except (AttributeError, ValueError): + # Python <3.9 fallback (we target 3.10+ but be defensive). + try: + candidate.resolve().relative_to(root.resolve()) + return True + except Exception: + return False + + +def safe_extract( + tarball_path: Path, + extract_root: Path, + *, + max_total_bytes: int = HARD_MAX_TOTAL_BYTES, + max_members: int = HARD_MAX_MEMBERS, +) -> str | None: + """Extract tarball_path under extract_root with policed members. + + Returns None on success, or a string describing the refusal. + Streams via `r|gz` so we can abort mid-extraction without having + materialised the rest of the archive. + """ + extract_root.mkdir(parents = True, exist_ok = True) + total = 0 + count = 0 + try: + # Open in streaming mode so we never seek backwards in the + # input. `r|gz` rejects malformed gzip frames immediately. + with tarfile.open(tarball_path, mode = "r|gz") as tf: + for member in tf: + count += 1 + if count > max_members: + return f"member count {count} exceeded cap {max_members}" + name = member.name + # Reject obvious path-escape. + if name.startswith("/") or ".." in Path(name).parts: + return f"refused unsafe member name {name!r}" + # Reject device files, FIFOs, sockets, symlinks, hardlinks. + if member.issym() or member.islnk(): + return f"refused link member {name!r} (sym/lnk)" + if member.isdev() or member.isfifo(): + return f"refused special member {name!r}" + # Cumulative cap is checked against DECLARED size up + # front to short-circuit obvious bombs without reading + # the body. + declared = max(member.size, 0) + if declared > HARD_MAX_BINARY_FILE_BYTES: + return ( + f"member {name!r} declared size {declared} > " + f"absolute cap {HARD_MAX_BINARY_FILE_BYTES}" + ) + if total + declared > max_total_bytes: + return ( + f"cumulative bytes {total + declared} > cap " + f"{max_total_bytes} at {name!r}" + ) + # Strip leading "package/" -- the npm convention. We do + # NOT trust npm to be right, so we explicitly resolve + # the destination and refuse anything that escapes. + dest = extract_root / name + if not _is_within(extract_root, dest): + return f"refused escape: {name!r} resolved outside root" + if member.isdir(): + dest.mkdir(parents = True, exist_ok = True) + continue + if not member.isfile(): + # Anything we didn't classify above is unknown. + return f"refused unknown member type for {name!r}" + dest.parent.mkdir(parents = True, exist_ok = True) + src = tf.extractfile(member) + if src is None: + continue + # Sniff first 16 bytes to classify text vs binary. + # Text-cap members get the tight 16 MiB limit; binary + # members (executables, .node, .wasm, native libs) + # get the generous binary cap. We bound BOTH cases. + header = src.read(16) + is_binary = _looks_binary(name, header) + file_cap = ( + HARD_MAX_BINARY_FILE_BYTES + if is_binary + else HARD_MAX_TEXT_FILE_BYTES + ) + if declared > file_cap: + return ( + f"member {name!r} declared size {declared} > " + f"cap {file_cap} ({'binary' if is_binary else 'text'})" + ) + # Read remainder, bounded. + remainder_cap = file_cap - len(header) + rest = src.read(remainder_cap + 1) + data = header + rest + if len(data) > file_cap: + return ( + f"member {name!r} body exceeded declared size cap " + f"({'binary' if is_binary else 'text'})" + ) + total += len(data) + # Write with restrictive mode (rw-r--r--) so even if + # someone runs the extract dir nothing is executable. + with open(dest, "wb") as out: + out.write(data) + os.chmod(dest, 0o644) + except tarfile.TarError as exc: + return f"tar parse error: {exc}" + except Exception as exc: + return f"unexpected extract error: {exc!r}" + return None + + +# ───────────────────────────────────────────────────────────────────── +# Content scanning. +# ───────────────────────────────────────────────────────────────────── + + +def _evidence(text: str, pat: re.Pattern, max_chars: int = 200) -> str: + m = pat.search(text) + if not m: + return "" + start = max(0, m.start() - 30) + end = min(len(text), m.end() + 30) + snippet = text[start:end].replace("\n", " ") + if len(snippet) > max_chars: + snippet = snippet[:max_chars] + "..." + return snippet + + +LIFECYCLE_HOOKS = ("preinstall", "install", "postinstall", "prepare") + + +def scan_package_json( + pkg: PackageEntry, + rel: str, + text: str, +) -> list[Finding]: + findings: list[Finding] = [] + try: + meta = json.loads(text) + except Exception: + return findings + if not isinstance(meta, dict): + return findings + scripts = meta.get("scripts") or {} + if not isinstance(scripts, dict): + return findings + for hook in LIFECYCLE_HOOKS: + body = scripts.get(hook) + if not isinstance(body, str): + continue + if _LIFECYCLE_FETCH_EXEC.search(body): + findings.append( + Finding( + severity = CRITICAL, + package = pkg.display, + filename = rel, + pattern = f"lifecycle-fetch-exec ({hook})", + evidence = body, + detail = ( + f"`scripts.{hook}` fetches an external " + "resource and pipes/chains it to an " + "interpreter; this is the install-time RCE " + "vector. Refusing to install." + ), + ) + ) + # Credential file paths inside a lifecycle script are + # exfiltration prep -- npm runs these scripts automatically + # on `npm ci`. Manual `scripts.*` entries (like a `docker` + # dev script) are out of scope: npm does not run them. + for path_substr, why in CRED_PATH_SUBSTRINGS: + if path_substr in body: + findings.append( + Finding( + severity = HIGH, + package = pkg.display, + filename = rel, + pattern = f"cred-path-in-lifecycle ({hook})", + evidence = body, + detail = ( + f"`scripts.{hook}` references {why} " + f"({path_substr!r}); install-time access " + "to local credential files is the " + "exfiltration prep step" + ), + ) + ) + if _JS_ENV_TOKEN.search(body): + findings.append( + Finding( + severity = HIGH, + package = pkg.display, + filename = rel, + pattern = f"cred-env-in-lifecycle ({hook})", + evidence = _evidence(body, _JS_ENV_TOKEN), + detail = ( + f"`scripts.{hook}` references a credential " + "env var (GITHUB_TOKEN / NPM_TOKEN / AWS_* " + "/ etc); install-time access to runner " + "secrets is the exfiltration prep step" + ), + ) + ) + # Optional deps pointing at github: are the TanStack-style + # injection vector. + opt = meta.get("optionalDependencies") or {} + if isinstance(opt, dict): + for k, v in opt.items(): + if isinstance(v, str) and ( + v.startswith("github:") + or v.startswith("git+") + or v.startswith("git://") + ): + findings.append( + Finding( + severity = HIGH, + package = pkg.display, + filename = rel, + pattern = "optional-dep-non-registry", + evidence = f"{k}={v}", + detail = ( + "package.json `optionalDependencies` " + "points at a non-registry source; this " + "is the Shai-Hulud worm injection shape." + ), + ) + ) + return findings + + +def _host_in_outbound_context(text: str, host: str) -> bool: + """True if `host` appears in a way consistent with an outbound call. + + A bare `"169.254.169.254"` array literal (defensive blocklist) is + safe; a `fetch("http://169.254.169.254/...")` is not. The signal + is co-occurrence with either an HTTP URL scheme or a fetch verb + within a short window. + + A defensive blocklist looks like: + const CLOUD_METADATA_IPS = ["169.254.169.254", "169.254.170.2"]; + An exfil call looks like: + fetch("http://169.254.169.254/latest/meta-data/...") + http.request({ host: "169.254.169.254", path: "/..." }) + """ + # Esc for use in a regex (IPs contain dots). + host_re = re.escape(host) + # 1. URL form: http://host or https://host or //host/ or //host" + url_form = re.compile( + rf"(?:https?:)?//{host_re}(?:[:/\"'?#]|$)", + ) + if url_form.search(text): + return True + # 2. host appears within 200 chars of a fetch verb (either side). + fetch_context = re.compile( + rf"(?:{_FETCH_VERBS_PAT})[^\n]{{0,200}}{host_re}" + rf"|{host_re}[^\n]{{0,200}}(?:{_FETCH_VERBS_PAT})", + re.IGNORECASE, + ) + if fetch_context.search(text): + return True + # 3. `host:` / `hostname:` config field referencing the IP. + cfg_form = re.compile( + rf"(?:host|hostname)\s*:\s*['\"`]{host_re}['\"`]", + re.IGNORECASE, + ) + if cfg_form.search(text): + return True + return False + + +def scan_text_blob( + pkg: PackageEntry, + rel: str, + text: str, +) -> list[Finding]: + findings: list[Finding] = [] + + # IOC substrings (literal, case-sensitive). + for needle, (sev, why) in KNOWN_IOC_STRINGS.items(): + if needle in text: + findings.append( + Finding( + severity = sev, + package = pkg.display, + filename = rel, + pattern = "known-ioc-string", + evidence = needle, + detail = f"{why}: {needle!r}", + ) + ) + + # Credential surfaces. Tier 1: hosts with no legitimate use, + # bare substring is enough. + for needle, why in CRED_HOST_ALWAYS_BAD: + if needle in text: + findings.append( + Finding( + severity = HIGH, + package = pkg.display, + filename = rel, + pattern = "cred-surface-host (always-bad)", + evidence = needle, + detail = ( + f"references {why} ({needle!r}); no legitimate " + "frontend use of this surface" + ), + ) + ) + + # Credential surfaces. Tier 2: hosts that do appear in defensive + # code; require co-occurrence with a fetch verb or URL prefix. + for needle, why in CRED_HOST_NEEDS_CONTEXT: + if needle in text and _host_in_outbound_context(text, needle): + findings.append( + Finding( + severity = HIGH, + package = pkg.display, + filename = rel, + pattern = "cred-surface-host (outbound)", + evidence = needle, + detail = ( + f"references {why} ({needle!r}) in an outbound " + "call / URL / host config; a defensive blocklist " + "literal would not match this rule" + ), + ) + ) + + # Credential PATHS are deliberately not scanned here; they have + # too high a false-positive rate at file scope (defensive code, + # docker mounts, AWS SDK docs strings). `scan_package_json` + # catches the malicious case -- credential paths inside a + # lifecycle script run automatically on `npm ci`. + + # JS-specific regex. + if _JS_FETCH_EVAL.search(text): + findings.append( + Finding( + severity = HIGH, + package = pkg.display, + filename = rel, + pattern = "js-fetch-eval", + evidence = _evidence(text, _JS_FETCH_EVAL), + detail = ( + "Function/eval against base64-decoded payload " + "(obfuscated dropper shape)" + ), + ) + ) + if _JS_ENV_TOKEN.search(text): + findings.append( + Finding( + severity = MEDIUM, + package = pkg.display, + filename = rel, + pattern = "js-env-token", + evidence = _evidence(text, _JS_ENV_TOKEN), + detail = ("references credential env vars in package source"), + ) + ) + if _OBFUSC_BLOB.search(text): + findings.append( + Finding( + severity = HIGH, + package = pkg.display, + filename = rel, + pattern = "obfuscated-blob", + evidence = _evidence(text, _OBFUSC_BLOB), + detail = ( + "large base64-ish blob fed to Function/eval; " + "matches the TanStack worm dropper shape" + ), + ) + ) + + return findings + + +# Filename suffix decides which scanners run. We deliberately treat +# *.cjs/*.mjs/*.ts the same as *.js -- attackers use whichever +# extension the consumer's bundler / loader resolves. +_TEXT_SUFFIXES = ( + ".js", + ".mjs", + ".cjs", + ".ts", + ".tsx", + ".json", + ".html", + ".htm", + ".sh", + ".bash", + ".zsh", + ".py", + ".rb", + ".yml", + ".yaml", +) + + +def scan_extracted_tree( + pkg: PackageEntry, + root: Path, +) -> list[Finding]: + findings: list[Finding] = [] + for path in sorted(root.rglob("*")): + if not path.is_file(): + continue + rel = path.relative_to(root).as_posix() + lower = rel.lower() + if not lower.endswith(_TEXT_SUFFIXES): + # Skip native binaries entirely -- regex over compiled + # machine code is just noise (false positives in WASM + # opcodes, .node BSS segments, image pixel data). Use + # content-magic detection so extensionless executables + # (eg `package/biome`) and versioned shared libraries + # are also skipped. + try: + if path.stat().st_size > HARD_MAX_TEXT_FILE_BYTES: + continue + with open(path, "rb") as fh: + header = fh.read(16) + if _looks_binary(rel, header): + continue + data = header + path.read_bytes()[len(header) :] + except OSError: + continue + text = data.decode("utf-8", errors = "replace") + for needle, (sev, why) in KNOWN_IOC_STRINGS.items(): + if needle in text: + findings.append( + Finding( + severity = sev, + package = pkg.display, + filename = rel, + pattern = "known-ioc-string", + evidence = needle, + detail = f"{why}: {needle!r}", + ) + ) + continue + try: + data = path.read_bytes() + except OSError: + continue + text = data.decode("utf-8", errors = "replace") + if rel.endswith("package.json"): + findings.extend(scan_package_json(pkg, rel, text)) + findings.extend(scan_text_blob(pkg, rel, text)) + return findings + + +# ───────────────────────────────────────────────────────────────────── +# Orchestrator. +# ───────────────────────────────────────────────────────────────────── + + +def scan_one( + pkg: PackageEntry, + workspace: Path, +) -> tuple[list[Finding], str | None]: + """Download + extract + scan a single package. Cleans up its dir. + + Returns (findings, error). `error` is non-None only on hard + failures (download error, integrity mismatch, malformed tarball); + on a clean run with findings the error is None and the caller + decides exit code based on severity. + """ + pkg_dir = workspace / f"{pkg.name.replace('/', '_')}-{pkg.version}" + pkg_dir.mkdir(parents = True, exist_ok = True) + tarball = pkg_dir / "pkg.tgz" + extract = pkg_dir / "x" + try: + _, err = download_tarball(pkg, tarball) + if err: + return [], err + err = safe_extract(tarball, extract) + if err: + return [], err + return scan_extracted_tree(pkg, extract), None + finally: + # Always wipe per-package data to keep the workspace bounded. + try: + shutil.rmtree(pkg_dir, ignore_errors = True) + except Exception: + pass + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description = "Pre-install npm tarball content scanner.", + ) + parser.add_argument( + "--lockfile", + default = str(REPO_ROOT / "studio" / "frontend" / "package-lock.json"), + help = "Path to package-lock.json (default: studio/frontend).", + ) + parser.add_argument( + "--max-packages", + type = int, + default = 0, + help = ( + "Cap on number of packages to scan (0 = no cap). Useful " + "for local triage; CI runs with 0." + ), + ) + parser.add_argument( + "--fail-on", + choices = ("info", "medium", "high", "critical"), + default = "high", + help = ( + "Lowest severity that fails the run (default: high). " + "Medium and below print but exit 0." + ), + ) + args = parser.parse_args(argv) + + lockfile = Path(args.lockfile).resolve() + if not lockfile.exists(): + print(f"[scan-npm] lockfile not found: {lockfile}", file = sys.stderr) + return 2 + + entries, struct_findings = parse_lockfile(lockfile) + if struct_findings: + print( + f"[scan-npm] {len(struct_findings)} structural finding(s) " + "from lockfile pass; subsequent download scan skipped for " + "those entries.", + flush = True, + ) + + if args.max_packages > 0: + entries = entries[: args.max_packages] + + workspace = Path(tempfile.mkdtemp(prefix = "npm-scan-")).resolve() + atexit.register(lambda: shutil.rmtree(workspace, ignore_errors = True)) + print( + f"[scan-npm] workspace: {workspace}\n" + f"[scan-npm] scanning {len(entries)} package(s) from {lockfile}", + flush = True, + ) + + all_findings: list[Finding] = list(struct_findings) + hard_errors: list[tuple[str, str]] = [] + + for i, pkg in enumerate(entries, start = 1): + print( + f"[scan-npm] [{i}/{len(entries)}] {pkg.display}", + flush = True, + ) + blocked = BLOCKED_NPM_VERSIONS.get(pkg.name, set()) + if pkg.version in blocked: + finding = Finding( + severity = CRITICAL, + package = pkg.display, + filename = "", + pattern = "blocked-known-malicious", + detail = f"{pkg.name}@{pkg.version} is on the BLOCKED_NPM_VERSIONS list", + ) + all_findings.append(finding) + print(str(finding), flush = True) + continue + findings, err = scan_one(pkg, workspace) + if err: + hard_errors.append((pkg.display, err)) + print(f"[scan-npm] ERROR {pkg.display}: {err}", flush = True) + continue + all_findings.extend(findings) + for f in findings: + print(str(f), flush = True) + + # Sort by severity then package. + all_findings.sort(key = lambda f: (_SEVERITY_RANK[f.severity], f.package)) + + print( + f"\n[scan-npm] summary: {len(entries)} package(s), " + f"{len(all_findings)} finding(s), " + f"{len(hard_errors)} hard error(s)", + flush = True, + ) + + if hard_errors: + print("\n[scan-npm] HARD ERRORS:", file = sys.stderr) + for pkg, err in hard_errors: + print(f" {pkg}: {err}", file = sys.stderr) + + threshold = { + "info": INFO, + "medium": MEDIUM, + "high": HIGH, + "critical": CRITICAL, + }[args.fail_on] + threshold_rank = _SEVERITY_RANK[threshold] + blocking = [f for f in all_findings if _SEVERITY_RANK[f.severity] <= threshold_rank] + if hard_errors or blocking: + if blocking: + print( + f"\n[scan-npm] FAIL: {len(blocking)} finding(s) " + f"at or above {threshold}", + file = sys.stderr, + ) + return 1 + print("\n[scan-npm] OK", flush = True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/scan_packages.py b/scripts/scan_packages.py index f74368d58a..6779b634f7 100644 --- a/scripts/scan_packages.py +++ b/scripts/scan_packages.py @@ -65,6 +65,15 @@ MEDIUM = "MEDIUM" SEVERITY_ORDER = {CRITICAL: 0, HIGH: 1, MEDIUM: 2} +# Hard pin-blocks for publicly confirmed malicious PyPI versions. +# Source: Socket.dev 2026-05-12 disclosure (Mini Shai-Hulud May-12 wave) and +# earlier Semgrep / Endor reports for the `lightning` entries. +BLOCKED_PYPI_VERSIONS: dict[str, set[str]] = { + "guardrails-ai": {"0.10.1"}, + "mistralai": {"2.4.6"}, + "lightning": {"2.6.2", "2.6.3"}, +} + # --------------------------------------------------------------------------- # Pattern definitions # --------------------------------------------------------------------------- @@ -336,6 +345,15 @@ RE_TOKEN_REGEX = re.compile( r"|\bglpat-[0-9A-Za-z_-]{20,}", # GitLab PAT ) +# Mini Shai-Hulud May-12 2026 wave indicators. The dropper artifact name +# `transformers.pyz` is high-confidence (no legit PyPI package ships a `.pyz` +# named after `transformers`); the host + slogans are CRITICAL. +RE_MAY12_IOC = re.compile( + r"(git-tanstack\.com|/tmp/transformers\.pyz|transformers\.pyz" + r"|With Love TeamPCP|We've been online over 2 hours)", + re.IGNORECASE, +) + # JavaScript-side obfuscation. The npm chalk/debug compromise and the # Lightning router_runtime.js use the same minifier-style hex-var name # pattern; a bundle full of `_0x1f2e3d` identifiers is a near-universal @@ -529,6 +547,7 @@ def check_py_file(content: str, filename: str, package: str) -> list[Finding]: has_openssl_cli = bool(RE_OPENSSL_CLI.search(content)) has_temp_exec = bool(RE_TEMP_EXEC.search(content)) has_c2_polling = bool(RE_C2_POLLING.search(content)) + has_may12_ioc = bool(RE_MAY12_IOC.search(content)) # --------------------------------------------------------------- # CRITICAL: combination patterns that strongly indicate malice @@ -572,6 +591,18 @@ def check_py_file(content: str, filename: str, package: str) -> list[Finding]: ) ) + # May-12 Shai-Hulud IOC string in Python source. + if has_may12_ioc: + findings.append( + Finding( + CRITICAL, + package, + filename, + "May-12 Shai-Hulud IOC string present in Python file", + _extract_evidence(content, RE_MAY12_IOC), + ) + ) + # C2 polling/beaconing loop if has_c2_polling: findings.append( @@ -1071,6 +1102,16 @@ def check_shell_file(content: str, filename: str, package: str) -> list[Finding] _extract_evidence(content, RE_WORKFLOW_INJECT), ) ) + if RE_MAY12_IOC.search(content): + findings.append( + Finding( + CRITICAL, + package, + filename, + "May-12 Shai-Hulud IOC string present in shell script", + _extract_evidence(content, RE_MAY12_IOC), + ) + ) return findings @@ -1111,6 +1152,16 @@ def check_workflow_file(content: str, filename: str, package: str) -> list[Findi _extract_evidence(content, RE_SHELL_DROPPER), ) ) + if RE_MAY12_IOC.search(content): + findings.append( + Finding( + CRITICAL, + package, + filename, + "May-12 Shai-Hulud IOC string present in workflow file", + _extract_evidence(content, RE_MAY12_IOC), + ) + ) return findings @@ -1118,33 +1169,154 @@ def check_workflow_file(content: str, filename: str, package: str) -> list[Findi # Archive handling # --------------------------------------------------------------------------- +# Tarbomb caps, mirrored from scripts/scan_npm_packages.py::safe_extract. +# Refuses zip-of-death / tar-of-death archives so a hostile sdist or +# wheel cannot exhaust memory or fill the temp dir before content +# scanning even starts. Keep these constants in sync with the npm side; +# we duplicate rather than import to keep `scan_packages.py` standalone. +HARD_MAX_FILE_BYTES = 64 * 1024 * 1024 # 64 MiB per member +HARD_MAX_TOTAL_BYTES = 512 * 1024 * 1024 # 512 MiB cumulative +HARD_MAX_MEMBERS = 50_000 # entries per archive + + +def _refuse_unsafe_member_name(name: str) -> str | None: + """Return a refusal reason for a member name, or None if safe. + + Mirrors `scan_npm_packages.py::safe_extract` semantics: no absolute + paths, no `..` traversal segments. The caller is responsible for + checking the resolved path lands inside the extract root, but for + iter_archive_files we never write to disk so the name-shape check + plus the in-memory size cap is sufficient. + """ + if name.startswith("/") or ".." in Path(name).parts: + return f"unsafe member name {name!r}" + return None + def iter_archive_files(archive_path: str): - """Yield (filename, text_content) for every file in a wheel/sdist.""" + """Yield (filename, text_content) for every file in a wheel/sdist. + + Streams members with size + count caps applied at the member level + so a tarbomb / zipbomb cannot blow up the scanner's memory budget. + On cap breach we emit a `[WARN]` log and short-circuit the archive. + """ path = Path(archive_path) if path.suffix == ".whl" or path.suffix == ".zip": + total = 0 + count = 0 with zipfile.ZipFile(path) as zf: for info in zf.infolist(): if info.is_dir(): continue + count += 1 + if count > HARD_MAX_MEMBERS: + print( + f" [WARN] {path.name}: refused; member count " + f"{count} exceeds cap {HARD_MAX_MEMBERS}", + file = sys.stderr, + ) + return + reason = _refuse_unsafe_member_name(info.filename) + if reason is not None: + print( + f" [WARN] {path.name}: refused member ({reason})", + file = sys.stderr, + ) + continue + # Declared (uncompressed) size cap. + if info.file_size > HARD_MAX_FILE_BYTES: + print( + f" [WARN] {path.name}: skipped {info.filename!r} " + f"(declared {info.file_size} > cap {HARD_MAX_FILE_BYTES})", + file = sys.stderr, + ) + continue + if total + info.file_size > HARD_MAX_TOTAL_BYTES: + print( + f" [WARN] {path.name}: cumulative bytes cap " + f"{HARD_MAX_TOTAL_BYTES} hit at {info.filename!r}", + file = sys.stderr, + ) + return try: data = zf.read(info.filename) + total += len(data) text = data.decode("utf-8", errors = "replace") yield info.filename, text except Exception: continue elif path.name.endswith((".tar.gz", ".tgz", ".tar.bz2", ".tar.xz", ".tar")): - with tarfile.open(path) as tf: - for member in tf.getmembers(): + total = 0 + count = 0 + # Streaming open so we never read the whole archive into memory. + with tarfile.open(path, mode = "r|*") as tf: + for member in tf: + count += 1 + if count > HARD_MAX_MEMBERS: + print( + f" [WARN] {path.name}: refused; member count " + f"{count} exceeds cap {HARD_MAX_MEMBERS}", + file = sys.stderr, + ) + return + # Refuse symlinks / hardlinks / devices outright -- the + # scanner never writes them anyway, but tar parsers + # have historically dereferenced them on extract. + if member.issym() or member.islnk(): + print( + f" [WARN] {path.name}: refused link member " + f"{member.name!r}", + file = sys.stderr, + ) + continue + if member.isdev() or member.isfifo(): + print( + f" [WARN] {path.name}: refused special member " + f"{member.name!r}", + file = sys.stderr, + ) + continue if not member.isfile(): continue + reason = _refuse_unsafe_member_name(member.name) + if reason is not None: + print( + f" [WARN] {path.name}: refused member ({reason})", + file = sys.stderr, + ) + continue + declared = max(member.size, 0) + if declared > HARD_MAX_FILE_BYTES: + print( + f" [WARN] {path.name}: skipped {member.name!r} " + f"(declared {declared} > cap {HARD_MAX_FILE_BYTES})", + file = sys.stderr, + ) + continue + if total + declared > HARD_MAX_TOTAL_BYTES: + print( + f" [WARN] {path.name}: cumulative bytes cap " + f"{HARD_MAX_TOTAL_BYTES} hit at {member.name!r}", + file = sys.stderr, + ) + return try: f = tf.extractfile(member) if f is None: continue - data = f.read() + # Bound the read so a tar header that lies about + # size cannot OOM us. + data = f.read(HARD_MAX_FILE_BYTES + 1) + if len(data) > HARD_MAX_FILE_BYTES: + print( + f" [WARN] {path.name}: body of " + f"{member.name!r} exceeded declared cap", + file = sys.stderr, + ) + continue + total += len(data) text = data.decode("utf-8", errors = "replace") yield member.name, text except Exception: @@ -1154,26 +1326,48 @@ def iter_archive_files(archive_path: str): def scan_archive(archive_path: str, package: str) -> list[Finding]: - """Scan all files in an archive for malicious patterns.""" - findings = [] - for filename, content in iter_archive_files(archive_path): - lower = filename.lower() - if lower.endswith(".pth"): - findings.extend(check_pth_file(content, filename, package)) - elif lower.endswith(".py"): - findings.extend(check_py_file(content, filename, package)) - elif lower.endswith((".js", ".mjs", ".cjs", ".ts")): - # Lightning 2.6.x hid its real payload in a 14.8 MB - # router_runtime.js inside a Python wheel. Without this - # branch we'd have only seen the small Python loader. - findings.extend(check_js_file(content, filename, package)) - elif lower.endswith((".sh", ".bash")): - findings.extend(check_shell_file(content, filename, package)) - elif "/.github/workflows/" in lower and lower.endswith((".yml", ".yaml")): - # Shai-Hulud / ForceMemo plant their own GHA workflow. - # A workflow file inside a *PyPI package* is on its own - # already a yellow flag; pattern-match the worm signatures. - findings.extend(check_workflow_file(content, filename, package)) + """Scan all files in an archive for malicious patterns. + + A corrupted archive container (truncated wheel, bad gzip header, + etc.) used to be silently skipped by an ``except Exception: continue`` + inside ``iter_archive_files``. Per the silent-failure hardening + (SF1) it now emits a CRITICAL ``archive_corrupted`` finding so the + main loop counts and surfaces it rather than reporting "0 findings". + """ + findings: list[Finding] = [] + try: + for filename, content in iter_archive_files(archive_path): + lower = filename.lower() + if lower.endswith(".pth"): + findings.extend(check_pth_file(content, filename, package)) + elif lower.endswith(".py"): + findings.extend(check_py_file(content, filename, package)) + elif lower.endswith((".js", ".mjs", ".cjs", ".ts")): + # Lightning 2.6.x hid its real payload in a 14.8 MB + # router_runtime.js inside a Python wheel. Without this + # branch we'd have only seen the small Python loader. + findings.extend(check_js_file(content, filename, package)) + elif lower.endswith((".sh", ".bash")): + findings.extend(check_shell_file(content, filename, package)) + elif "/.github/workflows/" in lower and lower.endswith((".yml", ".yaml")): + # Shai-Hulud / ForceMemo plant their own GHA workflow. + # A workflow file inside a *PyPI package* is on its own + # already a yellow flag; pattern-match the worm signatures. + findings.extend(check_workflow_file(content, filename, package)) + except (zipfile.BadZipFile, tarfile.TarError, EOFError, OSError) as exc: + # The archive cannot be opened or is structurally broken. A + # benign wheel/sdist always opens; a malformed one is either a + # transport corruption (treat as scan failure) or a deliberate + # attempt to bypass scanners that swallow archive errors. + findings.append( + Finding( + CRITICAL, + package, + os.path.basename(archive_path), + "archive_corrupted", + f"{type(exc).__name__}: {exc}"[:240], + ) + ) return findings @@ -1182,33 +1376,120 @@ def scan_archive(archive_path: str, package: str) -> list[Finding]: # --------------------------------------------------------------------------- +_RE_PYPI_SPEC_VERSION = re.compile(r"==\s*([A-Za-z0-9_.\-+!]+)") + + +def _check_blocked_pypi_versions( + specs: list[str], +) -> tuple[list[str], list[Finding]]: + """Filter ``specs`` against ``BLOCKED_PYPI_VERSIONS``. + + Returns ``(safe_specs, findings)``. Each blocked spec emits a CRITICAL + ``Finding`` and is removed from the returned spec list so the caller + never fetches the malicious tarball. Specs without an ``==X.Y.Z`` pin + pass through unchanged -- pip will resolve them at download time and + the existing scanners will catch the payload via the IOC regexes. + """ + safe: list[str] = [] + findings: list[Finding] = [] + for spec in specs: + name = _extract_pkg_name(spec).lower() + blocked = BLOCKED_PYPI_VERSIONS.get(name, set()) + if not blocked: + safe.append(spec) + continue + m = _RE_PYPI_SPEC_VERSION.search(spec) + version = m.group(1) if m else None + if version is not None and version in blocked: + findings.append( + Finding( + CRITICAL, + f"{name}=={version}", + "", + "blocked-known-malicious", + f"{name}=={version} is on the BLOCKED_PYPI_VERSIONS list", + ) + ) + # Drop the spec; do not download. + continue + safe.append(spec) + return safe, findings + + +def _pip_download_env() -> dict[str, str]: + """Return a scrubbed environment for invoking `pip download`. + + Hostile shells / CI configs can override the index with PIP_INDEX_URL, + PIP_EXTRA_INDEX_URL, or a user `pip.conf`. We strip every PIP_* + override and route the resolver explicitly at PyPI. PIP_CONFIG_FILE + is forced to /dev/null so a stray ~/.pip/pip.conf with an + extra-index-url cannot bypass the pin. + """ + env = {**os.environ} + # Drop any user override. + for key in [k for k in env if k.startswith("PIP_")]: + env.pop(key, None) + env["PIP_INDEX_URL"] = "https://pypi.org/simple" + env["PIP_EXTRA_INDEX_URL"] = "" + env["PIP_CONFIG_FILE"] = "/dev/null" + env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + return env + + +# Pip resolver flags shared by both download branches. Pinning the +# index URL on the CLI is belt + braces with the env scrub above. +# `--no-build-isolation` is deliberately NOT set; we never invoke +# setup.py at all because of `--only-binary :all:`. +_PIP_DOWNLOAD_PIN_FLAGS = [ + "--index-url", + "https://pypi.org/simple", + "--only-binary", + ":all:", +] + + +# Strip any character that could escape `dest` via `os.path.join`. This +# is the last line of defence before `pkg_dir = os.path.join(dest, ...)` +# so a spec like `../../etc/foo==1.0` cannot land outside the temp tree. +_RE_PKG_NAME_SANITIZE = re.compile(r"[^A-Za-z0-9._-]") + + def download_packages( specs: list[str], dest: str, *, with_deps: bool = False, -) -> list[tuple[str, str]]: +) -> tuple[list[tuple[str, str]], list[str]]: """Download packages to dest using pip download. NEVER installs. - Returns list of (spec_or_name, filepath) for every downloaded archive. + Returns ``(results, download_errors)`` where ``results`` is a list of + ``(spec_or_name, filepath)`` for every downloaded archive and + ``download_errors`` is a list of one-line transport-failure summaries. + A non-empty ``download_errors`` MUST cause the caller to exit non-zero + even if no findings were produced; a silent ``0 findings, scan + incomplete`` is the bug class this return-shape was widened to fix. When with_deps=True, downloads the full transitive dependency tree in a single pip invocation (all archives land in one flat dir). When with_deps=False (default), downloads each spec individually with --no-deps. """ - results = [] + results: list[tuple[str, str]] = [] + download_errors: list[str] = [] + env = _pip_download_env() if with_deps: # Single pip download call for all specs + their transitive deps. - # --no-build-isolation and --no-binary :none: are NOT used -- - # pip download only fetches wheels/sdists, never executes them. + # `--only-binary :all:` refuses sdists so we never execute a + # setup.py just to learn dependency metadata; combined with the + # scrubbed env, pip is wired hard at pypi.org. os.makedirs(dest, exist_ok = True) cmd = [ sys.executable, "-m", "pip", "download", + *_PIP_DOWNLOAD_PIN_FLAGS, "--dest", dest, ] + specs @@ -1218,14 +1499,18 @@ def download_packages( capture_output = True, text = True, timeout = 600, # transitive resolution can be slow + env = env, ) if proc.returncode != 0: - print( - f" [ERROR] pip download (with deps) failed: {proc.stderr.strip()[:500]}", - file = sys.stderr, + msg = ( + f"pip download (with deps) failed: " f"{proc.stderr.strip()[:500]}" ) + print(f" [ERROR] {msg}", file = sys.stderr) + download_errors.append(msg) except subprocess.TimeoutExpired: - print(f" [ERROR] pip download (with deps) timed out", file = sys.stderr) + msg = "pip download (with deps) timed out" + print(f" [ERROR] {msg}", file = sys.stderr) + download_errors.append(msg) # Collect every archive that landed in dest for fname in sorted(os.listdir(dest)): @@ -1236,9 +1521,11 @@ def download_packages( results.append((pkg_name, fpath)) else: for spec in specs: - pkg_dir = os.path.join( - dest, spec.split("==")[0].split(">=")[0].split("<=")[0].split("[")[0] - ) + raw_name = _extract_pkg_name(spec) + # Sanitize before joining into `dest` so a hostile spec + # cannot path-traverse out of the destination directory. + safe_name = _RE_PKG_NAME_SANITIZE.sub("_", raw_name) or "_pkg" + pkg_dir = os.path.join(dest, safe_name) os.makedirs(pkg_dir, exist_ok = True) cmd = [ sys.executable, @@ -1246,6 +1533,7 @@ def download_packages( "pip", "download", "--no-deps", + *_PIP_DOWNLOAD_PIN_FLAGS, "--dest", pkg_dir, spec, @@ -1256,15 +1544,20 @@ def download_packages( capture_output = True, text = True, timeout = 120, + env = env, ) if proc.returncode != 0: - print( - f" [ERROR] pip download failed for {spec}: {proc.stderr.strip()}", - file = sys.stderr, + msg = ( + f"pip download failed for {spec}: " + f"{proc.stderr.strip()[:500]}" ) + print(f" [ERROR] {msg}", file = sys.stderr) + download_errors.append(msg) continue except subprocess.TimeoutExpired: - print(f" [ERROR] pip download timed out for {spec}", file = sys.stderr) + msg = f"pip download timed out for {spec}" + print(f" [ERROR] {msg}", file = sys.stderr) + download_errors.append(msg) continue # Find downloaded file(s) @@ -1272,7 +1565,7 @@ def download_packages( fpath = os.path.join(pkg_dir, fname) if os.path.isfile(fpath): results.append((spec, fpath)) - return results + return results, download_errors # --------------------------------------------------------------------------- @@ -1586,6 +1879,13 @@ def update_req_file(filepath: str, updates: dict[int, str]) -> None: """Apply line-level updates to a requirements file. updates: {line_num (1-indexed): new_line_text} + + Writes atomically: stage in a sibling tmp file on the same + filesystem, fsync, then `os.replace` over the original. A SIGKILL + or power loss mid-write therefore either leaves the original + intact or leaves the fully new file -- never a half-written + requirements file (which would silently re-introduce a malicious + pin). """ with open(filepath) as f: lines = f.readlines() @@ -1597,8 +1897,24 @@ def update_req_file(filepath: str, updates: dict[int, str]) -> None: ending = "\n" if lines[idx].endswith("\n") else "" lines[idx] = new_text + ending - with open(filepath, "w") as f: - f.writelines(lines) + dirpath = os.path.dirname(os.path.abspath(filepath)) or "." + fd, tmp_path = tempfile.mkstemp( + prefix = ".req_fix.", + dir = dirpath, + ) + try: + with os.fdopen(fd, "w") as f: + f.writelines(lines) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp_path, filepath) + except Exception: + # Best effort cleanup; the destination was never touched. + try: + os.unlink(tmp_path) + except OSError: + pass + raise def _run_fix( @@ -1842,10 +2158,19 @@ def main() -> int: all_findings: list[Finding] = [] + # Hard pin-block: refuse to download known-malicious PyPI versions. + specs, blocked_findings = _check_blocked_pypi_versions(specs) + all_findings.extend(blocked_findings) + tmpdir = tempfile.mkdtemp(prefix = "pth_scan_") atexit.register(lambda d = tmpdir: shutil.rmtree(d, ignore_errors = True)) + download_errors: list[str] = [] try: - downloaded = download_packages(specs, tmpdir, with_deps = args.with_deps) + downloaded, download_errors = download_packages( + specs, + tmpdir, + with_deps = args.with_deps, + ) print(f" Downloaded {len(downloaded)} archive(s).") for spec, archive_path in downloaded: @@ -1871,6 +2196,26 @@ def main() -> int: ) _run_fix(critical_pkgs, entries, args.max_search) + # Surface any pip-download failures BEFORE the scan-result exit code so + # an empty / partial download cannot mask itself as "0 findings, all + # clean". This is item (4) of the silent-failure hardening: an + # unresolvable spec or PyPI timeout used to print to stderr and exit 0. + if download_errors: + print( + f"\n {'=' * 72}\n" + f" SCAN INCOMPLETE: {len(download_errors)} pip download " + f"failure(s):\n" + f" {'=' * 72}", + file = sys.stderr, + ) + for err in download_errors: + print(f" [ERROR] {err}", file = sys.stderr) + print( + " Refusing to report 'all clean' on a partial scan; " "exiting 2.", + file = sys.stderr, + ) + return 2 + # Exit code: 1 if any CRITICAL or HIGH if any(f.severity in (CRITICAL, HIGH) for f in all_findings): return 1 diff --git a/scripts/stamp_studio_release.py b/scripts/stamp_studio_release.py new file mode 100644 index 0000000000..ac538e9937 --- /dev/null +++ b/scripts/stamp_studio_release.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Stamp and verify display-only Studio release metadata for builds.""" + +from __future__ import annotations + +import argparse +import os +import re +import subprocess +import sys +import tarfile +import tempfile +import zipfile +from pathlib import Path + + +def _atomic_write_text(path: Path, data: str, encoding: str = "utf-8") -> None: + """Atomic version of ``Path.write_text``. + + A crash or signal mid-write leaves the prior file intact; the + Studio build never reads a partial ``_studio_release_build.py``. + """ + dirpath = str(path.parent) or "." + path.parent.mkdir(parents = True, exist_ok = True) + fd, tmp_path = tempfile.mkstemp(prefix = ".stamp_studio.", dir = dirpath) + try: + with os.fdopen(fd, "w", encoding = encoding) as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp_path, path) + except Exception: + try: + os.unlink(tmp_path) + except OSError: + pass + raise + + +REPO_ROOT = Path(__file__).resolve().parents[1] +BUILD_INFO_PATH = ( + REPO_ROOT / "studio" / "backend" / "utils" / "_studio_release_build.py" +) +BUILD_INFO_SUFFIX = "studio/backend/utils/_studio_release_build.py" +VERSION_RE = re.compile(r"^v\d+\.\d+\.\d+(?:-[0-9A-Za-z.][0-9A-Za-z.-]*)?$") +GIT_DESCRIBE_SUFFIX_RE = re.compile(r"-\d+-g[0-9A-Fa-f]+(?:-dirty)?$") +MAX_VERSION_LENGTH = 64 +PLACEHOLDER = """# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +\"\"\"Build-stamped Studio release metadata. + +Release builds may rewrite this module in the build workspace before creating +Python artifacts. Keep the committed value neutral so source checkouts do not +accidentally report a stale release tag. +\"\"\" + +STUDIO_RELEASE_VERSION = None +""" + + +def is_valid_version(value: object) -> bool: + if not isinstance(value, str): + return False + version = value.strip() + if not version or len(version) > MAX_VERSION_LENGTH: + return False + if version.endswith("-dirty") or GIT_DESCRIBE_SUFFIX_RE.search(version): + return False + return VERSION_RE.fullmatch(version) is not None + + +def _exact_git_tag() -> str | None: + try: + result = subprocess.run( + [ + "git", + "describe", + "--tags", + "--exact-match", + "--match", + "v[0-9]*", + "HEAD", + ], + cwd = REPO_ROOT, + check = False, + stdout = subprocess.PIPE, + stderr = subprocess.DEVNULL, + text = True, + timeout = 2.0, + ) + except (OSError, subprocess.TimeoutExpired): + return None + if result.returncode != 0: + return None + tag = result.stdout.strip() + return tag if is_valid_version(tag) else None + + +def _git_worktree_is_dirty() -> bool: + try: + result = subprocess.run( + ["git", "status", "--porcelain"], + cwd = REPO_ROOT, + check = False, + stdout = subprocess.PIPE, + stderr = subprocess.DEVNULL, + text = True, + timeout = 2.0, + ) + except (OSError, subprocess.TimeoutExpired): + return True + if result.returncode != 0: + return True + return bool(result.stdout.strip()) + + +def _github_tag() -> str | None: + if os.environ.get("GITHUB_REF_TYPE") != "tag": + return None + github_ref = os.environ.get("GITHUB_REF_NAME", "").strip() + return github_ref or None + + +def resolve_version() -> tuple[str | None, str]: + env_version = os.environ.get("UNSLOTH_STUDIO_RELEASE_VERSION", "").strip() + if env_version: + return (env_version, "UNSLOTH_STUDIO_RELEASE_VERSION") + + github_ref = _github_tag() + if github_ref: + return (github_ref, "GITHUB_REF_NAME") + + git_tag = _exact_git_tag() + if git_tag: + return (git_tag, "exact git tag") + + return (None, "none") + + +def build_info_source(version: str | None) -> str: + literal = repr(version) if version is not None else "None" + return f'''# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Build-stamped Studio release metadata.""" + +STUDIO_RELEASE_VERSION = {literal} +''' + + +def _env_version_conflicts(version: str) -> list[tuple[str, str]]: + conflicts: list[tuple[str, str]] = [] + github_ref = _github_tag() + if github_ref and is_valid_version(github_ref) and github_ref != version: + conflicts.append(("GITHUB_REF_NAME", github_ref)) + + git_tag = _exact_git_tag() + if git_tag and git_tag != version: + conflicts.append(("exact git tag", git_tag)) + + return conflicts + + +def stamp(require_release: bool) -> int: + version, source = resolve_version() + if version is not None and not is_valid_version(version): + print( + f"Invalid Studio release version from {source}: {version!r}", + file = sys.stderr, + ) + return 2 + + if version is not None and source == "UNSLOTH_STUDIO_RELEASE_VERSION": + conflicts = _env_version_conflicts(version) + if conflicts: + details = ", ".join(f"{name}={value!r}" for name, value in conflicts) + print( + "UNSLOTH_STUDIO_RELEASE_VERSION does not match available " + f"release tag metadata: {details}", + file = sys.stderr, + ) + return 2 + + if require_release and source == "exact git tag" and _git_worktree_is_dirty(): + print( + "Refusing to publish from a dirty exact-tag checkout. Set " + "UNSLOTH_STUDIO_RELEASE_VERSION explicitly from release automation " + "or publish from a clean tag checkout.", + file = sys.stderr, + ) + return 2 + + if version is None: + if require_release: + print( + "No Studio release version available. Set " + "UNSLOTH_STUDIO_RELEASE_VERSION, build from a GitHub tag, " + "or run from an exact local Studio release tag.", + file = sys.stderr, + ) + return 2 + _atomic_write_text(BUILD_INFO_PATH, PLACEHOLDER, encoding = "utf-8") + print("dev") + return 0 + + _atomic_write_text(BUILD_INFO_PATH, build_info_source(version), encoding = "utf-8") + print(f"Stamping Studio release version {version} from {source}", file = sys.stderr) + print(version) + return 0 + + +def _read_wheel_member(path: Path) -> str | None: + with zipfile.ZipFile(path) as archive: + for name in archive.namelist(): + if name.endswith(BUILD_INFO_SUFFIX): + return archive.read(name).decode("utf-8") + return None + + +def _read_sdist_member(path: Path) -> str | None: + with tarfile.open(path) as archive: + for member in archive.getmembers(): + if member.name.endswith(BUILD_INFO_SUFFIX): + extracted = archive.extractfile(member) + if extracted is None: + return None + return extracted.read().decode("utf-8") + return None + + +def verify_dist(expected: str, dist_dir: Path) -> int: + if not is_valid_version(expected): + print(f"Invalid expected Studio release version: {expected!r}", file = sys.stderr) + return 2 + + artifacts = list(dist_dir.glob("*.whl")) + list(dist_dir.glob("*.tar.gz")) + if not artifacts: + print(f"No wheel or sdist artifacts found in {dist_dir}", file = sys.stderr) + return 2 + + expected_line = f"STUDIO_RELEASE_VERSION = {expected!r}" + failures: list[str] = [] + for artifact in artifacts: + if artifact.suffix == ".whl": + content = _read_wheel_member(artifact) + else: + content = _read_sdist_member(artifact) + if content is None: + failures.append(f"{artifact.name}: missing {BUILD_INFO_SUFFIX}") + elif expected_line not in content: + failures.append(f"{artifact.name}: Studio release version mismatch") + + if failures: + for failure in failures: + print(failure, file = sys.stderr) + return 2 + + print(f"Verified Studio release version {expected} in {len(artifacts)} artifact(s)") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description = __doc__) + parser.add_argument("--require-release", action = "store_true") + parser.add_argument("--verify-dist", type = Path) + parser.add_argument("--expected") + args = parser.parse_args() + + if args.verify_dist is not None: + if not args.expected: + parser.error("--verify-dist requires --expected") + return verify_dist(args.expected, args.verify_dist) + + return stamp(require_release = args.require_release) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/studio/backend/auth/storage.py b/studio/backend/auth/storage.py index 9a03f5f542..3233aa05ef 100644 --- a/studio/backend/auth/storage.py +++ b/studio/backend/auth/storage.py @@ -480,6 +480,37 @@ def save_refresh_token( conn.close() +def consume_refresh_token(token: str) -> Optional[Tuple[str, bool]]: + """Atomically validate-and-delete a refresh token for single-use rotation. + + DELETE RETURNING fuses validate and delete into one statement so two + concurrent refresh requests cannot both consume the same token. + """ + token_hash = _hash_token(token) + now = datetime.now(timezone.utc).isoformat() + conn = get_connection() + try: + conn.execute( + "DELETE FROM refresh_tokens WHERE expires_at < ?", + (now,), + ) + cur = conn.execute( + """ + DELETE FROM refresh_tokens + WHERE token_hash = ? AND expires_at >= ? + RETURNING username, is_desktop + """, + (token_hash, now), + ) + row = cur.fetchone() + conn.commit() + if row is None: + return None + return row["username"], bool(row["is_desktop"]) + finally: + conn.close() + + def verify_refresh_token(token: str) -> Optional[Tuple[str, bool]]: """ Verify a refresh token and return the username plus desktop marker. diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 6c97ef8cb2..35933e6685 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -433,6 +433,8 @@ class LlamaCppBackend: self._hf_variant: Optional[str] = None self._is_vision: bool = False self._healthy = False + # Set by _classify_gpu_offload after _wait_for_health. + self._gpu_offload_active: Optional[bool] = None self._context_length: Optional[int] = None self._effective_context_length: Optional[int] = None self._max_context_length: Optional[int] = None @@ -956,6 +958,13 @@ class LlamaCppBackend: logger.debug(f"torch GPU probe failed: {e}") return [] + # Free-VRAM fraction at which Studio pins the GPU directly instead + # of deferring to ``--fit on``. 5% headroom covers CUDA context + + # compute buffers; 0.90 was too conservative and dropped 91-94% + # fits to CPU offload (#5106). The fork's --fit on still catches + # the truly-too-large case. + _GPU_PIN_VRAM_FRACTION = 0.95 + @staticmethod def _windows_pip_nvidia_dll_dirs(prefix: str) -> list[str]: """Return DLL dirs from pip-installed CUDA wheels under @@ -1024,11 +1033,11 @@ class LlamaCppBackend: """Pick GPU(s) for a model based on estimated VRAM and free memory. ``model_size_bytes`` should include both model weights and estimated - KV cache. The 90% threshold provides headroom for compute buffers, - CUDA context, and other runtime overhead. + KV cache. The ``_GPU_PIN_VRAM_FRACTION`` threshold provides headroom + for compute buffers, CUDA context, and other runtime overhead. Returns (gpu_indices, use_fit): - - ([1], False) model fits on 1 GPU at 90% of free + - ([1], False) model fits on 1 GPU at the headroom threshold - ([1, 2], False) model needs 2 GPUs - (None, True) model too large, let --fit handle it """ @@ -1036,12 +1045,13 @@ class LlamaCppBackend: return None, True model_size_mib = model_size_bytes / (1024 * 1024) + usable_fraction = LlamaCppBackend._GPU_PIN_VRAM_FRACTION # Sort GPUs by free memory descending ranked = sorted(gpus, key = lambda g: g[1], reverse = True) - # Try fitting on 1 GPU (90% of free memory threshold) - if ranked[0][1] * 0.90 >= model_size_mib: + # Try fitting on 1 GPU at the usable-VRAM threshold. + if ranked[0][1] * usable_fraction >= model_size_mib: return [ranked[0][0]], False # Try fitting on N GPUs (accumulate free memory from most-free) @@ -1049,7 +1059,7 @@ class LlamaCppBackend: selected = [] for idx, free_mib in ranked: selected.append(idx) - cumulative += free_mib * 0.90 + cumulative += free_mib * usable_fraction if cumulative >= model_size_mib: return sorted(selected), False @@ -1282,10 +1292,11 @@ class LlamaCppBackend: ) -> int: """Return the largest context length that fits in GPU VRAM. - Uses 90% of available VRAM as the budget (matching _select_gpus - threshold -- 10% reserved for compute buffers, CUDA context, - scratch space, flash-attn workspace, etc.). - If the model weights alone don't fit, returns min_ctx unchanged. + Uses 90% of available VRAM as the ctx-fit budget. Tighter than + ``_GPU_PIN_VRAM_FRACTION`` on purpose: over-promising context + OOMs at runtime, while pinning conservatively just defers to + --fit on. If the weights alone don't fit, returns + ``requested_ctx`` unchanged. ``kv_on_gpu`` mirrors ``--kv-offload`` (default on). When False the KV cache lives in CPU RAM and doesn't compete with weights @@ -2031,6 +2042,7 @@ class LlamaCppBackend: # still has valid state to publish. effective_ctx = n_ctx if n_ctx > 0 else (self._context_length or 0) max_available_ctx = self._context_length or effective_ctx + gpus: list[tuple[int, int]] = [] try: model_size = self._get_gguf_size_bytes(model_path) gpus = self._get_gpu_free_memory() @@ -2114,8 +2126,11 @@ class LlamaCppBackend: gpu_indices, use_fit = self._select_gpus(requested_total, gpus) # No silent shrink: effective_ctx stays == n_ctx. else: - # Auto context: prefer fewer GPUs, cap context to fit. + # Auto context: prefer fewer GPUs, cap context + # to fit. Same headroom threshold as + # _select_gpus (#5106). ranked = sorted(gpus, key = lambda g: g[1], reverse = True) + pin_fraction = self._GPU_PIN_VRAM_FRACTION for n_gpus in range(1, len(ranked) + 1): subset = ranked[:n_gpus] pool_mib = sum(free for _, free in subset) @@ -2130,18 +2145,31 @@ class LlamaCppBackend: capped, cache_type_kv, n_parallel = n_parallel ) total_mib = (model_size + kv) / (1024 * 1024) - if total_mib <= pool_mib * 0.90: + if total_mib <= pool_mib * pin_fraction: effective_ctx = capped gpu_indices = sorted(idx for idx, _ in subset) use_fit = False break else: - # No subset can host the weights (weights alone - # exceed 90% of every pool). Per spec, default - # the UI-visible context to 4096 and let - # --fit on flex -ngl so llama-server offloads - # layers to CPU RAM. + # Native ctx doesn't fit. Drop to 4096 and + # re-check before deferring to --fit on: + # a model that overflows at 131k may pin + # comfortably with a 4096 KV cache (#5106). effective_ctx = min(4096, effective_ctx) + if effective_ctx > 0: + for n_gpus in range(1, len(ranked) + 1): + subset = ranked[:n_gpus] + pool_mib = sum(free for _, free in subset) + kv = self._estimate_kv_cache_bytes( + effective_ctx, + cache_type_kv, + n_parallel = n_parallel, + ) + total_mib = (model_size + kv) / (1024 * 1024) + if total_mib <= pool_mib * pin_fraction: + gpu_indices = sorted(idx for idx, _ in subset) + use_fit = False + break elif gpus: # Can't estimate KV -- fall back to file-size-only check. @@ -2570,12 +2598,54 @@ class LlamaCppBackend: self._healthy = True + # Catch silent CPU fallback when GPU was intended (#5106). + self._gpu_offload_active = self._classify_gpu_offload( + gpu_indices is not None or use_fit, gpus or [] + ) + if self._gpu_offload_active is False: + logger.warning( + "llama-server appears to have loaded the model entirely " + "on CPU even though Studio detected at least one GPU. " + "This usually means the prebuilt binary's GPU backend " + "failed to load -- on Windows, cudart64_X.dll / " + "cublas64_X.dll could not be resolved. Reinstall the " + "Studio llama.cpp prebuilt or install a matching CUDA " + "toolkit (issue unslothai/unsloth#5106).", + ) + logger.info( f"llama-server ready on port {self._port} " f"for model '{model_identifier}'" ) return True + def _classify_gpu_offload( + self, + expected_gpu: bool, + detected_gpus: list[tuple[int, int]], + ) -> Optional[bool]: + """True if a GPU model buffer was allocated, False if only CPU + buffers landed despite GPU intent, None when there's no signal + (no GPU detected, no buffer-size lines, etc.).""" + if not detected_gpus or not expected_gpu: + return None + # llama-server logs one ``... model buffer size = N MiB`` line + # per backend buffer; CUDA0 / ROCm0 / Metal / Vulkan0 / + # OpenCL0 / SYCL0 are GPU, CPU / CPU_Mapped are not. + gpu_markers = ("CUDA", "ROCm", "Metal", "Vulkan", "OpenCL", "SYCL") + saw_buffer_line = False + saw_gpu_buffer = False + for line in self._stdout_lines: + if "model buffer size" not in line: + continue + saw_buffer_line = True + if any(marker in line for marker in gpu_markers): + saw_gpu_buffer = True + break + if not saw_buffer_line: + return None + return saw_gpu_buffer + def unload_model(self) -> bool: """Terminate the llama-server subprocess and cancel any in-flight download.""" self._cancel_event.set() diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 87cc933d4b..70db5477d4 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -10,6 +10,7 @@ Supports web search (DuckDuckGo), Python code execution, and terminal commands. import ast import http.client import os +import signal os.environ["UNSLOTH_IS_PRESENT"] = "1" @@ -58,21 +59,37 @@ _MAX_OUTPUT_CHARS = 8000 # truncate long output _BLOCKED_COMMANDS_COMMON = frozenset( { "rm", - "sudo", - "su", "dd", "chmod", "chown", "mkfs", - "shutdown", - "reboot", - "passwd", "mount", "umount", "fdisk", + "sudo", + "su", + "doas", + "pkexec", + "shutdown", + "reboot", + "halt", + "poweroff", "kill", "killall", "pkill", + "passwd", + "curl", + "wget", + "nc", + "ncat", + "netcat", + "socat", + "ssh", + "scp", + "sftp", + "rsync", + "eval", + "source", } ) _BLOCKED_COMMANDS_WIN = frozenset( @@ -221,35 +238,67 @@ def _build_safe_env(workdir: str) -> dict[str, str]: def _sandbox_preexec(): - """Pre-exec hook: drop privilege escalation ability and set resource limits. + """Best-effort sandbox setup for sandboxed subprocesses. - On Linux, applies PR_SET_NO_NEW_PRIVS so sudo/su/pkexec fail at the - kernel level. On Linux and macOS, sets RLIMIT_FSIZE. - No-op on Windows (use creationflags instead). - - Note: RLIMIT_NPROC is intentionally NOT set because Linux enforces it - per real UID, not per process tree, so it would starve the Studio - server and other sessions sharing the same user account. - - All modules and handles are resolved at import time (module level) so - this function does not trigger Python imports in the forked child, - avoiding potential deadlocks in multi-threaded servers. + Modules are resolved at import time so the forked child runs no imports. """ + try: + os.setsid() + except OSError: + pass + + try: + os.umask(0o077) + except OSError: + pass + if _libc is not None: try: - # PR_SET_NO_NEW_PRIVS = 38, arg2 = 1 (enable) - _libc.prctl(38, 1, 0, 0, 0) + _libc.prctl(38, 1, 0, 0, 0) # PR_SET_NO_NEW_PRIVS except (OSError, AttributeError): - pass # Not available (container, old kernel, etc.) + pass + + try: + _libc.prctl(1, 9, 0, 0, 0) # PR_SET_PDEATHSIG = SIGKILL + except (OSError, AttributeError): + pass + + # CLONE_NEWNET intentionally not applied: where userns is enabled it + # blocks all egress, including allowlisted hosts. Network policy is + # enforced by the AST host check and the bash blocklist. if _resource is not None: + # RLIMIT_NPROC is per-real-UID, so the cap is well above normal usage. + try: + nproc = int(os.environ.get("UNSLOTH_STUDIO_SANDBOX_NPROC", "10000")) + _resource.setrlimit(_resource.RLIMIT_NPROC, (nproc, nproc)) + except (ValueError, OSError, AttributeError): + pass try: - # Limit file size to 100MB (prevents disk filling) _resource.setrlimit( _resource.RLIMIT_FSIZE, (100 * 1024 * 1024, 100 * 1024 * 1024) ) except (ValueError, OSError): pass + try: + as_bytes = ( + int(os.environ.get("UNSLOTH_STUDIO_SANDBOX_AS_GB", "8")) + * 1024 + * 1024 + * 1024 + ) + _resource.setrlimit(_resource.RLIMIT_AS, (as_bytes, as_bytes)) + except (ValueError, OSError, AttributeError): + pass + try: + cpu_s = int(os.environ.get("UNSLOTH_STUDIO_SANDBOX_CPU_S", "600")) + _resource.setrlimit(_resource.RLIMIT_CPU, (cpu_s, cpu_s)) + except (ValueError, OSError, AttributeError): + pass + try: + _resource.setrlimit(_resource.RLIMIT_NOFILE, (1024, 1024)) + except (ValueError, OSError, AttributeError): + pass def _get_shell_cmd(command: str) -> list[str]: @@ -265,25 +314,36 @@ def _get_shell_cmd(command: str) -> list[str]: _workdirs: dict[str, str] = {} +# Non-matching session_ids collapse to ``_invalid`` to block cross-session escapes. +_SESSION_ID_RE = re.compile(r"\A[A-Za-z0-9_\-]{1,64}\Z") + + def _get_workdir(session_id: str | None = None) -> str: - """Return (and lazily create) a persistent working directory for tool execution.""" + """Return a per-session sandbox dir at mode 0o700.""" global _workdirs key = session_id or "_default" if key not in _workdirs or not os.path.isdir(_workdirs[key]): home = os.path.expanduser("~") sandbox_root = os.path.join(home, "studio_sandbox") - if session_id: - # Sanitize: strip path separators and parent-dir references - safe_id = os.path.basename(session_id.replace("..", "")) - if not safe_id: - safe_id = "_invalid" - workdir = os.path.join(sandbox_root, safe_id) - # Verify resolved path stays under sandbox root - if not os.path.realpath(workdir).startswith(os.path.realpath(sandbox_root)): + if session_id and _SESSION_ID_RE.match(session_id): + workdir = os.path.join(sandbox_root, session_id) + if not os.path.realpath(workdir).startswith( + os.path.realpath(sandbox_root) + os.sep + ): workdir = os.path.join(sandbox_root, "_invalid") + elif session_id: + workdir = os.path.join(sandbox_root, "_invalid") else: workdir = os.path.join(sandbox_root, "_default") os.makedirs(workdir, exist_ok = True) + try: + os.chmod(sandbox_root, 0o700) + except OSError: + pass + try: + os.chmod(workdir, 0o700) + except OSError: + pass _workdirs[key] = workdir return _workdirs[key] @@ -932,7 +992,12 @@ def _check_signal_escape_patterns(code: str): isinstance(shell_node, ast.Constant) and shell_node.value is False ) - if shell_func in _STRING_SHELL_FUNCS or not shell_safe: + # Dynamic shell-exec args (chr/format/concat bypasses). + if ( + shell_func in _STRING_SHELL_FUNCS + or shell_func in _SHELL_EXEC_FUNCS + or not shell_safe + ): def _is_safe_literal(n): if _extract_string_from_node(n) is not None: @@ -1006,15 +1071,418 @@ def _check_signal_escape_patterns(code: str): if visitor.imports_signal and not signal_tampering: warnings.append("Code imports 'signal' module - review manually for safety") + # Static host policy: block metadata hosts and any literal host outside + # the trusted allowlist; uploads blocked regardless of host. Dynamic hosts + # are caught by the bash blocklist instead. + network_calls: list[dict] = [] + sensitive_file_reads: list[dict] = [] + _NETWORK_FQ_PREFIXES = ( + "socket.socket", + "socket.create_connection", + "socket.getaddrinfo", + "urllib.request.urlopen", + "urllib.request.urlretrieve", + "urllib3.", + "requests.get", + "requests.post", + "requests.put", + "requests.delete", + "requests.patch", + "requests.head", + "requests.request", + "requests.Session", + "http.client.HTTPConnection", + "http.client.HTTPSConnection", + "httpx.get", + "httpx.post", + "httpx.put", + "httpx.patch", + "httpx.delete", + "httpx.request", + "httpx.Client", + "httpx.AsyncClient", + "aiohttp.ClientSession", + ) + _UPLOAD_HTTP_METHODS = ( + "requests.post", + "requests.put", + "requests.patch", + "requests.delete", + "requests.request", + "httpx.post", + "httpx.put", + "httpx.patch", + "httpx.delete", + "httpx.request", + "urllib.request.urlopen", + "urllib.request.Request", + ) + _UPLOAD_HF_FQ = ( + "huggingface_hub.upload_file", + "huggingface_hub.upload_folder", + "huggingface_hub.upload_large_folder", + "huggingface_hub.create_commit", + ) + _UPLOAD_HF_METHODS = frozenset( + { + "upload_file", + "upload_folder", + "upload_large_folder", + "create_commit", + } + ) + # Cloud-metadata / link-local hosts. + _METADATA_HOST_LITERALS = { + "169.254.169.254", + "fd00:ec2::254", + "metadata.google.internal", + "metadata", + "metadata.tencentyun.com", + "100.100.100.200", + "100.100.100.110", + "169.254.170.2", + "169.254.170.23", + } + _METADATA_HOST_PREFIXES = ( + "169.254.", + "100.64.", + ) + # Allowlist kept explicit so each entry is auditable. + _TRUSTED_PUBLIC_HOST_LITERALS = frozenset( + { + # search + "www.google.com", + "google.com", + "www.bing.com", + "bing.com", + "duckduckgo.com", + "html.duckduckgo.com", + # encyclopedic / reference + "wikipedia.org", + "www.wikipedia.org", + "wikimedia.org", + "www.wikimedia.org", + "wikidata.org", + "www.wikidata.org", + "commons.wikimedia.org", + "www.britannica.com", + "openlibrary.org", + "www.openstreetmap.org", + # ML / dev / data + "huggingface.co", + "hf.co", + "github.com", + "api.github.com", + "raw.githubusercontent.com", + "gist.github.com", + "docs.github.com", + "pypi.org", + "files.pythonhosted.org", + "www.npmjs.com", + "registry.npmjs.org", + "crates.io", + "static.crates.io", + # docs + "docs.python.org", + "python.org", + "www.python.org", + "developer.mozilla.org", + "developer.apple.com", + "learn.microsoft.com", + "docs.docker.com", + "pytorch.org", + "docs.pytorch.org", + "tensorflow.org", + "www.tensorflow.org", + "numpy.org", + "pandas.pydata.org", + "scipy.org", + "scikit-learn.org", + "matplotlib.org", + "fastapi.tiangolo.com", + "starlette.io", + # academic + "arxiv.org", + "export.arxiv.org", + "scholar.google.com", + "openreview.net", + "semanticscholar.org", + "www.semanticscholar.org", + "biorxiv.org", + "www.biorxiv.org", + "medrxiv.org", + "www.medrxiv.org", + "pubmed.ncbi.nlm.nih.gov", + "www.ncbi.nlm.nih.gov", + # Q&A / community + "stackoverflow.com", + "stackexchange.com", + "askubuntu.com", + "superuser.com", + "serverfault.com", + # standards + "www.w3.org", + "tools.ietf.org", + "datatracker.ietf.org", + "www.rfc-editor.org", + # reputable news + "www.bbc.com", + "www.bbc.co.uk", + "www.reuters.com", + "apnews.com", + "www.nature.com", + "www.science.org", + # government / open data + "data.gov", + "catalog.data.gov", + "www.census.gov", + "www.nasa.gov", + "data.nasa.gov", + "www.cdc.gov", + "www.nih.gov", + "www.who.int", + # weather / time + "api.weather.gov", + "worldtimeapi.org", + } + ) + _TRUSTED_PUBLIC_HOST_SUFFIXES = ( + ".wikipedia.org", + ".wikimedia.org", + ".wiktionary.org", + ".wikibooks.org", + ".wikiquote.org", + ".wikisource.org", + ".wikiversity.org", + ".wikivoyage.org", + ".stackexchange.com", + ".hf.co", + ".huggingface.co", + ".githubusercontent.com", + ".github.io", + ".arxiv.org", + ".readthedocs.io", + ".readthedocs.org", + ) + _SENSITIVE_FILE_PREFIXES = ( + "/etc/passwd", + "/etc/shadow", + "/etc/sudoers", + "/etc/ssh/", + ) + _SENSITIVE_FILE_RE = re.compile( + r"^/proc/(?:self|\d+)/(?:environ|cmdline|task/\d+/environ)$" + ) + + def _normalize_host(host: str) -> str: + if not host: + return "" + h = host.strip().lower().rstrip(".") + if "@" in h: + h = h.split("@", 1)[1] + if h.startswith("[") and "]" in h: + h = h[1 : h.index("]")] + elif h.count(":") == 1: + h = h.split(":", 1)[0] + return h + + def _is_metadata_host(host: str) -> bool: + h = _normalize_host(host) + if not h: + return False + if h in _METADATA_HOST_LITERALS: + return True + if any(h.startswith(p) for p in _METADATA_HOST_PREFIXES): + return True + return False + + def _is_trusted_host(host: str) -> bool: + h = _normalize_host(host) + if not h: + return False + if h in _TRUSTED_PUBLIC_HOST_LITERALS: + return True + return any(h.endswith(s) for s in _TRUSTED_PUBLIC_HOST_SUFFIXES) + + def _call_is_upload_shape(node: ast.Call, fq: str) -> bool: + """True for statically obvious upload shapes (files=, data=open(), bytes literal).""" + if fq in _UPLOAD_HF_FQ: + return True + if fq not in _UPLOAD_HTTP_METHODS: + return False + for kw in node.keywords or []: + if kw.arg == "files": + return True + if kw.arg == "data": + v = kw.value + if ( + isinstance(v, ast.Call) + and isinstance(v.func, ast.Name) + and v.func.id == "open" + ): + return True + if isinstance(v, ast.Constant) and isinstance( + v.value, (bytes, bytearray) + ): + return True + return False + + def _method_call_is_hf_upload(node: ast.Call) -> bool: + """True for HfApi upload method names on any receiver.""" + return ( + isinstance(node.func, ast.Attribute) + and node.func.attr in _UPLOAD_HF_METHODS + ) + + class NetworkAndIoVisitor(ast.NodeVisitor): + def visit_Call(self, node): + parts: list[str] = [] + cur = node.func + while isinstance(cur, ast.Attribute): + parts.insert(0, cur.attr) + cur = cur.value + if isinstance(cur, ast.Name): + parts.insert(0, cur.id) + fq = ".".join(parts) if parts else "" + + if _method_call_is_hf_upload(node): + network_calls.append( + { + "type": "upload_blocked", + "line": getattr(node, "lineno", -1), + "description": ("Blocked: file upload disallowed in sandbox"), + } + ) + + # Direct sock.connect((host, port)) bypasses the FQ-prefix branch below. + if ( + isinstance(node.func, ast.Attribute) + and node.func.attr == "connect" + and node.args + ): + a0 = node.args[0] + host_lit = None + if isinstance(a0, ast.Tuple) and a0.elts: + e0 = a0.elts[0] + if isinstance(e0, ast.Constant) and isinstance(e0.value, str): + host_lit = e0.value + elif isinstance(a0, ast.Constant) and isinstance(a0.value, str): + host_lit = a0.value + if host_lit: + if _is_metadata_host(host_lit): + network_calls.append( + { + "type": "metadata_host_blocked", + "line": getattr(node, "lineno", -1), + "description": "Blocked: cloud-metadata host", + } + ) + elif not _is_trusted_host(host_lit): + network_calls.append( + { + "type": "untrusted_host_blocked", + "line": getattr(node, "lineno", -1), + "description": ( + "Blocked: host not in sandbox allowlist; " + "use an allowed informational source" + ), + } + ) + + if fq and any(fq.startswith(p) for p in _NETWORK_FQ_PREFIXES): + # 1) Upload-shape check (host-independent). + if _call_is_upload_shape(node, fq): + network_calls.append( + { + "type": "upload_blocked", + "line": getattr(node, "lineno", -1), + "description": ( + "Blocked: file upload disallowed in sandbox" + ), + } + ) + + # 2) Extract literal host (URL string or (host, port) tuple). + host_arg = None + url_arg = None + if node.args: + a0 = node.args[0] + if isinstance(a0, ast.Constant) and isinstance(a0.value, str): + url_arg = a0.value + elif isinstance(a0, ast.Tuple) and a0.elts: + e0 = a0.elts[0] + if isinstance(e0, ast.Constant) and isinstance(e0.value, str): + host_arg = e0.value + if url_arg and host_arg is None: + m = re.match(r"^\w+://([^/?#]+)", url_arg) + if m: + host_arg = m.group(1) + + if host_arg: + if _is_metadata_host(host_arg): + network_calls.append( + { + "type": "metadata_host_blocked", + "line": getattr(node, "lineno", -1), + "description": "Blocked: cloud-metadata host", + } + ) + elif not _is_trusted_host(host_arg): + network_calls.append( + { + "type": "untrusted_host_blocked", + "line": getattr(node, "lineno", -1), + "description": ( + "Blocked: host not in sandbox allowlist; " + "use an allowed informational source" + ), + } + ) + + is_open_call = ( + (isinstance(node.func, ast.Name) and node.func.id == "open") + or fq in ("io.open", "pathlib.Path.open") + or fq.endswith(".open") + ) + if is_open_call and node.args: + a0 = node.args[0] + path_lit = None + if isinstance(a0, ast.Constant) and isinstance(a0.value, str): + path_lit = a0.value + if path_lit: + flagged = False + if any(path_lit.startswith(p) for p in _SENSITIVE_FILE_PREFIXES): + flagged = True + elif _SENSITIVE_FILE_RE.match(path_lit): + flagged = True + if flagged: + sensitive_file_reads.append( + { + "type": "sensitive_file_read", + "line": getattr(node, "lineno", -1), + "description": ( + f"open({path_lit!r}) targets a host identity / " + "credential file; sandboxed code may not read it" + ), + } + ) + self.generic_visit(node) + + NetworkAndIoVisitor().visit(tree) + is_safe = ( len(signal_tampering) == 0 and len(exception_catching) == 0 and len(shell_escapes) == 0 + and len(network_calls) == 0 + and len(sensitive_file_reads) == 0 ) return is_safe, { "signal_tampering": signal_tampering, "exception_catching": exception_catching, "shell_escapes": shell_escapes, + "network_calls": network_calls, + "sensitive_file_reads": sensitive_file_reads, "warnings": warnings, } @@ -1041,7 +1509,21 @@ def _check_code_safety(code: str) -> str | None: exception_reasons = [ item.get("description", "") for item in info.get("exception_catching", []) ] - all_reasons = [r for r in reasons + shell_reasons + exception_reasons if r] + network_reasons = [ + item.get("description", "") for item in info.get("network_calls", []) + ] + file_reasons = [ + item.get("description", "") for item in info.get("sensitive_file_reads", []) + ] + all_reasons = [ + r + for r in reasons + + shell_reasons + + exception_reasons + + network_reasons + + file_reasons + if r + ] if all_reasons: return ( f"Error: unsafe code detected ({'; '.join(all_reasons)}). " @@ -1051,11 +1533,31 @@ def _check_code_safety(code: str) -> str | None: return None +def _kill_process_tree(proc) -> None: + """SIGKILL the setsid process group; fall back to single-pid kill.""" + if proc.poll() is not None: + return + try: + pgid = os.getpgid(proc.pid) + except (ProcessLookupError, PermissionError): + pgid = None + if pgid is not None: + try: + os.killpg(pgid, signal.SIGKILL) + return + except (ProcessLookupError, PermissionError): + pass + try: + proc.kill() + except (ProcessLookupError, PermissionError): + pass + + def _cancel_watcher(proc, cancel_event, poll_interval = 0.2): """Daemon thread that kills a process when cancel_event is set.""" while proc.poll() is None: if cancel_event is not None and cancel_event.is_set(): - proc.kill() + _kill_process_tree(proc) return cancel_event.wait(poll_interval) if cancel_event else None @@ -1126,8 +1628,11 @@ def _python_exec( try: output, _ = proc.communicate(timeout = timeout) except subprocess.TimeoutExpired: - proc.kill() - proc.communicate() + _kill_process_tree(proc) + try: + proc.communicate(timeout = 5) + except subprocess.TimeoutExpired: + pass return _truncate(f"Execution timed out after {timeout} seconds.") if cancel_event is not None and cancel_event.is_set(): @@ -1211,8 +1716,11 @@ def _bash_exec( try: output, _ = proc.communicate(timeout = timeout) except subprocess.TimeoutExpired: - proc.kill() - proc.communicate() + _kill_process_tree(proc) + try: + proc.communicate(timeout = 5) + except subprocess.TimeoutExpired: + pass return _truncate(f"Execution timed out after {timeout} seconds.") if cancel_event is not None and cancel_event.is_set(): diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.py index 72b13c3225..549d733252 100644 --- a/studio/backend/core/training/training.py +++ b/studio/backend/core/training/training.py @@ -17,7 +17,9 @@ Pattern follows core/data_recipe/jobs/manager.py. import json as _json import math import multiprocessing as mp +import os import queue +import shutil import threading import time import structlog @@ -33,9 +35,54 @@ from utils.native_path_leases import ( native_path_secret_removed_for_child_start, run_without_native_path_secret, ) +from utils.paths import outputs_root logger = get_logger(__name__) + +def _cleanup_cancelled_checkpoints(output_dir: str | os.PathLike) -> None: + """Remove ``checkpoint-`` subdirs after a cancelled run. + Only paths whose realpath is under outputs_root are touched.""" + out = Path(output_dir) + if not out.exists(): + return + try: + out_real = out.resolve() + out_root_real = Path(outputs_root()).resolve() + except OSError: + return + try: + out_real.relative_to(out_root_real) + except ValueError: + # Refuse to delete anything outside the configured outputs root. + logger.warning( + "Skipping checkpoint cleanup - %s is not under outputs_root %s", + out_real, + out_root_real, + ) + return + removed = 0 + for entry in out.iterdir() if out.is_dir() else []: + if not entry.is_dir(): + continue + name = entry.name + if not name.startswith("checkpoint-"): + continue + tail = name[len("checkpoint-") :] + if not tail.isdigit(): + continue + try: + shutil.rmtree(entry, ignore_errors = False) + removed += 1 + except OSError as exc: + logger.warning("Could not remove %s: %s", entry, exc) + logger.info( + "Cancelled-run cleanup removed %d checkpoint dir(s) under %s", + removed, + out, + ) + + _CTX = mp.get_context("spawn") # Plot styling constants @@ -316,6 +363,8 @@ class TrainingBackend: ) self._proc.terminate() proc = self._proc + cancelled = self._cancel_requested + output_dir = self._output_dir if proc is not None: proc.join(timeout = 5.0) @@ -328,6 +377,17 @@ class TrainingBackend: if self._pump_thread is not None and self._pump_thread.is_alive(): self._pump_thread.join(timeout = 8.0) + # Drop checkpoint-* dirs on explicit cancel only; stop-and-save + # keeps its artifacts. + if cancelled and output_dir: + try: + _cleanup_cancelled_checkpoints(output_dir) + except Exception: + logger.exception( + "Failed to clean up cancelled-run checkpoints under %s", + output_dir, + ) + def is_training_active(self) -> bool: """Check if training is currently active.""" with self._lock: diff --git a/studio/backend/main.py b/studio/backend/main.py index 633b112dc8..4955e988e6 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -42,6 +42,7 @@ if _STUDIO_ROOT_RESOLVED != _LEGACY_STUDIO_ROOT: if not os.environ.get("UNSLOTH_LLAMA_CPP_PATH"): os.environ["UNSLOTH_LLAMA_CPP_PATH"] = str(_STUDIO_ROOT_RESOLVED / "llama.cpp") +import hashlib import mimetypes import re as _re import shutil @@ -103,7 +104,7 @@ if os.getenv("ENVIRONMENT_TYPE", "production") == "production": # warnings.filterwarnings("ignore", category=DeprecationWarning) # warnings.filterwarnings("ignore", module="triton.*") -from fastapi import Depends, FastAPI, Request +from fastapi import Depends, FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.staticfiles import StaticFiles from fastapi.responses import FileResponse, HTMLResponse, Response @@ -134,6 +135,11 @@ import utils.hardware.hardware as _hw_module from utils.cache_cleanup import clear_unsloth_compiled_cache from utils.native_path_leases import native_path_leases_supported +from utils.update_status import ( + get_studio_install_source_status, + get_studio_update_status, +) +from utils.studio_version import get_studio_version def get_unsloth_version() -> str: @@ -155,6 +161,25 @@ def get_unsloth_version() -> str: UNSLOTH_VERSION = get_unsloth_version() +STUDIO_VERSION = get_studio_version() + + +def _load_desktop_owner() -> dict[str, str] | None: + token = os.environ.pop("UNSLOTH_STUDIO_DESKTOP_OWNER_TOKEN", "") + kind = os.environ.pop("UNSLOTH_STUDIO_DESKTOP_OWNER_KIND", "") + if kind != "tauri" or not token: + return None + return { + "kind": "tauri", + "token_sha256": hashlib.sha256(token.encode("utf-8")).hexdigest(), + } + + +_DESKTOP_OWNER = _load_desktop_owner() + + +def _desktop_owner() -> dict[str, str] | None: + return _DESKTOP_OWNER @asynccontextmanager @@ -235,6 +260,181 @@ logger = LogConfig.setup_logging( app.add_middleware(LoggingMiddleware) + +# Web-search favicons load from *.gstatic.com; everything else is same-origin. +from starlette.middleware.base import BaseHTTPMiddleware # noqa: E402 +from starlette.requests import Request as _StarletteRequest # noqa: E402 + + +_CSP_SCRIPT_NONCE_HEADER = "x-internal-script-nonce" + + +def _build_csp(script_nonce: "str | None" = None) -> str: + script_src = "script-src 'self'" + if script_nonce: + script_src += f" 'nonce-{script_nonce}'" + return ( + "default-src 'self'; " + "img-src 'self' data: blob: https://t0.gstatic.com " + "https://t1.gstatic.com https://t2.gstatic.com " + "https://t3.gstatic.com; " + "connect-src 'self' https://huggingface.co https://datasets-server.huggingface.co; " + "style-src 'self' 'unsafe-inline'; " + f"{script_src}; " + "font-src 'self' data:; " + "frame-ancestors 'none'; " + "form-action 'self'; " + "base-uri 'self'" + ) + + +class SecurityHeadersMiddleware(BaseHTTPMiddleware): + """Set baseline security headers; splice per-response inline-script nonces into CSP.""" + + async def dispatch(self, request: _StarletteRequest, call_next): + response = await call_next(request) + # Strip the internal nonce hand-off header so it never reaches the client. + nonce = response.headers.get(_CSP_SCRIPT_NONCE_HEADER) + if nonce is not None: + del response.headers[_CSP_SCRIPT_NONCE_HEADER] + response.headers.setdefault("Content-Security-Policy", _build_csp(nonce)) + response.headers.setdefault("X-Frame-Options", "DENY") + response.headers.setdefault("X-Content-Type-Options", "nosniff") + response.headers.setdefault("Referrer-Policy", "no-referrer") + response.headers.setdefault( + "Permissions-Policy", + "camera=(), microphone=(), geolocation=(), interest-cohort=()", + ) + response.headers["server"] = "unsloth-studio" + return response + + +app.add_middleware(SecurityHeadersMiddleware) + + +# Cap upload body on protected POSTs; default 500 MB, env-tunable. +import json as _json_for_413 # noqa: E402 + + +_MAX_BODY_BYTES = int(os.environ.get("UNSLOTH_STUDIO_MAX_BODY_MB", "500")) * 1024 * 1024 +_BODY_PROTECTED_PREFIXES = ( + "/v1/chat/completions", + "/v1/completions", + "/api/inference", + "/api/data-recipe", + "/api/datasets", + "/api/train", + "/api/export", +) + + +async def _send_413(send, total_bytes: int) -> None: + payload = _json_for_413.dumps( + { + "detail": ( + f"Request body too large " + f"({total_bytes:,} bytes; max {_MAX_BODY_BYTES:,})." + ) + }, + ).encode("utf-8") + await send( + { + "type": "http.response.start", + "status": 413, + "headers": [ + (b"content-type", b"application/json"), + (b"content-length", str(len(payload)).encode("ascii")), + ], + } + ) + await send({"type": "http.response.body", "body": payload, "more_body": False}) + + +class MaxBodyMiddleware: + """Reject oversized bodies on protected POST/PUT/PATCH; raw ASGI so chunked uploads cannot bypass the cap.""" + + def __init__(self, app, max_bytes: int, protected_prefixes: tuple): + self.app = app + self.max_bytes = max_bytes + self.protected_prefixes = protected_prefixes + + async def __call__(self, scope, receive, send): + if scope["type"] != "http": + await self.app(scope, receive, send) + return + method = scope.get("method", "").upper() + path = scope.get("path", "") + if method not in ("POST", "PUT", "PATCH") or not any( + path.startswith(p) for p in self.protected_prefixes + ): + await self.app(scope, receive, send) + return + + declared = None + for name, value in scope.get("headers", []): + if name == b"content-length": + try: + declared = int(value.decode("latin-1")) + except (ValueError, UnicodeDecodeError): + declared = None + break + if declared is not None and declared > self.max_bytes: + await _send_413(send, declared) + return + + chunks: list = [] + total = 0 + while True: + msg = await receive() + mtype = msg.get("type") + if mtype == "http.disconnect": + return + if mtype != "http.request": + # Mid-stream unexpected frame: forwarding would corrupt downstream. + return + body = msg.get("body", b"") or b"" + if body: + total += len(body) + if total > self.max_bytes: + await _send_413(send, total) + return + chunks.append(body) + if not msg.get("more_body", False): + break + + replayed = {"sent": False} + + async def replay_receive(): + if not replayed["sent"]: + replayed["sent"] = True + return { + "type": "http.request", + "body": b"".join(chunks), + "more_body": False, + } + # After replay, fall through so http.disconnect still propagates. + return await receive() + + await self.app(scope, replay_receive, send) + + +app.add_middleware( + MaxBodyMiddleware, + max_bytes = _MAX_BODY_BYTES, + protected_prefixes = _BODY_PROTECTED_PREFIXES, +) + + +from starlette.responses import RedirectResponse as _RedirectResponse # noqa: E402 + + +@app.get("/recipes", include_in_schema = False) +@app.get("/recipes/{rest:path}", include_in_schema = False) +async def _recipes_redirect(rest: str = ""): + target = "/data-recipes" + (("/" + rest) if rest else "") + return _RedirectResponse(url = target, status_code = 308) + + # CORS middleware _api_only = os.environ.get("UNSLOTH_API_ONLY") == "1" _cors_origins = ["*"] @@ -286,28 +486,63 @@ app.include_router( @app.get("/api/health") -async def health_check(): - """Health check endpoint""" - platform_map = {"darwin": "mac", "win32": "windows", "linux": "linux"} - device_type = platform_map.get(sys.platform, sys.platform) - - return { +async def health_check(request: Request): + """Liveness only; full diagnostic dict gated on a valid bearer.""" + minimal = { "status": "healthy", "timestamp": datetime.now().isoformat(), + } + auth = request.headers.get("authorization", "") + if not auth.lower().startswith("bearer "): + return minimal + try: + from auth.authentication import get_current_subject as _gcs + from fastapi.security import HTTPAuthorizationCredentials + + creds = HTTPAuthorizationCredentials( + scheme = "Bearer", credentials = auth.split(" ", 1)[1] + ) + # Must await: a bare coroutine is truthy and would skip the auth check. + subject = await _gcs(creds) + except HTTPException: + return minimal + except Exception: + return minimal + if not subject: + return minimal + + platform_map = {"darwin": "mac", "win32": "windows", "linux": "linux"} + device_type = platform_map.get(sys.platform, sys.platform) + return { + **minimal, "service": "Unsloth UI Backend", "version": UNSLOTH_VERSION, + "studio_version": STUDIO_VERSION, "device_type": device_type, "chat_only": _hw_module.CHAT_ONLY, "desktop_protocol_version": 1, + "desktop_manageability_version": 1, "supports_desktop_auth": True, - # why: launchers compare against an install-time hash so a sibling - # Studio on the same port is rejected; hex digest avoids leaking the - # raw install path on -H 0.0.0.0. + "supports_desktop_backend_ownership": True, + # Hex digest of the install path; launchers reject sibling Studios on the same port. "studio_root_id": _studio_root_id(), "native_path_leases_supported": native_path_leases_supported(), + **({"desktop_owner": owner} if (owner := _desktop_owner()) else {}), } +@app.get("/api/studio/install-source") +def studio_install_source(_current_subject: str = Depends(get_current_subject)): + """Return source-aware install metadata without remote update checks.""" + return get_studio_install_source_status(UNSLOTH_VERSION) + + +@app.get("/api/studio/update-status") +def studio_update_status(_current_subject: str = Depends(get_current_subject)): + """Return source-aware manual update status for browser-served Studio.""" + return get_studio_update_status(UNSLOTH_VERSION) + + @app.post("/api/shutdown") async def shutdown_server( request: Request, @@ -422,21 +657,22 @@ def _strip_crossorigin(html_bytes: bytes) -> bytes: return html.encode("utf-8") -def _inject_bootstrap(html_bytes: bytes, app: FastAPI) -> bytes: - """Inject bootstrap credentials into HTML when password change is required. +def _inject_bootstrap(html_bytes: bytes, app: FastAPI): + """Inject bootstrap credentials when password change is pending. - The script tag is only injected while the default admin account still - has ``must_change_password=True``. Once the user changes the password - the HTML is served clean — no credentials leak. + Returns ``(html_bytes, script_nonce_or_None)``. Callers must forward + the nonce via ``_CSP_SCRIPT_NONCE_HEADER`` so the inline script is + not blocked by CSP. """ import json as _json + import secrets as _secrets if not storage.requires_password_change(storage.DEFAULT_ADMIN_USERNAME): - return html_bytes + return html_bytes, None bootstrap_pw = getattr(app.state, "bootstrap_password", None) if not bootstrap_pw: - return html_bytes + return html_bytes, None payload = _json.dumps( { @@ -444,10 +680,11 @@ def _inject_bootstrap(html_bytes: bytes, app: FastAPI) -> bytes: "password": bootstrap_pw, } ) - tag = f"" + nonce = _secrets.token_urlsafe(16) + tag = f'' html = html_bytes.decode("utf-8") html = html.replace("", f"{tag}", 1) - return html.encode("utf-8") + return html.encode("utf-8"), nonce def setup_frontend(app: FastAPI, build_path: Path): @@ -460,17 +697,23 @@ def setup_frontend(app: FastAPI, build_path: Path): if assets_dir.exists(): app.mount("/assets", StaticFiles(directory = assets_dir), name = "assets") - @app.get("/") - async def serve_root(): + def _build_index_response() -> Response: content = (build_path / "index.html").read_bytes() content = _strip_crossorigin(content) - content = _inject_bootstrap(content, app) + content, nonce = _inject_bootstrap(content, app) + headers = {"Cache-Control": "no-cache, no-store, must-revalidate"} + if nonce: + headers[_CSP_SCRIPT_NONCE_HEADER] = nonce return Response( content = content, media_type = "text/html", - headers = {"Cache-Control": "no-cache, no-store, must-revalidate"}, + headers = headers, ) + @app.get("/") + async def serve_root(): + return _build_index_response() + @app.get("/{full_path:path}") async def serve_frontend(full_path: str): if full_path in {"api", "v1"} or full_path.startswith(("api/", "v1/")): @@ -486,13 +729,6 @@ def setup_frontend(app: FastAPI, build_path: Path): return FileResponse(file_path) # Serve index.html as bytes — avoids Content-Length mismatch - content = (build_path / "index.html").read_bytes() - content = _strip_crossorigin(content) - content = _inject_bootstrap(content, app) - return Response( - content = content, - media_type = "text/html", - headers = {"Cache-Control": "no-cache, no-store, must-revalidate"}, - ) + return _build_index_response() return True diff --git a/studio/backend/models/auth.py b/studio/backend/models/auth.py index 23eb0ac4c0..b7870379f7 100644 --- a/studio/backend/models/auth.py +++ b/studio/backend/models/auth.py @@ -37,7 +37,10 @@ class AuthStatusResponse(BaseModel): initialized: bool = Field( ..., description = "True if the auth database contains a login user" ) - default_username: str = Field(..., description = "Default seeded admin username") + default_username: str = Field( + "unsloth", + description = "Default admin username for first-boot UI prefill.", + ) requires_password_change: bool = Field( ..., description = "True if the seeded admin must still change the default password", diff --git a/studio/backend/models/export.py b/studio/backend/models/export.py index a86596f199..86ce2b05bf 100644 --- a/studio/backend/models/export.py +++ b/studio/backend/models/export.py @@ -5,10 +5,36 @@ Pydantic schemas for Export API. """ -from pydantic import BaseModel, Field +from pathlib import Path + +from pydantic import BaseModel, Field, field_validator from typing import List, Optional, Literal, Dict, Any +def _validate_save_directory(value: str) -> str: + """Reject save_directory values that escape the export root.""" + if value is None: + raise ValueError("save_directory is required") + raw = str(value).strip() + if not raw: + raise ValueError("save_directory must not be empty") + if "\x00" in raw: + raise ValueError("save_directory may not contain null bytes") + if any(ch in raw for ch in ("\r", "\n")): + raise ValueError("save_directory may not contain control characters") + if len(raw) > 255: + raise ValueError("save_directory must be <= 255 characters") + path = Path(raw).expanduser() + if path.is_absolute(): + raise ValueError( + "save_directory must be a name or relative path under the " + "export root; absolute paths are rejected" + ) + if ".." in path.parts: + raise ValueError("save_directory may not contain '..' segments") + return raw + + class LoadCheckpointRequest(BaseModel): """Request for loading a checkpoint into the export backend.""" @@ -64,6 +90,12 @@ class ExportCommonOptions(BaseModel): ..., description = "Local directory where the exported artifacts will be written", ) + + @field_validator("save_directory", mode = "before") + @classmethod + def _check_save_directory(cls, v): + return _validate_save_directory(v) + push_to_hub: bool = Field( False, description = "If True, also push the exported model to the Hugging Face Hub", @@ -108,6 +140,12 @@ class ExportGGUFRequest(BaseModel): ..., description = "Directory where GGUF files will be saved", ) + + @field_validator("save_directory", mode = "before") + @classmethod + def _check_save_directory(cls, v): + return _validate_save_directory(v) + quantization_method: str = Field( "Q4_K_M", description = 'GGUF quantization method (e.g. "Q4_K_M")', diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index 7a4c7d0b3c..746ac8bbc2 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -425,14 +425,6 @@ class ChatMessage(BaseModel): @model_validator(mode = "after") def _validate_role_shape(self) -> "ChatMessage": - # Enforce the per-role OpenAI spec shape at the request boundary. - # Without this, malformed messages (e.g. user entries with no - # content, tool_calls on a user/system role, role="tool" without - # tool_call_id) would be silently forwarded to llama-server via - # the passthrough path, surfacing as opaque upstream errors or - # broken tool-call reconciliation downstream. - - # Tool-call metadata must appear only on the appropriate role. if self.tool_calls is not None and self.role != "assistant": raise ValueError('"tool_calls" is only valid on role="assistant" messages.') if self.tool_call_id is not None and self.role != "tool": @@ -440,23 +432,20 @@ class ChatMessage(BaseModel): if self.name is not None and self.role != "tool": raise ValueError('"name" is only valid on role="tool" messages.') - # Per-role content requirements. OpenAI-compatible clients may send - # ``content=""`` for image-only turns when the image travels in a - # companion field such as Studio's ``image_base64`` extension, so treat - # empty strings as present content for user/system messages. if self.role == "tool": if not self.tool_call_id: - raise ValueError( - 'role="tool" messages require "tool_call_id" per the OpenAI spec.' - ) + # Frontend's second-round POST drops the streamed id; + # synthesise one so the request round-trips. + import secrets as _secrets + + self.tool_call_id = f"call_{_secrets.token_hex(8)}" if not self.content: raise ValueError('role="tool" messages require non-empty "content".') elif self.role == "assistant": - # Assistant messages may omit content when tool_calls is set. - if not self.content and not self.tool_calls: - raise ValueError( - 'role="assistant" messages require either "content" or "tool_calls".' - ) + # Tolerate the post-Stop empty-assistant sentinel by + # collapsing content="" to None. + if (self.content == "" or self.content == []) and not self.tool_calls: + self.content = None else: # "user" | "system" if self.content is None or self.content == []: raise ValueError(f'role="{self.role}" messages require "content".') diff --git a/studio/backend/models/training.py b/studio/backend/models/training.py index 0c5825c54e..31f1d575d7 100644 --- a/studio/backend/models/training.py +++ b/studio/backend/models/training.py @@ -5,10 +5,43 @@ Pydantic schemas for Training API """ -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from typing import Any, Optional, List, Dict, Literal +_MAX_BATCH_SIZE = 4096 +_MAX_GRAD_ACCUM = 4096 +_MAX_STEPS = 1_000_000 +_MAX_EPOCHS = 1000 +# 2M is a sanity cap; host RAM runs out long before this. +_MAX_SEQ_LENGTH = 2_000_000 +_MAX_LR_VALUE = 1.0 +_MAX_LORA_R = 16_384 +_MAX_LORA_ALPHA = 32_768 + + +def _parse_lr(v: Any) -> float: + """Parse learning_rate as a positive float strictly below _MAX_LR_VALUE.""" + if v is None: + raise ValueError("learning_rate is required") + if isinstance(v, bool): + raise ValueError("learning_rate must be a number, not a bool") + try: + lr = float(v) + except (TypeError, ValueError): + raise ValueError(f"learning_rate must be parseable as float (got {v!r})") + if not (lr > 0.0): + raise ValueError( + f"learning_rate must be > 0 (got {lr!r}); " "typical range is 1e-6 .. 1e-3" + ) + if lr >= _MAX_LR_VALUE: + raise ValueError( + f"learning_rate must be < 1.0 (got {lr!r}); " + "values that large always diverge training" + ) + return lr + + class TrainingStartRequest(BaseModel): """Request schema for starting training""" @@ -64,6 +97,150 @@ class TrainingStartRequest(BaseModel): values.setdefault("train_split", values.pop("split")) return values + @field_validator("learning_rate", mode = "before") + @classmethod + def _check_learning_rate(cls, v): + # Stringify because downstream call sites float() it themselves. + lr = _parse_lr(v) + return str(lr) + + @field_validator("batch_size") + @classmethod + def _check_batch_size(cls, v: int) -> int: + if v is None: + raise ValueError("batch_size is required") + if v < 1 or v > _MAX_BATCH_SIZE: + raise ValueError( + f"batch_size must be in [1, {_MAX_BATCH_SIZE}] (got {v!r})" + ) + return v + + @field_validator("gradient_accumulation_steps") + @classmethod + def _check_grad_accum(cls, v: int) -> int: + if v is None: + return 1 + if v < 1 or v > _MAX_GRAD_ACCUM: + raise ValueError( + f"gradient_accumulation_steps must be in [1, {_MAX_GRAD_ACCUM}] " + f"(got {v!r})" + ) + return v + + @field_validator("num_epochs") + @classmethod + def _check_num_epochs(cls, v: int) -> int: + # 0 is a sentinel meaning "use max_steps instead"; the frontend's + # steps-vs-epochs toggle sends it. + if v is None: + return 1 + if v < 0 or v > _MAX_EPOCHS: + raise ValueError(f"num_epochs must be in [0, {_MAX_EPOCHS}] (got {v!r})") + return v + + @field_validator("max_steps") + @classmethod + def _check_max_steps(cls, v: Optional[int]) -> Optional[int]: + # 0 is the frontend's sentinel for "use num_epochs instead". + if v is None: + return v + if not isinstance(v, int) or v < 0 or v > _MAX_STEPS: + raise ValueError( + f"max_steps must be a non-negative int <= {_MAX_STEPS} (got {v!r})" + ) + return v + + @field_validator("max_seq_length") + @classmethod + def _check_max_seq_length(cls, v: int) -> int: + if v is None or v < 1 or v > _MAX_SEQ_LENGTH: + raise ValueError( + f"max_seq_length must be in [1, {_MAX_SEQ_LENGTH}] (got {v!r})" + ) + return v + + @field_validator("warmup_steps") + @classmethod + def _check_warmup_steps(cls, v: Optional[int]) -> Optional[int]: + if v is None: + return v + if not isinstance(v, int) or v < 0 or v > _MAX_STEPS: + raise ValueError( + f"warmup_steps must be a non-negative int <= {_MAX_STEPS} " + f"(got {v!r})" + ) + return v + + @field_validator("warmup_ratio") + @classmethod + def _check_warmup_ratio(cls, v): + if v is None: + return v + try: + r = float(v) + except (TypeError, ValueError): + raise ValueError(f"warmup_ratio must be a number (got {v!r})") + if not (0.0 <= r <= 1.0): + raise ValueError(f"warmup_ratio must be in [0.0, 1.0] (got {r!r})") + return r + + @field_validator("save_steps") + @classmethod + def _check_save_steps(cls, v: int) -> int: + if v is None: + return 100 + if v < 0 or v > _MAX_STEPS: + raise ValueError(f"save_steps must be in [0, {_MAX_STEPS}] (got {v!r})") + return v + + @field_validator("weight_decay") + @classmethod + def _check_weight_decay(cls, v: float) -> float: + if v is None: + return 0.0 + try: + wd = float(v) + except (TypeError, ValueError): + raise ValueError(f"weight_decay must be a number (got {v!r})") + if wd < 0 or wd > 10.0: + raise ValueError( + f"weight_decay must be in [0, 10] (got {wd!r}); typical 0..0.1" + ) + return wd + + @field_validator("lora_r") + @classmethod + def _check_lora_r(cls, v: int) -> int: + if v is None: + return 16 + if v < 1 or v > _MAX_LORA_R: + raise ValueError(f"lora_r must be in [1, {_MAX_LORA_R}] (got {v!r})") + return v + + @field_validator("lora_alpha") + @classmethod + def _check_lora_alpha(cls, v: int) -> int: + if v is None: + return 16 + if v < 1 or v > _MAX_LORA_ALPHA: + raise ValueError( + f"lora_alpha must be in [1, {_MAX_LORA_ALPHA}] (got {v!r})" + ) + return v + + @field_validator("lora_dropout") + @classmethod + def _check_lora_dropout(cls, v: float) -> float: + if v is None: + return 0.0 + try: + d = float(v) + except (TypeError, ValueError): + raise ValueError(f"lora_dropout must be a number (got {v!r})") + if not (0.0 <= d < 1.0): + raise ValueError(f"lora_dropout must be in [0.0, 1.0) (got {d!r})") + return d + custom_format_mapping: Optional[Dict[str, Any]] = Field( None, description = ( @@ -147,6 +324,16 @@ class TrainingStartRequest(BaseModel): description = "Physical GPU indices to use, for example [0, 1]. Omit or pass [] to use automatic selection. Explicit gpu_ids are unsupported when the parent CUDA_VISIBLE_DEVICES uses UUID/MIG entries.", ) + @model_validator(mode = "after") + def _check_steps_or_epochs(self) -> "TrainingStartRequest": + # num_epochs and max_steps each accept 0 as a "use the other one" + # sentinel. If both resolve to 0 there's nothing to train against. + if (self.max_steps is None or self.max_steps == 0) and self.num_epochs == 0: + raise ValueError( + "Either num_epochs or max_steps must be > 0; both cannot be 0." + ) + return self + class TrainingJobResponse(BaseModel): """Immediate response when training is initiated""" diff --git a/studio/backend/requirements/studio.txt b/studio/backend/requirements/studio.txt index 186ba82fe0..1bf751c368 100644 --- a/studio/backend/requirements/studio.txt +++ b/studio/backend/requirements/studio.txt @@ -3,6 +3,7 @@ typer fastapi uvicorn pydantic +packaging matplotlib pandas nest_asyncio diff --git a/studio/backend/routes/auth.py b/studio/backend/routes/auth.py index 3deeb6793b..30221c2c93 100644 --- a/studio/backend/routes/auth.py +++ b/studio/backend/routes/auth.py @@ -5,8 +5,11 @@ Authentication API routes """ -from fastapi import APIRouter, Depends, HTTPException, status +from fastapi import APIRouter, Depends, HTTPException, Request, Response, status +import threading +import time +from collections import deque from datetime import datetime, timedelta, timezone from models.auth import ( @@ -33,14 +36,52 @@ from auth.authentication import ( router = APIRouter() +# In-memory per-IP login rate limiter; multi-process deployment needs a shared store. +_LOGIN_BUCKETS: dict[str, deque] = {} +_LOGIN_BUCKETS_LOCK = threading.Lock() +_LOGIN_WINDOW_SECONDS = 60.0 +_LOGIN_MAX_FAILS = 5 +_LOGIN_LOCKOUT_SECONDS = 60 + + +def _client_key(request: Request | None) -> str: + if request is None or request.client is None: + return "_unknown" + return request.client.host or "_unknown" + + +def _record_login_failure(ip: str) -> int: + now = time.monotonic() + with _LOGIN_BUCKETS_LOCK: + bucket = _LOGIN_BUCKETS.setdefault(ip, deque()) + while bucket and now - bucket[0] > _LOGIN_WINDOW_SECONDS: + bucket.popleft() + bucket.append(now) + return len(bucket) + + +def _login_blocked(ip: str) -> int: + """Return seconds until the next attempt is allowed, or 0.""" + now = time.monotonic() + with _LOGIN_BUCKETS_LOCK: + bucket = _LOGIN_BUCKETS.get(ip) + if not bucket: + return 0 + while bucket and now - bucket[0] > _LOGIN_WINDOW_SECONDS: + bucket.popleft() + if len(bucket) >= _LOGIN_MAX_FAILS: + return max(1, int(_LOGIN_WINDOW_SECONDS - (now - bucket[0]))) + return 0 + + +def _clear_login_bucket(ip: str) -> None: + with _LOGIN_BUCKETS_LOCK: + _LOGIN_BUCKETS.pop(ip, None) + + @router.get("/status", response_model = AuthStatusResponse) async def auth_status() -> AuthStatusResponse: - """ - Check whether auth has already been initialized. - - - initialized = False -> frontend should wait for the seeded admin bootstrap. - - initialized = True -> frontend should show login or force the first password change. - """ + """Auth initialization state; ``default_username`` is exposed for first-boot UI prefill only.""" return AuthStatusResponse( initialized = storage.is_initialized(), default_username = storage.DEFAULT_ADMIN_USERNAME, @@ -53,12 +94,23 @@ async def auth_status() -> AuthStatusResponse: @router.post("/login", response_model = Token) -async def login(payload: AuthLoginRequest) -> Token: - """ - Login with username/password and receive access + refresh tokens. - """ +async def login(payload: AuthLoginRequest, request: Request) -> Token: + """Login with username/password. Rate-limited per source IP.""" + ip = _client_key(request) + blocked_for = _login_blocked(ip) + if blocked_for > 0: + raise HTTPException( + status_code = status.HTTP_429_TOO_MANY_REQUESTS, + detail = ( + f"Too many failed login attempts from {ip}. " + f"Try again in {blocked_for} seconds." + ), + headers = {"Retry-After": str(blocked_for)}, + ) + record = storage.get_user_and_secret(payload.username) if record is None: + _record_login_failure(ip) raise HTTPException( status_code = status.HTTP_401_UNAUTHORIZED, detail = "Incorrect password. Run 'unsloth studio reset-password' in your terminal to reset it.", @@ -66,11 +118,13 @@ async def login(payload: AuthLoginRequest) -> Token: salt, pwd_hash, _jwt_secret, must_change_password = record if not hashing.verify_password(payload.password, salt, pwd_hash): + _record_login_failure(ip) raise HTTPException( status_code = status.HTTP_401_UNAUTHORIZED, detail = "Incorrect password. Run 'unsloth studio reset-password' in your terminal to reset it.", ) + _clear_login_bucket(ip) access_token = create_access_token(subject = payload.username) refresh_token = create_refresh_token(subject = payload.username) return Token( @@ -81,6 +135,23 @@ async def login(payload: AuthLoginRequest) -> Token: ) +@router.post("/logout", status_code = status.HTTP_204_NO_CONTENT) +async def logout( + request: Request, + current_subject: str = Depends(get_current_subject_allow_password_change), +) -> Response: + """Revoke refresh tokens for the subject; the access token is stateless and expires on its own.""" + try: + storage.revoke_user_refresh_tokens(current_subject) + except Exception: + pass + try: + request.app.state.bootstrap_password = None + except AttributeError: + pass + return Response(status_code = status.HTTP_204_NO_CONTENT) + + @router.post("/desktop-login", response_model = Token) async def desktop_login(payload: DesktopLoginRequest) -> Token: """Exchange a local desktop secret for normal admin-subject tokens.""" @@ -101,21 +172,20 @@ async def desktop_login(payload: DesktopLoginRequest) -> Token: @router.post("/refresh", response_model = Token) async def refresh(payload: RefreshTokenRequest) -> Token: - """ - Exchange a valid refresh token for a new access token. - - The refresh token itself is reusable until it expires (7 days). - """ - new_access_token, username, is_desktop = refresh_access_token(payload.refresh_token) - if new_access_token is None or username is None: + """Exchange a refresh token for a new access+refresh pair (single-use).""" + consumed = storage.consume_refresh_token(payload.refresh_token) + if consumed is None: raise HTTPException( status_code = status.HTTP_401_UNAUTHORIZED, detail = "Invalid or expired refresh token", ) + username, is_desktop = consumed + new_access_token = create_access_token(subject = username, desktop = is_desktop) + new_refresh_token = create_refresh_token(subject = username, desktop = is_desktop) return Token( access_token = new_access_token, - refresh_token = payload.refresh_token, + refresh_token = new_refresh_token, token_type = "bearer", must_change_password = False if is_desktop @@ -126,6 +196,7 @@ async def refresh(payload: RefreshTokenRequest) -> Token: @router.post("/change-password", response_model = Token) async def change_password( payload: ChangePasswordRequest, + request: Request, current_subject: str = Depends(get_current_subject_allow_password_change), ) -> Token: """Allow the authenticated user to replace the default password.""" @@ -150,6 +221,10 @@ async def change_password( storage.update_password(current_subject, payload.new_password) storage.revoke_user_refresh_tokens(current_subject) + try: + request.app.state.bootstrap_password = None + except AttributeError: + pass access_token = create_access_token(subject = current_subject) refresh_token = create_refresh_token(subject = current_subject) return Token( diff --git a/studio/backend/routes/export.py b/studio/backend/routes/export.py index 798859fc87..7dbc52dbed 100644 --- a/studio/backend/routes/export.py +++ b/studio/backend/routes/export.py @@ -7,6 +7,7 @@ Export API routes: checkpoint discovery and model export operations. import asyncio import json +import os import sys import time from pathlib import Path @@ -184,14 +185,18 @@ async def get_export_status( def _export_details(output_path: Optional[str]) -> Optional[Dict[str, Any]]: - """Wrap the resolved on-disk export path into the details dict the - frontend reads to populate the Export Complete screen. Returns None - when the export had no local component (Hub-only push) so the - Pydantic field stays absent rather than ``{"output_path": null}``. - """ + """Return the export path relative to exports_root so the install path is not leaked.""" if not output_path: return None - return {"output_path": output_path} + try: + from utils.paths.storage_roots import exports_root + + rel = os.path.relpath(output_path, exports_root()) + if rel.startswith(".."): + rel = os.path.basename(output_path) + return {"output_path": rel} + except Exception: + return {"output_path": os.path.basename(output_path)} @router.post("/export/merged", response_model = ExportOperationResponse) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 6b559b9c45..7102e12bf8 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1743,7 +1743,7 @@ async def openai_chat_completions( try: import base64 as _b64 from io import BytesIO as _BytesIO - from PIL import Image as _Image + from PIL import Image as _Image, UnidentifiedImageError as _UIE raw = _b64.b64decode(image_b64) # Normalize to RGB so PNG encoding succeeds regardless of @@ -1754,9 +1754,15 @@ async def openai_chat_completions( buf = _BytesIO() img.save(buf, format = "PNG") image_b64 = _b64.b64encode(buf.getvalue()).decode("ascii") - except Exception as e: + except _UIE: raise HTTPException( - status_code = 400, detail = f"Failed to process image: {e}" + status_code = 400, + detail = "Unsupported or corrupt image format.", + ) + except Exception: + raise HTTPException( + status_code = 400, + detail = "Failed to process image.", ) # Build message list with system prompt prepended @@ -3426,10 +3432,10 @@ def _normalize_anthropic_openai_images( buf = io.BytesIO() img.save(buf, format = "PNG") png_b64 = base64.b64encode(buf.getvalue()).decode("ascii") - except Exception as e: + except Exception: raise HTTPException( status_code = 400, - detail = f"Failed to process image: {e}", + detail = "Failed to process image.", ) part["image_url"] = {"url": f"data:image/png;base64,{png_b64}"} @@ -3465,6 +3471,7 @@ async def anthropic_messages( [m.model_dump() for m in payload.messages], payload.system, ) + openai_messages = _drop_empty_assistant_sentinels(openai_messages) # Enforce vision guard + re-encode embedded images to PNG so the # Anthropic endpoint matches the behavior of /v1/chat/completions. @@ -4190,6 +4197,19 @@ async def _anthropic_passthrough_non_streaming( # ===================================================================== +def _drop_empty_assistant_sentinels(messages: list[dict]) -> list[dict]: + """Drop bare ``{"role":"assistant"}`` Stop-button sentinels; passthrough backends reject them.""" + out: list[dict] = [] + for m in messages: + if m.get("role") == "assistant": + has_content = bool(m.get("content")) + has_tool_calls = bool(m.get("tool_calls")) + if not has_content and not has_tool_calls: + continue + out.append(m) + return out + + def _openai_messages_for_passthrough(payload) -> list[dict]: """Build OpenAI-format message dicts for the /v1/chat/completions passthrough path. @@ -4206,7 +4226,9 @@ def _openai_messages_for_passthrough(payload) -> list[dict]: ``image_url`` content part so vision + function-calling requests work transparently. """ - messages = [m.model_dump(exclude_none = True) for m in payload.messages] + messages = _drop_empty_assistant_sentinels( + [m.model_dump(exclude_none = True) for m in payload.messages] + ) if not payload.image_base64: return messages @@ -4221,10 +4243,10 @@ def _openai_messages_for_passthrough(payload) -> list[dict]: buf = _BytesIO() img.save(buf, format = "PNG") png_b64 = _b64.b64encode(buf.getvalue()).decode("ascii") - except Exception as e: + except Exception: raise HTTPException( status_code = 400, - detail = f"Failed to process image: {e}", + detail = "Failed to process image.", ) data_url = f"data:image/png;base64,{png_b64}" diff --git a/studio/backend/run.py b/studio/backend/run.py index 1dd1230a17..0787e04c47 100644 --- a/studio/backend/run.py +++ b/studio/backend/run.py @@ -307,7 +307,6 @@ def run_server( import asyncio from threading import Thread, Event - import time import uvicorn from main import app, setup_frontend @@ -336,10 +335,6 @@ def run_server( print("=" * 50) print("") - # Output port for Tauri to parse when in api-only mode - if api_only: - print(f"TAURI_PORT={port}", flush = True) - # Setup frontend if path provided (skip in api-only mode) if frontend_path and not api_only: if setup_frontend(app, frontend_path): @@ -349,11 +344,26 @@ def run_server( if not silent: print(f"[WARNING] Frontend not found at {frontend_path}") - # Create the uvicorn server and expose it for signal handlers + ready_event = Event() + startup_failed = Event() + startup_errors = [] + + class _ReadyServer(uvicorn.Server): + async def startup(self, *args, **kwargs): + await super().startup(*args, **kwargs) + if getattr(self, "started", False) and not self.should_exit: + ready_event.set() + + # server_header=False suppresses uvicorn's "Server: uvicorn"; SecurityHeadersMiddleware sets its own. config = uvicorn.Config( - app, host = host, port = port, log_level = "info", access_log = False + app, + host = host, + port = port, + log_level = "info", + access_log = False, + server_header = False, ) - _server = uvicorn.Server(config) + _server = _ReadyServer(config) _shutdown_event = Event() # Expose the actual bound port so request-handling code can build @@ -365,21 +375,8 @@ def run_server( app.state.server_port = port if port and port > 0 else None app.state.llama_parallel_slots = llama_parallel_slots - # Run server in a daemon thread - def _run(): - asyncio.run(_server.serve()) - - thread = Thread(target = _run, daemon = True) - thread.start() - time.sleep(3) - - _write_pid_file() - import atexit - - atexit.register(_remove_pid_file) - - # Expose a shutdown callable via app.state so the /api/shutdown endpoint - # can trigger graceful shutdown without circular imports. + # Expose a shutdown callable via app.state before the server can accept + # requests so /api/shutdown is available as soon as readiness is published. def _trigger_shutdown(): _graceful_shutdown(_server) if _shutdown_event is not None: @@ -387,6 +384,47 @@ def run_server( app.state.trigger_shutdown = _trigger_shutdown + # Run server in a daemon thread + def _run(): + try: + asyncio.run(_server.serve()) + except BaseException as exc: + startup_errors.append(exc) + startup_failed.set() + finally: + if not ready_event.is_set(): + startup_failed.set() + + thread = Thread(target = _run, daemon = True) + thread.start() + + # Wait until uvicorn has completed lifespan startup and bound sockets, or + # until the server exits/fails before startup. This intentionally has no + # correctness deadline: a slow but live startup should remain in progress. + try: + while not ready_event.is_set(): + if startup_failed.is_set() or not thread.is_alive(): + if startup_errors: + raise RuntimeError( + "Uvicorn server failed before startup completed" + ) from startup_errors[0] + raise RuntimeError("Uvicorn server exited before startup completed") + ready_event.wait(timeout = 0.1) + except KeyboardInterrupt: + _graceful_shutdown(_server) + _shutdown_event.set() + raise + + _write_pid_file() + import atexit + + atexit.register(_remove_pid_file) + + # Output port for Tauri to parse when in api-only mode. Emit only after + # uvicorn sockets are bound and FastAPI lifespan/startup has completed. + if api_only: + print(f"TAURI_PORT={port}", flush = True) + if not silent: display_host = _resolve_external_ip() if host == "0.0.0.0" else host print_studio_access_banner( diff --git a/studio/backend/tests/test_desktop_auth.py b/studio/backend/tests/test_desktop_auth.py index a5508c1c8b..a7201ac433 100644 --- a/studio/backend/tests/test_desktop_auth.py +++ b/studio/backend/tests/test_desktop_auth.py @@ -227,6 +227,60 @@ def test_desktop_refresh_preserves_desktop_marker(): assert payload["desktop"] is True +def test_consume_refresh_token_second_call_returns_none(): + """Single-use rotation rejects the same token on a second consume.""" + seed_user() + from datetime import datetime, timedelta, timezone + + raw = secrets.token_urlsafe(48) + expires = (datetime.now(timezone.utc) + timedelta(days = 30)).isoformat() + storage.save_refresh_token(raw, storage.DEFAULT_ADMIN_USERNAME, expires) + + first = storage.consume_refresh_token(raw) + assert first == (storage.DEFAULT_ADMIN_USERNAME, False) + second = storage.consume_refresh_token(raw) + assert second is None + + +def test_consume_refresh_token_concurrent_only_one_succeeds(tmp_path, monkeypatch): + """64-thread pile-up against one token; DELETE RETURNING permits one winner.""" + seed_user() + from concurrent.futures import ThreadPoolExecutor + from datetime import datetime, timedelta, timezone + + raw = secrets.token_urlsafe(48) + expires = (datetime.now(timezone.utc) + timedelta(days = 30)).isoformat() + storage.save_refresh_token(raw, storage.DEFAULT_ADMIN_USERNAME, expires) + + workers = 64 + + def attempt(_idx: int): + try: + return storage.consume_refresh_token(raw) + except sqlite3.OperationalError: + # "database is locked" under heavy contention; treat as losing the race. + return None + + with ThreadPoolExecutor(max_workers = workers) as pool: + results = list(pool.map(attempt, range(workers))) + + successes = [r for r in results if r is not None] + assert ( + len(successes) == 1 + ), f"expected exactly one consumer to win, got {len(successes)}" + assert successes[0] == (storage.DEFAULT_ADMIN_USERNAME, False) + + +def test_consume_refresh_token_expired_returns_none(): + seed_user() + from datetime import datetime, timedelta, timezone + + raw = secrets.token_urlsafe(48) + expires = (datetime.now(timezone.utc) - timedelta(hours = 1)).isoformat() + storage.save_refresh_token(raw, storage.DEFAULT_ADMIN_USERNAME, expires) + assert storage.consume_refresh_token(raw) is None + + def test_desktop_session_uses_real_admin_identity_for_api_keys(): seed_user(must_change_password = True) raw = storage.create_desktop_secret() @@ -392,7 +446,21 @@ def test_health_response_reports_desktop_capability_fields(monkeypatch): monkeypatch.setattr(backend_main._hw_module, "CHAT_ONLY", False) - body = asyncio.run(backend_main.health_check()) + seed_user() + from auth.authentication import create_access_token + + token = create_access_token(storage.DEFAULT_ADMIN_USERNAME) + + app = FastAPI() + app.add_api_route("/api/health", backend_main.health_check, methods = ["GET"]) + client = TestClient(app) + + response = client.get( + "/api/health", + headers = {"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 200 + body = response.json() assert body["desktop_protocol_version"] == 1 assert body["supports_desktop_auth"] is True diff --git a/studio/backend/tests/test_llama_cpp_context_fit.py b/studio/backend/tests/test_llama_cpp_context_fit.py index caa6397901..1ea76edd15 100644 --- a/studio/backend/tests/test_llama_cpp_context_fit.py +++ b/studio/backend/tests/test_llama_cpp_context_fit.py @@ -192,6 +192,7 @@ def _drive( else: ranked = sorted(gpus, key = lambda g: g[1], reverse = True) matched = False + pin_fraction = LlamaCppBackend._GPU_PIN_VRAM_FRACTION for n_gpus in range(1, len(ranked) + 1): subset = ranked[:n_gpus] pool_mib = sum(free for _, free in subset) @@ -203,7 +204,7 @@ def _drive( ) kv = inst._estimate_kv_cache_bytes(capped, cache_type_kv) total_mib = (model_size + kv) / (1024 * 1024) - if total_mib <= pool_mib * 0.90: + if total_mib <= pool_mib * pin_fraction: effective_ctx = capped gpu_indices = sorted(idx for idx, _ in subset) use_fit = False @@ -211,6 +212,17 @@ def _drive( break if not matched: effective_ctx = min(FALLBACK_CTX, effective_ctx) + # Mirror llama_cpp.py: re-check fit at FALLBACK_CTX. + if effective_ctx > 0: + for n_gpus in range(1, len(ranked) + 1): + subset = ranked[:n_gpus] + pool_mib = sum(free for _, free in subset) + kv = inst._estimate_kv_cache_bytes(effective_ctx, cache_type_kv) + total_mib = (model_size + kv) / (1024 * 1024) + if total_mib <= pool_mib * pin_fraction: + gpu_indices = sorted(idx for idx, _ in subset) + use_fit = False + break elif gpus: gpu_indices, use_fit = inst._select_gpus(model_size, gpus) if use_fit and not explicit_ctx: @@ -378,6 +390,52 @@ class TestFittableAutoPickRegressions: assert plan["gpu_indices"] == [0] +# --------------------------------------------------------------------------- +# #5106 regression: 91-95% utilization must still pin GPU. +# --------------------------------------------------------------------------- + + +class TestTightFitPinsToGPU: + """Models that fit at 91-95% of free VRAM must use the GPU.""" + + def test_rtx_4090_qwen_24gb_class(self): + # noahterbest's #5106 log: 20.8 GB model on 22805 MiB free + # GPU, ctx=4096 -> ~94% utilization, ~1.4 GiB headroom. + plan = _drive( + n_ctx = 0, + model_gib = 20.8, + gpus = [(0, 22_805)], + native_ctx = 131072, + kv_per_token_bytes = 25_000, + ) + assert plan["use_fit"] is False + assert plan["gpu_indices"] == [0] + + def test_explicit_ctx_at_94_pct_pins_to_gpu(self): + # Explicit-ctx branch must agree with auto-ctx on headroom. + plan = _drive( + n_ctx = 4096, + model_gib = 20.8, + gpus = [(0, 22_805)], + native_ctx = 131072, + kv_per_token_bytes = 25_000, + ) + assert plan["use_fit"] is False + assert plan["gpu_indices"] == [0] + + def test_genuine_overflow_still_uses_fit(self): + # Beyond 95% must still defer to --fit on. + plan = _drive( + n_ctx = 4096, + model_gib = 23, + gpus = [(0, 22_000)], + native_ctx = 131072, + kv_per_token_bytes = 25_000, + ) + assert plan["use_fit"] is True + assert plan["gpu_indices"] is None + + # --------------------------------------------------------------------------- # Platform-agnostic input shape # --------------------------------------------------------------------------- @@ -391,3 +449,81 @@ def test_identical_decision_across_platforms(platform_tag): plan_a = _drive(n_ctx = 0, model_gib = 8, gpus = [(0, 24_000)]) plan_b = _drive(n_ctx = 0, model_gib = 8, gpus = [(0, 24_000)]) assert plan_a == plan_b, platform_tag + + +# --------------------------------------------------------------------------- +# _classify_gpu_offload: detect silent CPU fallback (#5106). +# --------------------------------------------------------------------------- + + +class TestClassifyGpuOffload: + def _backend(self, stdout_lines): + inst = LlamaCppBackend.__new__(LlamaCppBackend) + inst._stdout_lines = list(stdout_lines) + return inst + + def test_cuda_buffer_present_returns_true(self): + inst = self._backend( + [ + "load_tensors: offloaded 33/33 layers to GPU", + "load_tensors: CUDA0 model buffer size = 21000.0 MiB", + "load_tensors: CPU_Mapped model buffer size = 0.6 MiB", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is True + + def test_cpu_only_buffer_returns_false(self): + # llama-server printed buffer lines but only CPU buffers -- + # this is the silent CPU fallback symptom we want to catch. + inst = self._backend( + [ + "load_tensors: CPU_Mapped model buffer size = 21000.0 MiB", + "load_tensors: CPU model buffer size = 0.6 MiB", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is False + + def test_no_buffer_lines_returns_none(self): + # If we can't see buffer-allocation lines at all, don't guess. + inst = self._backend( + [ + "INFO [main] starting server", + "load_tensors: file format = GGUF V3", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is None + + def test_no_gpus_detected_returns_none(self): + # CPU-only systems are valid; suppress the warning entirely. + inst = self._backend( + [ + "load_tensors: CPU_Mapped model buffer size = 21000.0 MiB", + ] + ) + assert inst._classify_gpu_offload(False, []) is None + + def test_user_did_not_intend_gpu_returns_none(self): + # Studio called start_llama_server without expecting GPU use; + # don't warn. + inst = self._backend( + [ + "load_tensors: CPU_Mapped model buffer size = 21000.0 MiB", + ] + ) + assert inst._classify_gpu_offload(False, [(0, 22805)]) is None + + def test_rocm_buffer_marker_returns_true(self): + inst = self._backend( + [ + "load_tensors: ROCm0 model buffer size = 21000.0 MiB", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is True + + def test_metal_buffer_marker_returns_true(self): + inst = self._backend( + [ + "load_tensors: Metal model buffer size = 8000.0 MiB", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is True diff --git a/studio/backend/tests/test_middleware.py b/studio/backend/tests/test_middleware.py new file mode 100644 index 0000000000..bdf8e6d5a5 --- /dev/null +++ b/studio/backend/tests/test_middleware.py @@ -0,0 +1,269 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Tests for MaxBodyMiddleware, SecurityHeadersMiddleware, and the /api/health auth gate.""" + +import asyncio +import importlib.util +import json +import os +import sys +from pathlib import Path + +import pytest +from fastapi import FastAPI, HTTPException, Request +from fastapi.responses import Response +from fastapi.testclient import TestClient + + +_BACKEND_ROOT = Path(__file__).resolve().parents[1] +if str(_BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(_BACKEND_ROOT)) + + +@pytest.fixture(scope = "module") +def main_module(): + import main as _main # noqa: F401 + + return _main + + +# ===================================================================== +# MaxBodyMiddleware +# ===================================================================== + + +def _make_protected_app(max_bytes: int, main_module): + app = FastAPI() + app.add_middleware( + main_module.MaxBodyMiddleware, + max_bytes = max_bytes, + protected_prefixes = ("/v1/chat/completions", "/api/train"), + ) + + @app.post("/v1/chat/completions") + async def chat(payload: dict): + return {"ok": True, "n": len(payload.get("text", ""))} + + @app.post("/api/other") + async def other(payload: dict): + return {"ok": True, "unprotected": True} + + @app.get("/api/train/status") + async def status_get(): + return {"ok": True, "get": True} + + return app + + +class TestMaxBodyMiddleware: + def test_small_protected_body_passes(self, main_module): + app = _make_protected_app(1024, main_module) + c = TestClient(app) + r = c.post("/v1/chat/completions", json = {"text": "x" * 100}) + assert r.status_code == 200 + assert r.json()["n"] == 100 + + def test_large_declared_content_length_rejected(self, main_module): + app = _make_protected_app(1024, main_module) + c = TestClient(app) + r = c.post("/v1/chat/completions", json = {"text": "x" * 5000}) + assert r.status_code == 413 + assert "too large" in r.json()["detail"].lower() + + def test_unprotected_prefix_passes_large_body(self, main_module): + app = _make_protected_app(1024, main_module) + c = TestClient(app) + r = c.post("/api/other", json = {"text": "x" * 5000}) + assert r.status_code == 200 + assert r.json()["unprotected"] is True + + def test_chunked_upload_over_cap_rejected(self, main_module): + # Regression: declared-Content-Length-only check could be bypassed + # by chunked transfer-encoding. + app = _make_protected_app(1024, main_module) + c = TestClient(app) + + def gen(): + yield b'{"text":"' + yield b"x" * 800 + yield b'"}' + yield b"\n" + b"y" * 500 + + r = c.post( + "/v1/chat/completions", + content = gen(), + headers = {"content-type": "application/json"}, + ) + assert r.status_code == 413 + assert "too large" in r.json()["detail"].lower() + + def test_chunked_upload_under_cap_passes(self, main_module): + app = _make_protected_app(1024, main_module) + c = TestClient(app) + + def gen(): + yield b'{"text":"' + yield b"x" * 50 + yield b'"}' + + r = c.post( + "/v1/chat/completions", + content = gen(), + headers = {"content-type": "application/json"}, + ) + assert r.status_code == 200 + assert r.json()["n"] == 50 + + def test_get_not_subject_to_cap(self, main_module): + app = _make_protected_app(1024, main_module) + c = TestClient(app) + r = c.get("/api/train/status") + assert r.status_code == 200 + + +# ===================================================================== +# SecurityHeadersMiddleware / CSP +# ===================================================================== + + +def _make_csp_app(main_module, attach_nonce: str | None = None): + app = FastAPI() + app.add_middleware(main_module.SecurityHeadersMiddleware) + + @app.get("/plain") + async def plain(): + return {"ok": True} + + @app.get("/with-nonce") + async def with_nonce(): + headers = {} + if attach_nonce: + headers[main_module._CSP_SCRIPT_NONCE_HEADER] = attach_nonce + return Response( + content = b"", + media_type = "text/html", + headers = headers, + ) + + return app + + +class TestSecurityHeadersMiddleware: + def test_csp_has_no_unsafe_inline_for_script_src(self, main_module): + app = _make_csp_app(main_module) + c = TestClient(app) + r = c.get("/plain") + assert r.status_code == 200 + csp = r.headers["content-security-policy"] + # Parse per-directive so style-src unsafe-inline does not false-match. + directives = { + chunk.strip().split(" ", 1)[0]: chunk.strip() + for chunk in csp.split(";") + if chunk.strip() + } + assert "script-src" in directives + assert "'unsafe-inline'" not in directives["script-src"] + # style-src keeps unsafe-inline for Vite-injected styles. + assert "'unsafe-inline'" in directives["style-src"] + + def test_default_security_headers_present(self, main_module): + app = _make_csp_app(main_module) + c = TestClient(app) + r = c.get("/plain") + assert r.headers["x-frame-options"] == "DENY" + assert r.headers["x-content-type-options"] == "nosniff" + assert r.headers["referrer-policy"] == "no-referrer" + assert "camera=()" in r.headers["permissions-policy"] + assert r.headers["server"] == "unsloth-studio" + + def test_internal_nonce_header_is_spliced_into_csp_and_stripped(self, main_module): + nonce = "test-nonce-abc" + app = _make_csp_app(main_module, attach_nonce = nonce) + c = TestClient(app) + r = c.get("/with-nonce") + csp = r.headers["content-security-policy"] + assert f"'nonce-{nonce}'" in csp + # Internal handoff header must not leak to clients. + assert main_module._CSP_SCRIPT_NONCE_HEADER not in { + k.lower() for k in r.headers.keys() + } + + def test_build_csp_helper_shape(self, main_module): + plain = main_module._build_csp() + assert "script-src 'self';" in plain + assert "'unsafe-inline'" not in plain.split("script-src", 1)[1].split(";", 1)[0] + nonced = main_module._build_csp("XYZ") + assert "script-src 'self' 'nonce-XYZ';" in nonced + + +# ===================================================================== +# /api/health auth gate +# ===================================================================== + + +@pytest.fixture +def health_app(tmp_path, monkeypatch): + """Mount /api/health on a fresh app against an isolated auth db.""" + from auth import storage + + monkeypatch.setattr(storage, "DB_PATH", tmp_path / "auth.db") + monkeypatch.setattr(storage, "_BOOTSTRAP_PW_PATH", tmp_path / ".bootstrap_password") + monkeypatch.setattr(storage, "_bootstrap_password", None) + + import main as _main + + app = FastAPI() + app.add_api_route("/api/health", _main.health_check, methods = ["GET"]) + + import secrets as _secrets + + storage.create_initial_user( + username = storage.DEFAULT_ADMIN_USERNAME, + password = "human-password-123", + jwt_secret = _secrets.token_urlsafe(64), + must_change_password = False, + ) + return app + + +class TestHealthAuthGate: + def test_no_auth_returns_minimal_payload(self, health_app): + c = TestClient(health_app) + r = c.get("/api/health") + assert r.status_code == 200 + body = r.json() + assert body["status"] == "healthy" + assert "timestamp" in body + for forbidden in ("version", "device_type", "studio_root_id"): + assert forbidden not in body + + def test_invalid_bearer_returns_minimal_payload(self, health_app): + # Regression: calling the async dep without await made any Bearer header pass. + c = TestClient(health_app) + r = c.get( + "/api/health", + headers = {"Authorization": "Bearer not-a-real-token"}, + ) + assert r.status_code == 200 + body = r.json() + assert body["status"] == "healthy" + for forbidden in ("version", "device_type", "studio_root_id"): + assert forbidden not in body + + def test_valid_bearer_returns_full_payload(self, health_app): + from auth import storage + from auth.authentication import create_access_token + + token = create_access_token(storage.DEFAULT_ADMIN_USERNAME) + c = TestClient(health_app) + r = c.get( + "/api/health", + headers = {"Authorization": f"Bearer {token}"}, + ) + assert r.status_code == 200 + body = r.json() + assert body["status"] == "healthy" + assert "version" in body + assert "device_type" in body + assert "studio_root_id" in body diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py index cdb7f5d270..a379282b70 100644 --- a/studio/backend/tests/test_openai_tool_passthrough.py +++ b/studio/backend/tests/test_openai_tool_passthrough.py @@ -125,22 +125,21 @@ class TestChatMessageToolRoles: ) assert msg.content is None - def test_tool_role_missing_tool_call_id_rejected(self): - # Per OpenAI spec, role="tool" messages must carry tool_call_id so - # upstream backends can associate the result with its prior call. - # Pin the boundary-level rejection so a malformed tool-result - # message never reaches the passthrough path. - with pytest.raises(ValidationError) as exc_info: - ChatMessage(role = "tool", content = '{"temperature": 72}') - assert "tool_call_id" in str(exc_info.value) + def test_tool_role_missing_tool_call_id_synthesised(self): + # Frontend drops the id on second-round POST; validator synthesises one. + msg = ChatMessage(role = "tool", content = '{"temperature": 72}') + assert msg.tool_call_id is not None + assert msg.tool_call_id.startswith("call_") + assert len(msg.tool_call_id) >= len("call_") + 8 - def test_tool_role_empty_tool_call_id_rejected(self): - with pytest.raises(ValidationError): - ChatMessage( - role = "tool", - tool_call_id = "", - content = '{"temperature": 72}', - ) + def test_tool_role_empty_tool_call_id_synthesised(self): + msg = ChatMessage( + role = "tool", + tool_call_id = "", + content = '{"temperature": 72}', + ) + assert msg.tool_call_id is not None + assert msg.tool_call_id.startswith("call_") # ── Role-aware content requirements ──────────────────────────── @@ -162,10 +161,19 @@ class TestChatMessageToolRoles: ChatMessage(role = "tool", tool_call_id = "call_1", content = "") assert "content" in str(exc_info.value) - def test_assistant_without_content_or_tool_calls_rejected(self): - with pytest.raises(ValidationError) as exc_info: - ChatMessage(role = "assistant") - assert "content" in str(exc_info.value) or "tool_calls" in str(exc_info.value) + def test_assistant_without_content_or_tool_calls_tolerated(self): + # Stop-button leaves an empty assistant turn; tolerate so replay round-trips. + msg = ChatMessage(role = "assistant") + assert msg.content is None + assert msg.tool_calls is None + + def test_assistant_empty_string_content_normalised_to_none(self): + msg = ChatMessage(role = "assistant", content = "") + assert msg.content is None + + def test_assistant_empty_list_content_normalised_to_none(self): + msg = ChatMessage(role = "assistant", content = []) + assert msg.content is None # ── Role-constrained tool-call metadata ──────────────────────── @@ -472,3 +480,91 @@ class TestFriendlyErrorHttpx: assert ( _friendly_error(RuntimeError("unrelated")) == "An internal error occurred" ) + + +from routes.inference import ( # noqa: E402 + _drop_empty_assistant_sentinels, + _openai_messages_for_passthrough, +) + + +class TestDropEmptyAssistantSentinels: + def test_drops_empty_assistant_between_real_turns(self): + msgs = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": ""}, + {"role": "user", "content": "again"}, + ] + out = _drop_empty_assistant_sentinels(msgs) + assert out == [ + {"role": "user", "content": "hi"}, + {"role": "user", "content": "again"}, + ] + + def test_drops_assistant_with_no_content_key(self): + # exclude_none=True strips the content key entirely; filter must catch this. + msgs = [ + {"role": "user", "content": "hi"}, + {"role": "assistant"}, + {"role": "user", "content": "ok"}, + ] + out = _drop_empty_assistant_sentinels(msgs) + assert out == [ + {"role": "user", "content": "hi"}, + {"role": "user", "content": "ok"}, + ] + + def test_preserves_assistant_with_text(self): + msgs = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello back"}, + ] + out = _drop_empty_assistant_sentinels(msgs) + assert out == msgs + + def test_preserves_assistant_with_tool_calls_only(self): + msgs = [ + {"role": "user", "content": "weather?"}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": "{}"}, + }, + ], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": '{"t": 72}', + }, + ] + out = _drop_empty_assistant_sentinels(msgs) + assert out == msgs + + def test_preserves_user_and_system_with_empty_content(self): + # Filter scoped to role="assistant" only. + msgs = [ + {"role": "system", "content": ""}, + {"role": "user", "content": ""}, + ] + out = _drop_empty_assistant_sentinels(msgs) + assert out == msgs + + def test_openai_messages_for_passthrough_drops_sentinel(self): + """End-to-end: Stop-sentinel must not reach the wire.""" + req = ChatCompletionRequest( + model = "default", + messages = [ + ChatMessage(role = "user", content = "hi"), + ChatMessage(role = "assistant", content = ""), + ChatMessage(role = "user", content = "again"), + ], + ) + out = _openai_messages_for_passthrough(req) + roles = [m["role"] for m in out] + assert roles == ["user", "user"] + for m in out: + assert m.get("content"), m diff --git a/studio/backend/tests/test_sandbox_tools.py b/studio/backend/tests/test_sandbox_tools.py new file mode 100644 index 0000000000..fcc531c212 --- /dev/null +++ b/studio/backend/tests/test_sandbox_tools.py @@ -0,0 +1,241 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Tests for the sandboxed-Python AST policy in core/inference/tools.py.""" + +import os +import sys +from pathlib import Path + +import pytest + +_BACKEND_ROOT = Path(__file__).resolve().parents[1] +if str(_BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(_BACKEND_ROOT)) + +from core.inference.tools import _check_code_safety + + +def _ok(code: str): + assert _check_code_safety(code) is None, code + + +def _blocked(code: str, *, expect_phrase: str): + msg = _check_code_safety(code) + assert msg is not None, code + assert expect_phrase in msg, (expect_phrase, msg) + + +class TestMetadataHostDenylist: + def test_aws_imds_literal_blocked(self): + _blocked( + 'import requests; requests.get("http://169.254.169.254/latest/meta-data/")', + expect_phrase = "Blocked: cloud-metadata host", + ) + + def test_gcp_metadata_dns_blocked(self): + _blocked( + 'import requests; requests.get("http://metadata.google.internal/")', + expect_phrase = "Blocked: cloud-metadata host", + ) + + def test_alibaba_ecs_literal_blocked(self): + _blocked( + 'import socket; s=socket.socket(); s.connect(("100.100.100.200", 80))', + expect_phrase = "Blocked: cloud-metadata host", + ) + + def test_ipv6_imds_literal_blocked(self): + _blocked( + 'import urllib.request; urllib.request.urlopen("http://[fd00:ec2::254]/")', + expect_phrase = "Blocked: cloud-metadata host", + ) + + def test_metadata_link_local_prefix_blocked(self): + _blocked( + 'import requests; requests.get("http://169.254.170.2/v3/")', + expect_phrase = "Blocked: cloud-metadata host", + ) + + +class TestTrustedHostAllowlist: + @pytest.mark.parametrize( + "url", + [ + "https://en.wikipedia.org/wiki/Python_(programming_language)", + "https://fr.wikipedia.org/wiki/Python_(langage)", + "https://www.google.com/search?q=foo", + "https://duckduckgo.com/?q=foo", + "https://huggingface.co/unsloth", + "https://cdn-lfs.huggingface.co/repos/abc/def/file.bin", + "https://raw.githubusercontent.com/foo/bar/main/README.md", + "https://api.github.com/repos/foo/bar", + "https://arxiv.org/abs/2401.12345", + "https://export.arxiv.org/abs/2401.12345", + "https://stackoverflow.com/questions/12345", + "https://math.stackexchange.com/questions/12345", + "https://developer.mozilla.org/en-US/docs/Web/JavaScript", + "https://docs.python.org/3/library/asyncio.html", + "https://pypi.org/project/requests/", + "https://files.pythonhosted.org/packages/foo/bar.whl", + "https://www.bbc.com/news", + "https://api.weather.gov/points/40,-90", + "https://numpy.org/doc/stable/", + "https://pytorch.org/docs/stable/index.html", + ], + ) + def test_trusted_host_passes(self, url): + _ok(f"import requests; requests.get({url!r})") + + def test_wikipedia_subdomain_passes(self): + _ok( + 'import urllib.request; urllib.request.urlopen("https://m.en.wikipedia.org/wiki/Foo")' + ) + + def test_hf_co_short_form_passes(self): + _ok('import requests; requests.get("https://hf.co/unsloth/Qwen3.5-4B-GGUF")') + + def test_github_io_pages_pass(self): + _ok('import requests; requests.get("https://unslothai.github.io/")') + + +class TestUntrustedHostBlock: + def test_example_com_blocked(self): + _blocked( + 'import requests; requests.get("https://example.com/")', + expect_phrase = "Blocked: host not in sandbox allowlist", + ) + + def test_random_blog_blocked(self): + _blocked( + 'import urllib.request; urllib.request.urlopen("https://random-blog-host.example/")', + expect_phrase = "Blocked: host not in sandbox allowlist", + ) + + def test_socket_connect_random_host_blocked(self): + _blocked( + 'import socket; s=socket.socket(); s.connect(("evil.example", 80))', + expect_phrase = "Blocked: host not in sandbox allowlist", + ) + + def test_dynamic_url_not_statically_blocked(self): + # Static AST cannot resolve runtime URLs; bash blocklist is the fallback. + _ok('import requests; url = "https://example.com/"; requests.get(url)') + + +class TestHostNormalization: + def test_trailing_dot_treated_same(self): + _ok('import requests; requests.get("https://wikipedia.org./")') + + def test_explicit_port_does_not_unblock_or_misblock(self): + _ok('import requests; requests.get("https://en.wikipedia.org:443/wiki/Foo")') + _blocked( + 'import requests; requests.get("https://example.com:8080/")', + expect_phrase = "Blocked: host not in sandbox allowlist", + ) + + def test_userinfo_at_does_not_smuggle_metadata_host(self): + _blocked( + 'import requests; requests.get("https://wikipedia.org@169.254.169.254/latest/")', + expect_phrase = "Blocked: cloud-metadata host", + ) + + def test_uppercase_host_normalised(self): + _ok('import requests; requests.get("https://EN.WIKIPEDIA.ORG/wiki/Foo")') + + +class TestUploadDenylist: + def test_requests_post_files_blocked(self): + _blocked( + ( + "import requests\n" + 'requests.post("https://huggingface.co/api/repos/upload", ' + 'files={"f": open("x.bin", "rb")})' + ), + expect_phrase = "Blocked: file upload disallowed in sandbox", + ) + + def test_requests_put_data_bytes_blocked(self): + _blocked( + ( + "import requests\n" + 'requests.put("https://huggingface.co/api/repos/upload", ' + 'data=b"\\x00\\x01\\x02")' + ), + expect_phrase = "Blocked: file upload disallowed in sandbox", + ) + + def test_requests_post_data_open_handle_blocked(self): + _blocked( + ( + "import requests\n" + 'requests.post("https://huggingface.co/api/repos/upload", ' + 'data=open("x.bin", "rb"))' + ), + expect_phrase = "Blocked: file upload disallowed in sandbox", + ) + + def test_httpx_post_files_blocked(self): + _blocked( + ( + "import httpx\n" + 'httpx.post("https://huggingface.co/api/repos/upload", ' + 'files={"f": open("x.bin", "rb")})' + ), + expect_phrase = "Blocked: file upload disallowed in sandbox", + ) + + def test_hf_api_upload_file_blocked(self): + _blocked( + ( + "from huggingface_hub import HfApi\n" + 'HfApi().upload_file(path_or_fileobj="x.bin", ' + 'path_in_repo="x.bin", repo_id="foo/bar")' + ), + expect_phrase = "Blocked: file upload disallowed in sandbox", + ) + + def test_hf_module_upload_folder_blocked(self): + _blocked( + ( + "import huggingface_hub\n" + 'huggingface_hub.upload_folder(folder_path="./", repo_id="foo/bar")' + ), + expect_phrase = "Blocked: file upload disallowed in sandbox", + ) + + def test_hf_create_commit_method_blocked(self): + _blocked( + ( + "import huggingface_hub\n" + "api = huggingface_hub.HfApi()\n" + 'api.create_commit(repo_id="foo/bar", operations=[])' + ), + expect_phrase = "Blocked: file upload disallowed in sandbox", + ) + + def test_plain_post_json_not_blocked(self): + _ok( + "import requests\n" + 'requests.post("https://api.weather.gov/lookup", json={"k": "v"})' + ) + + +class TestSandboxCpuRlimitDefault: + """Pin the default so a regression below 600s without opt-in is caught.""" + + def test_default_cpu_s_is_600(self): + src = (_BACKEND_ROOT / "core" / "inference" / "tools.py").read_text() + assert 'UNSLOTH_STUDIO_SANDBOX_CPU_S", "600"' in src + + def test_clone_newnet_removed(self): + src = (_BACKEND_ROOT / "core" / "inference" / "tools.py").read_text() + assert "_libc.unshare(0x40000000)" not in src + # Explanatory comment retained. + assert "CLONE_NEWNET" in src + + +class TestMaxBodyDefault: + def test_default_is_500_mb(self): + src = (_BACKEND_ROOT / "main.py").read_text() + assert 'UNSLOTH_STUDIO_MAX_BODY_MB", "500"' in src diff --git a/studio/backend/tests/test_studio_train_validation.py b/studio/backend/tests/test_studio_train_validation.py new file mode 100644 index 0000000000..7ffa9bb384 --- /dev/null +++ b/studio/backend/tests/test_studio_train_validation.py @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Pin TrainingStartRequest hyperparameter caps at the at-cap / over-cap boundary.""" + +import sys +from pathlib import Path + +import pytest +from pydantic import ValidationError + +_BACKEND_ROOT = Path(__file__).resolve().parents[1] +if str(_BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(_BACKEND_ROOT)) + +from models.training import ( + _MAX_BATCH_SIZE, + _MAX_LORA_ALPHA, + _MAX_LORA_R, + _MAX_SEQ_LENGTH, +) + + +def _check_field(field_name: str, value): + """Run the field validator without constructing a full TrainingStartRequest.""" + from models.training import TrainingStartRequest + + schema_field = TrainingStartRequest.model_fields[field_name] + return TrainingStartRequest.__pydantic_validator__.validate_assignment( + TrainingStartRequest.model_construct(), + field_name, + value, + ) + + +class TestSeqLengthCap: + def test_at_cap_accepts(self): + _check_field("max_seq_length", _MAX_SEQ_LENGTH) + assert _MAX_SEQ_LENGTH == 2_000_000 + + def test_over_cap_rejects(self): + with pytest.raises(ValidationError) as exc: + _check_field("max_seq_length", _MAX_SEQ_LENGTH + 1) + assert "max_seq_length" in str(exc.value) + + def test_below_min_rejects(self): + with pytest.raises(ValidationError): + _check_field("max_seq_length", 0) + + +class TestBatchSizeCap: + def test_at_cap_accepts(self): + _check_field("batch_size", _MAX_BATCH_SIZE) + assert _MAX_BATCH_SIZE == 4096 + + def test_over_cap_rejects(self): + with pytest.raises(ValidationError): + _check_field("batch_size", _MAX_BATCH_SIZE + 1) + + def test_below_min_rejects(self): + with pytest.raises(ValidationError): + _check_field("batch_size", 0) + + +class TestLoraRCap: + def test_at_cap_accepts(self): + _check_field("lora_r", _MAX_LORA_R) + assert _MAX_LORA_R == 16_384 + + def test_over_cap_rejects(self): + with pytest.raises(ValidationError): + _check_field("lora_r", _MAX_LORA_R + 1) + + def test_below_min_rejects(self): + with pytest.raises(ValidationError): + _check_field("lora_r", 0) + + +class TestLoraAlphaCap: + def test_at_cap_accepts(self): + _check_field("lora_alpha", _MAX_LORA_ALPHA) + assert _MAX_LORA_ALPHA == 32_768 + + def test_over_cap_rejects(self): + with pytest.raises(ValidationError): + _check_field("lora_alpha", _MAX_LORA_ALPHA + 1) + + def test_below_min_rejects(self): + with pytest.raises(ValidationError): + _check_field("lora_alpha", 0) diff --git a/studio/backend/tests/test_trained_model_scan.py b/studio/backend/tests/test_trained_model_scan.py index 84be681fca..8ba97af701 100644 --- a/studio/backend/tests/test_trained_model_scan.py +++ b/studio/backend/tests/test_trained_model_scan.py @@ -28,7 +28,16 @@ from utils.models.model_config import ( ) -def test_scan_trained_models_includes_lora_and_full_finetune_outputs(tmp_path: Path): +def test_scan_trained_models_includes_lora_and_full_finetune_outputs( + tmp_path: Path, monkeypatch +): + # resolve_output_dir refuses absolutes outside outputs_root; point it at tmp_path. + from utils.models import model_config as _mc + from utils.paths import storage_roots as _sr + + monkeypatch.setattr(_sr, "outputs_root", lambda: tmp_path) + monkeypatch.setattr(_mc, "outputs_root", lambda: tmp_path) + lora_dir = tmp_path / "unsloth_SmolLM-135M_1775412608" lora_dir.mkdir() (lora_dir / "adapter_config.json").write_text( diff --git a/studio/backend/utils/_studio_release_build.py b/studio/backend/utils/_studio_release_build.py new file mode 100644 index 0000000000..267197a202 --- /dev/null +++ b/studio/backend/utils/_studio_release_build.py @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Build-stamped Studio release metadata. + +Release builds may rewrite this module in the build workspace before creating +Python artifacts. Keep the committed value neutral so source checkouts do not +accidentally report a stale release tag. +""" + +STUDIO_RELEASE_VERSION = None diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 9eacf893ad..ebf85c5320 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -1696,20 +1696,21 @@ def get_base_model_from_checkpoint(checkpoint_path: str) -> Optional[str]: ) return base_model - training_args_path = checkpoint_path_obj / "training_args.bin" - if training_args_path.exists(): - try: - import torch - - training_args = torch.load(training_args_path) - if hasattr(training_args, "model_name_or_path"): - base_model = training_args.model_name_or_path - logger.info( - "Detected base model from training_args.bin: %s", base_model - ) - return base_model - except Exception as e: - logger.warning(f"Could not load training_args.bin: {e}") + # TODO: torch.load default weights_only=True (torch >= 2.6) rejects pickled TrainingArguments; re-enable via safe_globals or weights_only=False once threat model allows. + # training_args_path = checkpoint_path_obj / "training_args.bin" + # if training_args_path.exists(): + # try: + # import torch + # + # training_args = torch.load(training_args_path) + # if hasattr(training_args, "model_name_or_path"): + # base_model = training_args.model_name_or_path + # logger.info( + # "Detected base model from training_args.bin: %s", base_model + # ) + # return base_model + # except Exception as e: + # logger.warning(f"Could not load training_args.bin: {e}") dir_name = checkpoint_path_obj.name if dir_name.startswith("unsloth_"): @@ -1757,20 +1758,21 @@ def get_base_model_from_lora(lora_path: str) -> Optional[str]: return base_model # Fallback: try training_args.bin (requires torch) - training_args_path = lora_path_obj / "training_args.bin" - if training_args_path.exists(): - try: - import torch - - training_args = torch.load(training_args_path) - if hasattr(training_args, "model_name_or_path"): - base_model = training_args.model_name_or_path - logger.info( - f"Detected base model from training_args.bin: {base_model}" - ) - return base_model - except Exception as e: - logger.warning(f"Could not load training_args.bin: {e}") + # TODO: torch.load default weights_only=True (torch >= 2.6) rejects pickled TrainingArguments; also an RCE sink for third-party LoRAs via this route, re-enable behind a trust check if needed. + # training_args_path = lora_path_obj / "training_args.bin" + # if training_args_path.exists(): + # try: + # import torch + # + # training_args = torch.load(training_args_path) + # if hasattr(training_args, "model_name_or_path"): + # base_model = training_args.model_name_or_path + # logger.info( + # f"Detected base model from training_args.bin: {base_model}" + # ) + # return base_model + # except Exception as e: + # logger.warning(f"Could not load training_args.bin: {e}") # Last resort: parse from directory name # Format: unsloth_Meta-Llama-3.1-8B-Instruct-bnb-4bit_timestamp diff --git a/studio/backend/utils/paths/storage_roots.py b/studio/backend/utils/paths/storage_roots.py index 58a4d7967c..763d18bf3e 100644 --- a/studio/backend/utils/paths/storage_roots.py +++ b/studio/backend/utils/paths/storage_roots.py @@ -276,21 +276,52 @@ def _clean_relative_path( return Path(*parts) if parts else Path() +def _assert_contained(resolved: Path, root: Path) -> None: + """Raise ValueError if ``resolved`` realpaths outside ``root``.""" + try: + resolved_real = Path(os.path.realpath(resolved)) + root_real = Path(os.path.realpath(root)) + except OSError as exc: + raise ValueError(f"path resolution failed: {exc}") from exc + try: + resolved_real.relative_to(root_real) + except ValueError as exc: + raise ValueError( + f"path escapes root: {resolved!s} -> {resolved_real!s} " + f"is not under {root_real!s}" + ) from exc + + def resolve_under_root( path_value: str | None, *, root: Path, strip_prefixes: tuple[str, ...] = (), ) -> Path: + """Resolve ``path_value`` and assert the result is under ``root``. + + Absolutes are accepted only if already contained (so internal pre-resolved + paths re-enter idempotently); user-facing schemas reject absolutes upstream. + """ if not path_value or not str(path_value).strip(): return root - path = Path(str(path_value).strip()).expanduser() + raw = str(path_value).strip() + if "\x00" in raw: + raise ValueError("path may not contain null bytes") + + path = Path(raw).expanduser() + if ".." in path.parts: + raise ValueError(f"path may not contain '..' segments: {raw!r}") + if path.is_absolute(): + _assert_contained(path, root) return path - cleaned = _clean_relative_path(str(path), strip_prefixes = strip_prefixes) - return root / cleaned + cleaned = _clean_relative_path(raw, strip_prefixes = strip_prefixes) + candidate = root / cleaned + _assert_contained(candidate, root) + return candidate def resolve_output_dir(path_value: str | None = None) -> Path: @@ -318,9 +349,22 @@ def resolve_tensorboard_dir(path_value: str | None = None) -> Path: def resolve_dataset_path(path_value: str) -> Path: - path = Path(path_value).expanduser() + raw = str(path_value or "").strip() + if "\x00" in raw: + raise ValueError("dataset path may not contain null bytes") + path = Path(raw).expanduser() + if ".." in path.parts: + raise ValueError(f"dataset path may not contain '..' segments: {raw!r}") if path.is_absolute(): - return path + for root_fn in (datasets_root, dataset_uploads_root, recipe_datasets_root): + try: + _assert_contained(path, root_fn()) + return path + except ValueError: + continue + raise ValueError( + f"dataset path must be relative or under a dataset root: {raw!r}" + ) parts = [part for part in Path(path_value).parts if part not in ("", ".")] if parts[:2] == ["assets", "datasets"]: diff --git a/studio/backend/utils/studio_version.py b/studio/backend/utils/studio_version.py new file mode 100644 index 0000000000..70059f8a3c --- /dev/null +++ b/studio/backend/utils/studio_version.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Network-free Studio release version resolution for display-only UI.""" + +from __future__ import annotations + +import re +import subprocess +from pathlib import Path + +from utils import _studio_release_build + +_DEV_VERSION = "dev" +_GIT_TIMEOUT_SECONDS = 1.0 +_STUDIO_TAG_RE = re.compile(r"^v\d+\.\d+\.\d+(?:-[0-9A-Za-z.][0-9A-Za-z.-]*)?$") +_GIT_DESCRIBE_SUFFIX_RE = re.compile(r"-\d+-g[0-9A-Fa-f]+(?:-dirty)?$") +_MAX_VERSION_LENGTH = 64 + + +def is_valid_studio_release_version(value: object) -> bool: + """Return True for Studio release tags such as ``v0.1.39-beta``.""" + if not isinstance(value, str): + return False + version = value.strip() + if not version or len(version) > _MAX_VERSION_LENGTH: + return False + if version.endswith("-dirty") or _GIT_DESCRIBE_SUFFIX_RE.search(version): + return False + return _STUDIO_TAG_RE.fullmatch(version) is not None + + +def _repo_root() -> Path: + return Path(__file__).resolve().parents[3] + + +def _path_is_in_site_packages(path: Path) -> bool: + return any(part in {"site-packages", "dist-packages"} for part in path.parts) + + +def _is_source_checkout(repo_root: Path) -> bool: + return (repo_root / ".git").exists() and not _path_is_in_site_packages( + Path(__file__).resolve() + ) + + +def _exact_git_studio_tag(repo_root: Path) -> str | None: + try: + result = subprocess.run( + [ + "git", + "describe", + "--tags", + "--exact-match", + "--match", + "v[0-9]*", + "HEAD", + ], + cwd = repo_root, + check = False, + stdout = subprocess.PIPE, + stderr = subprocess.DEVNULL, + text = True, + timeout = _GIT_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.TimeoutExpired): + return None + + if result.returncode != 0: + return None + + tag = result.stdout.strip() + return tag if is_valid_studio_release_version(tag) else None + + +def get_studio_version(repo_root: Path | None = None) -> str: + """Return the installed Studio release tag for display, or ``dev``. + + This value is intentionally separate from the PyPI ``unsloth`` package + version used by update checks. It never performs network requests. + """ + resolved_repo_root = repo_root or _repo_root() + + if _is_source_checkout(resolved_repo_root): + git_tag = _exact_git_studio_tag(resolved_repo_root) + return git_tag if git_tag is not None else _DEV_VERSION + + stamped_version = _studio_release_build.STUDIO_RELEASE_VERSION + if is_valid_studio_release_version(stamped_version): + return stamped_version.strip() + + return _DEV_VERSION diff --git a/studio/backend/utils/update_status.py b/studio/backend/utils/update_status.py new file mode 100644 index 0000000000..9142203a69 --- /dev/null +++ b/studio/backend/utils/update_status.py @@ -0,0 +1,374 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Web update status helpers for browser-served Unsloth Studio. + +This module is intentionally side-effect light: no network work happens at +import time or from /api/health. The PyPI check is lazy, cached, and only used +for normal PyPI-managed installs. +""" + +from __future__ import annotations + +import json +import os +import threading +import time +import urllib.request +from dataclasses import dataclass +from datetime import datetime, timezone +from importlib.metadata import PackageNotFoundError, distribution +from pathlib import Path +from typing import Any + +from packaging.version import InvalidVersion, Version + +PACKAGE_NAME = "unsloth" +PYPI_JSON_URL = "https://pypi.org/pypi/unsloth/json" +PYPI_TIMEOUT_SECONDS = 3 +PYPI_RESPONSE_MAX_BYTES = 5 * 1024 * 1024 +PYPI_SUCCESS_TTL_SECONDS = 12 * 60 * 60 +PYPI_FAILURE_TTL_SECONDS = 60 * 60 +RELEASE_NOTES_URL = "https://unsloth.ai/docs/new/changelog" +DISABLE_ENV_VAR = "UNSLOTH_DISABLE_UPDATE_CHECK" + +LOCAL_INSTALL_SOURCES = {"editable", "local_path", "vcs", "local_repo"} + + +@dataclass(frozen = True) +class LatestVersionResult: + latest_version: str | None + checked_at: str + reason: str | None = None + error: str | None = None + + +@dataclass +class _LatestVersionCacheEntry: + result: LatestVersionResult + expires_at: float + + +_cache_condition = threading.Condition() +_latest_version_cache: _LatestVersionCacheEntry | None = None +_latest_version_fetching = False + + +def reset_update_status_cache() -> None: + """Clear the in-process PyPI cache. Intended for tests.""" + global _latest_version_cache, _latest_version_fetching + with _cache_condition: + _latest_version_cache = None + _latest_version_fetching = False + _cache_condition.notify_all() + + +def detect_install_source() -> str: + """Return a coarse install source without exposing local paths. + + Sources are intentionally conservative. PEP 610 local/vcs metadata wins. + Legacy source installs are treated as local only when package files resolve + outside site-packages/dist-packages and under a Git checkout. + """ + try: + dist = distribution(PACKAGE_NAME) + except PackageNotFoundError: + return ( + "local_repo" + if _path_has_git_parent(_repo_root_from_this_file()) + else "unknown" + ) + + try: + direct_url = dist.read_text("direct_url.json") + except Exception: + return "unknown" + if direct_url: + return _source_from_direct_url(direct_url) + + for package_path in _distribution_package_paths(dist): + if not _path_is_under_python_package_dir(package_path) and _path_has_git_parent( + package_path + ): + return "local_repo" + + return "pypi" + + +def get_studio_install_source_status(current_version: str) -> dict[str, Any]: + """Return install-source metadata without remote update checks.""" + install_source = detect_install_source() + reason = None + if install_source in LOCAL_INSTALL_SOURCES: + reason = "local_source" + elif install_source == "unknown": + reason = "unknown_source" + + return _status_response( + current_version = current_version, + latest_version = None, + install_source = install_source, + reason = reason, + ) + + +def get_studio_update_status(current_version: str) -> dict[str, Any]: + """Return public, read-only update status for the web UI.""" + install_source = detect_install_source() + + if os.environ.get(DISABLE_ENV_VAR) == "1": + return _status_response( + current_version = current_version, + latest_version = None, + install_source = install_source, + reason = "disabled", + ) + + if install_source in LOCAL_INSTALL_SOURCES: + return _status_response( + current_version = current_version, + latest_version = None, + install_source = install_source, + reason = "local_source", + ) + + if install_source != "pypi": + return _status_response( + current_version = current_version, + latest_version = None, + install_source = install_source, + reason = "unknown_source", + ) + + current = _parse_current_version(current_version) + if current is None: + return _status_response( + current_version = current_version, + latest_version = None, + install_source = install_source, + reason = "invalid_current_version" + if current_version != "dev" + else "dev_build", + ) + latest_result = get_latest_pypi_version() + if latest_result.latest_version is None: + return _status_response( + current_version = current_version, + latest_version = None, + install_source = install_source, + reason = latest_result.reason or "offline", + error = latest_result.error, + checked_at = latest_result.checked_at, + ) + + try: + latest = Version(latest_result.latest_version) + except InvalidVersion: + return _status_response( + current_version = current_version, + latest_version = latest_result.latest_version, + install_source = install_source, + reason = "invalid_latest_version", + error = "PyPI returned an invalid version.", + checked_at = latest_result.checked_at, + ) + + if latest > current: + return _status_response( + current_version = current_version, + latest_version = latest_result.latest_version, + install_source = install_source, + update_available = True, + can_show_web_notification = True, + checked_at = latest_result.checked_at, + ) + + return _status_response( + current_version = current_version, + latest_version = latest_result.latest_version, + install_source = install_source, + reason = "current_not_older", + checked_at = latest_result.checked_at, + ) + + +def get_latest_pypi_version() -> LatestVersionResult: + """Return the latest PyPI version using a small in-process TTL cache.""" + global _latest_version_cache, _latest_version_fetching + + while True: + now = time.monotonic() + with _cache_condition: + if _latest_version_cache and _latest_version_cache.expires_at > now: + return _latest_version_cache.result + if not _latest_version_fetching: + _latest_version_fetching = True + break + _cache_condition.wait(timeout = PYPI_TIMEOUT_SECONDS + 1) + + try: + result = _fetch_latest_pypi_version() + except Exception: + result = LatestVersionResult( + latest_version = None, + checked_at = _utc_now_iso(), + reason = "offline", + error = "Could not check PyPI update metadata.", + ) + + ttl = ( + PYPI_SUCCESS_TTL_SECONDS if result.latest_version else PYPI_FAILURE_TTL_SECONDS + ) + with _cache_condition: + _latest_version_cache = _LatestVersionCacheEntry( + result = result, + expires_at = time.monotonic() + ttl, + ) + _latest_version_fetching = False + _cache_condition.notify_all() + return result + + +def _fetch_latest_pypi_version() -> LatestVersionResult: + checked_at = _utc_now_iso() + request = urllib.request.Request( + PYPI_JSON_URL, + headers = {"User-Agent": "unsloth-studio-update-check"}, + ) + + try: + with urllib.request.urlopen(request, timeout = PYPI_TIMEOUT_SECONDS) as response: + body = response.read(PYPI_RESPONSE_MAX_BYTES + 1) + if len(body) > PYPI_RESPONSE_MAX_BYTES: + return LatestVersionResult( + latest_version = None, + checked_at = checked_at, + reason = "malformed_response", + error = "PyPI returned oversized update metadata.", + ) + payload = json.loads(body.decode("utf-8")) + except json.JSONDecodeError: + return LatestVersionResult( + latest_version = None, + checked_at = checked_at, + reason = "malformed_response", + error = "PyPI returned malformed update metadata.", + ) + except OSError: + return LatestVersionResult( + latest_version = None, + checked_at = checked_at, + reason = "offline", + error = "Could not reach PyPI for update metadata.", + ) + + latest = ( + payload.get("info", {}).get("version") if isinstance(payload, dict) else None + ) + if not isinstance(latest, str) or not latest.strip(): + return LatestVersionResult( + latest_version = None, + checked_at = checked_at, + reason = "malformed_response", + error = "PyPI update metadata did not include a version.", + ) + + return LatestVersionResult(latest_version = latest.strip(), checked_at = checked_at) + + +def _status_response( + *, + current_version: str, + latest_version: str | None, + install_source: str, + reason: str | None = None, + error: str | None = None, + update_available: bool = False, + can_show_web_notification: bool = False, + checked_at: str | None = None, +) -> dict[str, Any]: + return { + "current_version": current_version, + "latest_version": latest_version, + "update_available": update_available, + "install_source": install_source, + "can_show_web_notification": can_show_web_notification, + "release_notes_url": RELEASE_NOTES_URL, + "checked_at": checked_at or _utc_now_iso(), + "reason": reason, + "error": error, + } + + +def _source_from_direct_url(direct_url: str) -> str: + try: + payload = json.loads(direct_url) + except json.JSONDecodeError: + return "unknown" + + if not isinstance(payload, dict): + return "unknown" + + dir_info = payload.get("dir_info") + if isinstance(dir_info, dict) and dir_info.get("editable") is True: + return "editable" + + if isinstance(payload.get("vcs_info"), dict): + return "vcs" + + url = payload.get("url") + if isinstance(url, str) and url.startswith("file:"): + return "local_path" + + return "unknown" + + +def _distribution_package_paths(dist: Any) -> list[Path]: + paths: list[Path] = [] + files = getattr(dist, "files", None) or [] + for file in files: + text = str(file) + if not text.startswith(("unsloth/", "unsloth_cli/", "studio/")): + continue + try: + paths.append(Path(dist.locate_file(file)).resolve()) + except OSError: + continue + return paths + + +def _path_is_under_python_package_dir(path: Path) -> bool: + return any(part in {"site-packages", "dist-packages"} for part in path.parts) + + +def _path_has_git_parent(path: Path) -> bool: + for candidate in (path, *path.parents): + if (candidate / ".git").exists(): + return True + return False + + +def _repo_root_from_this_file() -> Path: + # update_status.py -> utils -> backend -> studio -> repo root + try: + return Path(__file__).resolve().parents[3] + except IndexError: + return Path(__file__).resolve().parent + + +def _parse_current_version(current_version: str) -> Version | None: + if current_version == "dev": + return None + try: + return Version(current_version) + except InvalidVersion: + return None + + +def _utc_now_iso() -> str: + return ( + datetime.now(timezone.utc) + .replace(microsecond = 0) + .isoformat() + .replace("+00:00", "Z") + ) diff --git a/studio/frontend/.npmrc b/studio/frontend/.npmrc new file mode 100644 index 0000000000..8e21abe7a2 --- /dev/null +++ b/studio/frontend/.npmrc @@ -0,0 +1,28 @@ +# Studio frontend npm configuration. +# +# Mini Shai-Hulud / Axios-style supply chain defense. +# Requires npm >=11.10.0. Refuses tarballs published less than 7 days ago, +# closing the typical 4-72h attack window between malicious publish and +# upstream removal. npm interprets the bare integer as DAYS; do not +# append `d`, npm 11.x will parse `7d` as a Date string and abort. +min-release-age=7 +# Defensive alias: `minimum-release-age` takes minutes (10080 = 7 days). +# Some npm versions / wrappers consult one key but not the other; setting +# both means a single setting-name parse change upstream cannot silently +# disable the cooldown. The two keys MUST agree; do not let them drift. +minimum-release-age=10080 +# Belt-and-braces: refuse to write back loose `^x.y.z` ranges into +# package.json when a maintainer runs `npm install ` locally. This +# does NOT rewrite already-present ranges (those need an explicit +# `npm install @ --save-exact` pass) but it stops new +# carets from creeping into the manifest as patch-version footguns. +save-exact=true +# Lock the registry. A user-set PIP_INDEX_URL-style override (here: +# NPM_CONFIG_REGISTRY env var or a stale ~/.npmrc) shouldn't redirect +# our installs to an attacker registry. +registry=https://registry.npmjs.org/ +audit-level=high +fund=false +# Maintainer note: use `npm ci` (never `npm install`) in CI and locally +# when reproducing a build. The 7-day cooldown above is enforced by npm +# itself; downgrading or removing it bypasses the supply-chain gate. diff --git a/studio/frontend/package.json b/studio/frontend/package.json index 463fbd9261..3a02bb926a 100644 --- a/studio/frontend/package.json +++ b/studio/frontend/package.json @@ -41,7 +41,7 @@ "@streamdown/math": "1.0.2", "@streamdown/mermaid": "1.0.2", "@tailwindcss/vite": "^4.2.2", - "@tanstack/react-router": "^1.159.10", + "@tanstack/react-router": "1.169.2", "@tanstack/react-table": "^8.21.3", "@tauri-apps/api": "^2.10.1", "@tauri-apps/plugin-clipboard-manager": "^2.3.2", @@ -83,6 +83,11 @@ "unpdf": "^1.4.0", "zustand": "^5.0.11" }, + "overrides": { + "@tanstack/react-router": "1.169.2", + "@tanstack/router-core": "1.169.2", + "@tanstack/history": "1.161.6" + }, "devDependencies": { "@biomejs/biome": "^1.9.4", "@eslint/js": "^9.39.1", diff --git a/studio/frontend/src/app/provider.tsx b/studio/frontend/src/app/provider.tsx index 62e78b809a..8360186d1e 100644 --- a/studio/frontend/src/app/provider.tsx +++ b/studio/frontend/src/app/provider.tsx @@ -9,6 +9,7 @@ import { shouldUseCustomWindowTitlebar, } from "@/components/tauri/window-titlebar"; import { Toaster } from "@/components/ui/sonner"; +import { WebUpdateBanner } from "@/components/web/update-banner"; import { getTauriAuthFailure, tauriAutoAuth } from "@/features/auth"; import { NativeIntentDrain } from "@/features/native-intents/native-intent-drain"; import { useTauriBackend, type BackendStatus } from "@/hooks/use-tauri-backend"; @@ -22,10 +23,6 @@ interface AppProviderProps { children: ReactNode; } -// --------------------------------------------------------------------------- -// Tauri window helpers (only imported in Tauri mode) -// --------------------------------------------------------------------------- - type TauriWindowMode = "setup" | "app"; type WindowLayoutGuard = () => boolean; @@ -52,19 +49,15 @@ async function applyAppWindowLayout(isCurrent: WindowLayoutGuard): Promise let finalH = 600; if (monitor) { - // Convert physical pixels to logical using scale factor const scale = monitor.scaleFactor; const screenW = monitor.size.width / scale; const screenH = monitor.size.height / scale; - // Target: 75% of screen width, golden ratio height, capped at min 900x600 finalW = Math.max(900, Math.round(screenW * 0.75)); const targetH = Math.max(600, Math.round(finalW / 1.618)); - // Don't exceed screen height finalH = Math.min(targetH, Math.round(screenH * 0.85)); } - // Apply constraints and finalize without animating through intermediate sizes if (!isCurrent()) return; await win.setSize(new LogicalSize(finalW, finalH)); if (!isCurrent()) return; @@ -107,10 +100,6 @@ function getTauriWindowMode( } } -// --------------------------------------------------------------------------- -// TauriWrapper -// --------------------------------------------------------------------------- - function TauriUpdateLayer({ isExternalServer }: { isExternalServer: boolean }) { const update = useTauriUpdate(isExternalServer); const isUpdating = @@ -140,6 +129,8 @@ function TauriUpdateLayer({ isExternalServer }: { isExternalServer: boolean }) { dismissed={update.dismissed} lastFailure={update.lastFailure} isExternalServer={isExternalServer} + updatePolicyMode={update.updatePolicyMode} + manualReleaseUrl={update.manualReleaseUrl} onInstall={update.installUpdate} onDismiss={update.dismiss} onCopyDiagnostics={update.copyDiagnostics} @@ -154,6 +145,13 @@ const HIDDEN_TITLEBAR_SIDEBAR_ROUTES = new Set([ "/signup", ]); +const WEB_UPDATE_HIDDEN_ROUTES = new Set([ + "/onboarding", + "/login", + "/change-password", + "/signup", +]); + function TauriWrapper({ children }: { children: ReactNode }) { const pathname = useRouterState({ select: (s) => s.location.pathname }); const { @@ -176,8 +174,7 @@ function TauriWrapper({ children }: { children: ReactNode }) { }; }, []); - // Keep the Tauri window hidden during preflight, then show it centered in setup - // mode or apply the final app layout in one instant step. + // Keep the Tauri window hidden until setup or app layout is ready. useEffect(() => { if (!isTauri) return; @@ -234,7 +231,14 @@ function TauriWrapper({ children }: { children: ReactNode }) { return () => { disposed = true; }; }, [status, desktopAuthRetry]); - if (!isTauri) return <>{children}; + if (!isTauri) { + return ( + <> + {children} + + + ); + } const showApp = status === "running" && desktopAuthReady; const startupStatus = status === "running" ? "starting" : status; diff --git a/studio/frontend/src/components/assistant-ui/reasoning.tsx b/studio/frontend/src/components/assistant-ui/reasoning.tsx index fe913baf2a..e4401cc12e 100644 --- a/studio/frontend/src/components/assistant-ui/reasoning.tsx +++ b/studio/frontend/src/components/assistant-ui/reasoning.tsx @@ -19,10 +19,8 @@ import { useAuiState, } from "@assistant-ui/react"; import { copyToClipboard } from "@/lib/copy-to-clipboard"; -import { Idea01Icon } from "@hugeicons/core-free-icons"; -import { HugeiconsIcon } from "@hugeicons/react"; import { type VariantProps, cva } from "class-variance-authority"; -import { ChevronDownIcon, CopyIcon, CheckIcon } from "lucide-react"; +import { CheckIcon, ChevronDownIcon, CopyIcon, LightbulbIcon } from "lucide-react"; import { type CSSProperties, type ComponentProps, @@ -128,10 +126,7 @@ function ReasoningTrigger({ )} {...props} > - + { }; const ThreadWelcome: FC<{ hideComposer?: boolean }> = ({ hideComposer }) => { + const [currentEmoji, setCurrentEmoji] = useState("large sloth drink.png"); + + useEffect(() => { + const hour = new Date().getHours(); + if (hour >= 6 && hour < 12) setCurrentEmoji("large sloth drink.png"); + else if (hour >= 12 && hour < 17) setCurrentEmoji("sloth magnify final.png"); + else if (hour >= 17 && hour < 21) setCurrentEmoji("sloth shy large.png"); + else setCurrentEmoji("unsloth-gem.png"); + }, []); + + const currentEmojiSrc = + currentEmoji === "unsloth-gem.png" + ? `/${currentEmoji}` + : `/Sloth emojis/${currentEmoji}`; + return (
Sloth mascot diff --git a/studio/frontend/src/components/tauri/update-banner.tsx b/studio/frontend/src/components/tauri/update-banner.tsx index 79fbae6bb7..62038f92a9 100644 --- a/studio/frontend/src/components/tauri/update-banner.tsx +++ b/studio/frontend/src/components/tauri/update-banner.tsx @@ -2,7 +2,12 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import { Button } from "@/components/ui/button"; -import type { RetainedUpdateFailure, UpdateInfo, UpdateStatus } from "@/hooks/use-tauri-update"; +import type { + DesktopUpdatePolicyMode, + RetainedUpdateFailure, + UpdateInfo, + UpdateStatus, +} from "@/hooks/use-tauri-update"; import type { CopySupportDiagnosticsResult } from "@/lib/tauri-diagnostics"; import { AnimatePresence, motion } from "motion/react"; import { useState } from "react"; @@ -13,6 +18,8 @@ interface UpdateBannerProps { dismissed: boolean; lastFailure: RetainedUpdateFailure | null; isExternalServer?: boolean; + updatePolicyMode: DesktopUpdatePolicyMode; + manualReleaseUrl: string | null; onInstall: () => void; onDismiss: () => void; onCopyDiagnostics: () => Promise; @@ -26,6 +33,8 @@ export function UpdateBanner({ dismissed, lastFailure, isExternalServer = false, + updatePolicyMode, + manualReleaseUrl, onInstall, onDismiss, onCopyDiagnostics, @@ -36,6 +45,10 @@ export function UpdateBanner({ const showFailure = Boolean(lastFailure) && !dismissed; const showAvailable = status === "available" && !dismissed && !showFailure; const show = showFailure || (showAvailable && Boolean(info)); + const isManualLinuxPackage = updatePolicyMode === "manual_linux_package"; + const installDisabled = isManualLinuxPackage + ? manualReleaseUrl === null + : isExternalServer; async function handleCopyDiagnostics() { setCopying(true); @@ -67,18 +80,16 @@ export function UpdateBanner({ className="fixed top-4 right-4 z-[9999] w-[380px]" >
- {/* Close button */} - {/* Header */}
🦥
@@ -88,37 +99,39 @@ export function UpdateBanner({

{showFailure ? "Backend recovered. Diagnostics are still available." - : isExternalServer - ? "Run `unsloth studio update` from your terminal" - : "A new app update is available"} + : isManualLinuxPackage + ? "Open the GitHub release page to install the Linux package" + : isExternalServer + ? "Run `unsloth studio update` from your terminal" + : "A new app update is available"}

- {/* Retained failure */} {showFailure && lastFailure && (

{lastFailure.error}

)} - {/* Actions */}
{showFailure ? ( <> - - ) : ( <> - - @@ -132,7 +145,7 @@ export function UpdateBanner({ )} {manualReport && (