diff --git a/.github/workflows/security-audit.yml b/.github/workflows/security-audit.yml index 642b214b64..7c0e84834b 100644 --- a/.github/workflows/security-audit.yml +++ b/.github/workflows/security-audit.yml @@ -261,6 +261,126 @@ jobs: echo '```' } >> "$GITHUB_STEP_SUMMARY" + # ───────────────────────────────────────────────────────────── + # OSV-Scanner: cross-ecosystem advisory DB (PyPI + npm + cargo) + # ───────────────────────────────────────────────────────────── + - name: OSV-Scanner (PyPI + npm + cargo, cross-ecosystem advisories) + # OSV's advisory feed is a superset of GitHub-Advisory + RustSec + # + npm advisories; running it alongside the per-ecosystem audit + # tools catches CVEs that haven't propagated to the per-ecosystem + # DBs yet (e.g. langchain-core CVE-2025-68664 was on OSV before + # GitHub Advisory). Single binary, one transitive resolver, all + # three lockfile types in one pass. Non-blocking until baselines + # close. + continue-on-error: true + run: | + set +e + curl -fsSL -o /tmp/osv-scanner.tar.gz \ + https://github.com/google/osv-scanner/releases/download/v2.0.2/osv-scanner_linux_amd64.tar.gz + tar -xzf /tmp/osv-scanner.tar.gz -C /tmp osv-scanner + /tmp/osv-scanner --version + /tmp/osv-scanner scan source \ + --lockfile=studio/frontend/package-lock.json \ + --lockfile=studio/src-tauri/Cargo.lock \ + --lockfile=requirements.txt:audit-reqs/unsloth-deps.txt \ + --lockfile=requirements.txt:audit-reqs/studio.txt \ + --lockfile=requirements.txt:audit-reqs/no-torch-runtime.txt \ + --lockfile=requirements.txt:audit-reqs/overrides.txt \ + --lockfile=requirements.txt:audit-reqs/extras.txt \ + --lockfile=requirements.txt:audit-reqs/extras-no-deps.txt \ + --format=table 2>&1 | tee logs-osv-scanner.txt + { + echo "## OSV-Scanner (cross-ecosystem)" + echo + echo '```' + tail -200 logs-osv-scanner.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + # ───────────────────────────────────────────────────────────── + # Semgrep: design-flaw detection (catches what regex-pattern + # scanning of malicious authors cannot — first-party logic bugs + # like langchain-core CVE-2025-68664 dumps/dumpd injection, + # n8n CVE-2025-68668 _pyodide.eval_code sandbox escape, marimo + # CVE-2026-39987 unauth WebSocket). + # ───────────────────────────────────────────────────────────── + - name: Semgrep (supply-chain + python rule packs) + continue-on-error: true + run: | + set +e + python -m pip install --quiet 'semgrep>=1.95' + semgrep --version + semgrep scan \ + --config p/supply-chain \ + --config p/python \ + --config p/javascript \ + --config p/security-audit \ + --severity ERROR --severity WARNING \ + --metrics off \ + --timeout 120 \ + studio/backend unsloth scripts \ + 2>&1 | tee logs-semgrep.txt + { + echo "## Semgrep (supply-chain + python + javascript rules)" + echo + echo '```' + tail -200 logs-semgrep.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + # ───────────────────────────────────────────────────────────── + # Lockfile pin verifier. The litellm 1.82.7 attack window was + # ~40 minutes; anyone resolving with `>=` got the malicious + # version automatically. Flag every spec in the requirements + # files that does not pin to an exact `==` (or `@` for git + # refs, or `===` for arbitrary equality). Warning-only for now; + # graduate to blocking once the baseline is clean. + # ───────────────────────────────────────────────────────────── + - name: Lockfile pin verifier (Python requirements) + continue-on-error: true + run: | + python <<'PY' | tee logs-pin-verifier.txt + import re + from pathlib import Path + + # Specs that look like `pkg==1.2.3` or `pkg @ git+...` or + # bare comments / -r lines are pinned-or-not-applicable. + PINNED = re.compile(r"^\s*[A-Za-z0-9_.\-]+\s*(?:===|==)\s*[^,;]+\s*$") + GIT_OR_URL = re.compile(r"^\s*[A-Za-z0-9_.\-]+\s*@\s*(?:git\+|https?://)") + + unpinned = [] + for f in sorted(Path("studio/backend/requirements").glob("*.txt")): + for i, raw in enumerate(f.read_text().splitlines(), 1): + line = raw.strip() + if not line or line.startswith("#") or line.startswith("-"): + continue + spec = line.split("#", 1)[0].strip().split(";", 1)[0].strip() + if not spec: + continue + if "git+" in spec or PINNED.match(spec) or GIT_OR_URL.match(spec): + continue + unpinned.append((str(f), i, line)) + + print(f"::group::Lockfile pin status") + if unpinned: + print(f"WARN: {len(unpinned)} non-`==` specs across requirements/*.txt") + print("(litellm 1.82.7 wave hit anyone on `>=`; tighten when feasible.)") + for f, i, line in unpinned[:80]: + print(f" {f}:{i}: {line}") + if len(unpinned) > 80: + print(f" ... and {len(unpinned) - 80} more") + else: + print("OK: every spec is exact-pinned.") + print("::endgroup::") + PY + { + echo "## Lockfile pin verifier" + echo + echo '```' + cat logs-pin-verifier.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@v4 if: always() with: @@ -270,6 +390,9 @@ jobs: logs-npm-audit.txt logs-npm-audit.json logs-cargo-audit.txt + logs-osv-scanner.txt + logs-semgrep.txt + logs-pin-verifier.txt audit-reqs/ retention-days: 30 diff --git a/scripts/scan_packages.py b/scripts/scan_packages.py index dc37bb002a..fc10ac8d52 100644 --- a/scripts/scan_packages.py +++ b/scripts/scan_packages.py @@ -306,6 +306,84 @@ RE_C2_POLLING = re.compile( re.DOTALL, ) +# Developer-tool persistence hooks. The PyTorch Lightning 2.6.x compromise +# planted SessionStart hooks into Claude Code, VS Code tasks, and Cursor +# settings so the payload re-attached on every editor open. Catches any +# package writing into a known dev-tool config that supports auto-run. +RE_DEV_TOOL_HIJACK = re.compile( + r"\.claude/settings\.json" + r"|\.cursor/.*hooks" + r"|\.vscode/(?:tasks|settings|launch)\.json" + r"|SessionStart|folderOpen|onCommand:.*runTask" + r"|/etc/profile\.d/" + r"|\b\.bashrc\b|\b\.zshrc\b|\b\.profile\b" + r"|\bautomator\b.*\.workflow\b", +) + +# Hard-coded credential / API-token regexes embedded in source. Packages +# that ship regexes for OTHER people's secrets are nearly always +# stealers (litellm 1.82.7, elementary-data 0.23.3, Shai-Hulud). +RE_TOKEN_REGEX = re.compile( + r"\bgh[psoru]_[A-Za-z0-9_]{20,}" # GitHub PAT/OAuth/etc. + r"|\bgithub_pat_[A-Za-z0-9_]{20,}" + r"|\bnpm_[A-Za-z0-9]{30,}" # npm token + r"|\bsk-[A-Za-z0-9]{20,}" # OpenAI / Anthropic + r"|\bxox[bpaesr]-" # Slack + r"|\bAIza[0-9A-Za-z_-]{20,}" # Google API key + r"|\bAKIA[0-9A-Z]{16}" # AWS access key id + r"|\bASIA[0-9A-Z]{16}" # AWS STS + r"|\bgithub.com/login/oauth/access_token" + r"|\bglpat-[0-9A-Za-z_-]{20,}", # GitLab PAT +) + +# JavaScript-side obfuscation. The npm chalk/debug compromise and the +# Lightning router_runtime.js use the same minifier-style hex-var name +# pattern; a bundle full of `_0x1f2e3d` identifiers is a near-universal +# tell for a malicious npm payload (and very rare in legit minified code +# that ships in PyPI wheels). +RE_JS_OBFUSCATION = re.compile( + r"_0x[a-f0-9]{4,6}\s*=\s*function" + r"|var\s+_0x[a-f0-9]{4,6}\b" + r"|(?:\\x[0-9a-f]{2}){10,}" # \x-escape strings + r"|String\.fromCharCode\s*\(\s*\d+\s*(?:,\s*\d+\s*){10,}\)", +) + +# Web3 / wallet-hijack pattern. The Qix npm phish overrode fetch / +# XMLHttpRequest and attached a `window.ethereum` listener that +# Levenshtein-swapped recipient addresses on the way to the network. +RE_WEB3_HIJACK = re.compile( + r"\bwindow\.ethereum\b" + r"|\bweb3\.eth\.\w+\s*\(" + r"|XMLHttpRequest\.prototype\.(?:open|send)\s*=" + r"|(?:^|\s)fetch\s*=\s*\(?\s*async" + r"|TronWeb|solanaWeb3", +) + +# Self-propagating supply-chain worms (Shai-Hulud, ForceMemo) plant +# their own GitHub workflow in every repo they can reach, and lean on +# trufflehog/gitleaks for credential discovery. The combo of any of +# these strings inside a *package payload* is overwhelming evidence of +# repo-takeover intent. +RE_WORKFLOW_INJECT = re.compile( + r"\.github/workflows/[^\"\']*\.ya?ml" + r"|\btrufflehog\b|\bgitleaks\b" + r"|/user/repos\?affiliation=.*owner.*collaborator" + r"|\bshai-hulud\b|EveryBoiWeBuildIsAWormyBoi" + r"|\bgit\s+push\s+--force\b.*--no-verify", + re.IGNORECASE | re.DOTALL, +) + +# Shell-side patterns specific to install.sh / postinstall scripts that +# pipe remote code into a shell. `curl ... | sh` and friends are the +# canonical npm postinstall dropper. +RE_SHELL_DROPPER = re.compile( + r"\bcurl\b[^\n|]*\|\s*(?:sh|bash|zsh)\b" + r"|\bwget\b[^\n|]*-O-\s*\|\s*(?:sh|bash|zsh)\b" + r"|\bnpx\b\s+-y\s+[^\s]+@latest\s*\|" + r"|\beval\s+\$\(\s*curl\b" + r"|\bbash\s+<\(\s*curl\b", +) + # --------------------------------------------------------------------------- # Finding dataclass @@ -868,6 +946,125 @@ def _extract_evidence(content: str, pattern: re.Pattern, max_matches: int = 3) - return " | ".join(matches) if matches else "" +# --------------------------------------------------------------------------- +# Non-Python checkers +# --------------------------------------------------------------------------- +# Several recent PyPI compromises (PyTorch Lightning 2.6.x, ForceMemo) +# carried the active payload in a bundled .js / .sh / workflow yaml so +# the Python imports looked clean on first glance. These checkers scan +# those file types when they appear inside a Python wheel/sdist. + +def check_js_file(content: str, filename: str, package: str) -> list[Finding]: + """Run JS-side checks. Triggered by .js / .mjs / .cjs / .ts.""" + findings = [] + + # A JS file *inside a Python wheel* that's larger than 100 KB is + # itself anomalous (legit Python packages don't ship hand-written + # JS bundles). Combined with ANY of the other JS heuristics it is + # CRITICAL; standalone it is HIGH. + is_large = len(content) > 100 * 1024 + has_obf = bool(RE_JS_OBFUSCATION.search(content)) + has_web3 = bool(RE_WEB3_HIJACK.search(content)) + has_token_regex = bool(RE_TOKEN_REGEX.search(content)) + has_workflow_inj = bool(RE_WORKFLOW_INJECT.search(content)) + has_network = bool(RE_NETWORK.search(content)) + + if has_obf: + sev = CRITICAL if (is_large or has_web3 or has_token_regex) else HIGH + findings.append(Finding( + sev, package, filename, + "JS minifier-style hex-var obfuscation (npm-payload signature)", + _extract_evidence(content, RE_JS_OBFUSCATION), + )) + if has_web3: + findings.append(Finding( + CRITICAL, package, filename, + "JS Web3 / wallet hijack (window.ethereum or fetch override)", + _extract_evidence(content, RE_WEB3_HIJACK), + )) + if has_token_regex and has_network: + findings.append(Finding( + CRITICAL, package, filename, + "JS embeds credential regexes AND makes network calls (stealer)", + _extract_evidence(content, RE_TOKEN_REGEX), + )) + if has_workflow_inj: + findings.append(Finding( + CRITICAL, package, filename, + "JS self-propagation: workflow injection / repo takeover signature", + _extract_evidence(content, RE_WORKFLOW_INJECT), + )) + if is_large and not findings: + findings.append(Finding( + HIGH, package, filename, + f"Python wheel ships large ({len(content) // 1024} KB) JS bundle " + "(uncommon; manually review)", + "", + )) + return findings + + +def check_shell_file(content: str, filename: str, package: str) -> list[Finding]: + """Run shell-side checks. Triggered by .sh / .bash / install scripts.""" + findings = [] + if RE_SHELL_DROPPER.search(content): + findings.append(Finding( + CRITICAL, package, filename, + "Shell pipes remote code into an interpreter (curl|sh dropper)", + _extract_evidence(content, RE_SHELL_DROPPER), + )) + if RE_DEV_TOOL_HIJACK.search(content) and ( + RE_NETWORK.search(content) or RE_SUBPROCESS.search(content) + ): + findings.append(Finding( + CRITICAL, package, filename, + "Shell installs developer-tool persistence hook (.bashrc / " + "profile.d / vscode tasks) AND has network or exec", + _extract_evidence(content, RE_DEV_TOOL_HIJACK), + )) + if RE_TOKEN_REGEX.search(content) and RE_NETWORK.search(content): + findings.append(Finding( + CRITICAL, package, filename, + "Shell embeds credential regexes AND makes network calls", + _extract_evidence(content, RE_TOKEN_REGEX), + )) + if RE_WORKFLOW_INJECT.search(content): + findings.append(Finding( + CRITICAL, package, filename, + "Shell self-propagation: workflow injection / repo takeover signature", + _extract_evidence(content, RE_WORKFLOW_INJECT), + )) + return findings + + +def check_workflow_file(content: str, filename: str, package: str) -> list[Finding]: + """Run GitHub-Actions workflow checks. Triggered by .github/workflows/*.yml.""" + findings = [] + # A GitHub workflow file inside a *PyPI package* is itself + # suspicious (Shai-Hulud's whole MO is to plant `shai-hulud.yml` + # in every repo it can write to). Anything matching the workflow + # injection signature gets flagged CRITICAL. + if RE_WORKFLOW_INJECT.search(content): + findings.append(Finding( + CRITICAL, package, filename, + "Workflow file inside PyPI package matches self-propagation signature", + _extract_evidence(content, RE_WORKFLOW_INJECT), + )) + if RE_TOKEN_REGEX.search(content): + findings.append(Finding( + HIGH, package, filename, + "Workflow file embeds credential regexes (token harvesting?)", + _extract_evidence(content, RE_TOKEN_REGEX), + )) + if RE_SHELL_DROPPER.search(content): + findings.append(Finding( + CRITICAL, package, filename, + "Workflow pipes remote code into a shell (curl|sh dropper)", + _extract_evidence(content, RE_SHELL_DROPPER), + )) + return findings + + # --------------------------------------------------------------------------- # Archive handling # --------------------------------------------------------------------------- @@ -916,6 +1113,18 @@ def scan_archive(archive_path: str, package: str) -> list[Finding]: findings.extend(check_pth_file(content, filename, package)) elif lower.endswith(".py"): findings.extend(check_py_file(content, filename, package)) + elif lower.endswith((".js", ".mjs", ".cjs", ".ts")): + # Lightning 2.6.x hid its real payload in a 14.8 MB + # router_runtime.js inside a Python wheel. Without this + # branch we'd have only seen the small Python loader. + findings.extend(check_js_file(content, filename, package)) + elif lower.endswith((".sh", ".bash")): + findings.extend(check_shell_file(content, filename, package)) + elif "/.github/workflows/" in lower and lower.endswith((".yml", ".yaml")): + # Shai-Hulud / ForceMemo plant their own GHA workflow. + # A workflow file inside a *PyPI package* is on its own + # already a yellow flag; pattern-match the worm signatures. + findings.extend(check_workflow_file(content, filename, package)) return findings