unsloth/studio/backend/core/inference/tools.py
danielhanchen 81bb65f4e0 Harden sandbox: dynamic/environb/update PATH mutations; sqlite URI decode + shell/pipe dot-commands; getattr gadget dunders; find -exec in argv
Close seven bypasses Codex found on the round-50 branch.

- dynamic PATH assignment: os.environ['PATH'] = '.:' + os.environ['PATH'] (or an
  f-string) was accepted because the value is non-literal. Fold what we can and fail
  closed when a COMPLETE, fully-literal PATH entry the value contributes is a
  relative / cwd / empty entry; a dynamic ABSOLUTE extension ('/usr/local/bin:' +
  $PATH, venv + ':' + $PATH) stays allowed.

- os.environb mutations: os.environb[b'PATH'] = b'.:...' updates the same inherited
  environment, but only os.environ[...] was recognized. Match environ / environb
  (attribute and bare) and decode a bytes key / value before the policy check.

- os.environ.update / setdefault: a mapping mutator
  (os.environ.update({'PATH': '.:...'}), .update(PATH=...), .setdefault('PATH', ...))
  never hit the subscript check. Run each (key, value) pair through the mutation
  policy in visit_Call.

- sqlite URI percent-decode: sqlite3.connect('file:%2Ftmp%2Fescape.db', uri=True)
  passed the runtime guard as a relative-looking string while SQLite decodes the
  filename and opens /tmp/escape.db. Percent-decode the URI path (with the guard's
  captured chr/int) before the workdir check.

- sqlite shell / pipe dot-commands: the CLI scanner only path-checked file
  dot-commands, but .shell CMD / .system CMD run a system shell and .output |CMD
  opens a pipe. Block a .shell / .system / .excel dot-command and an .output/.once
  target that begins with '|'.

- getattr gadget dunders in the workdir vetter: a helper module could call
  getattr(open, '__closure__') / getattr(cell, 'cell_contents') to recover the guard
  wrapper's original unguarded open, because the getattr branch only rejected a few
  sensitive receivers. Reject a gadget-dunder name on ANY receiver (mirrors the
  direct-attribute check).

- find -exec in subprocess argv: the read scanner flattened the argv and checked
  each element independently, missing subprocess.run(['find','/etc',...,'-exec',
  'cat','{}',';']) reading /etc/passwd (the {} placeholder loses the escaping search
  root). Reconstruct a find child-exec argv into a shell string and run it through
  the read scanner, which carries the find-root + -exec logic.

Regression coverage: TestRound51Bypasses in tests/test_sandbox_tools.py (dynamic /
environb / update PATH mutations, sqlite .shell/.system/.output-pipe, find -exec
argv, plus a round51 benign-allowed set: absolute dynamic PATH, benign env vars,
local sqlite .output/.dump, workdir find -exec) and, in
tests/test_sandbox_runtime_backstop.py, the sqlite percent-encoded URI escape (with
a benign local URI) and the getattr gadget-dunder workdir-module denial.
2026-07-10 21:49:54 +00:00

11864 lines
537 KiB
Python

# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Tool definitions and executors for LLM tool calling: web search
(DuckDuckGo), Python code execution, and terminal commands."""
import ast
import http.client
import os
import signal
os.environ["UNSLOTH_IS_PRESENT"] = "1"
import asyncio
import base64
import binascii
import codecs
import random
import re
import shlex
import ssl
import subprocess
import sys
import tempfile
import threading
import urllib.request
import zlib
from core.inference.mcp_client import (
MCP_TOOL_PREFIX,
TOOL_CACHE_INVALIDATING_FIELDS,
cache_tools,
call_tool_sync,
get_cached_tools,
in_failure_cooloff,
is_stdio,
list_tools_async,
parse_server_headers,
probe_timeout,
record_probe_failure,
stdio_mcp_enabled,
)
from storage import mcp_servers_db
from loggers import get_logger
logger = get_logger(__name__)
_EXEC_TIMEOUT = 300 # 5 minutes
# Splits the UI source-map from the result; loops strip it (like __IMAGES__).
RAG_SOURCES_SENTINEL = "\n__RAG_SOURCES__:"
# Import these at module level so the preexec_fn closure triggers no imports in
# the forked child (which can deadlock multi-threaded servers).
_libc = None
if sys.platform == "linux":
try:
import ctypes
import ctypes.util
_libc_name = ctypes.util.find_library("c")
if _libc_name:
_libc = ctypes.CDLL(_libc_name, use_errno = True)
except (OSError, AttributeError):
pass
_resource = None
if sys.platform != "win32":
try:
import resource as _resource
except ImportError:
pass
# Raster-image allowlist for sandbox file serving.
# No .svg (XSS via embedded scripts), no .html, no .pdf.
_IMAGE_EXTS = frozenset({".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp"})
_MAX_OUTPUT_CHARS = 8000 # truncate long output
_BLOCKED_COMMANDS_COMMON = frozenset(
{
"rm",
"dd",
"chmod",
"chown",
"mkfs",
"mount",
"umount",
"fdisk",
"sudo",
"su",
"doas",
"pkexec",
"shutdown",
"reboot",
"halt",
"poweroff",
"kill",
"killall",
"pkill",
"passwd",
"curl",
"wget",
"nc",
"ncat",
"netcat",
"socat",
"ssh",
"scp",
"sftp",
"rsync",
"eval",
"source",
"ln",
# flock [options] <file>|<fd> <command> (or flock -c <command>) runs an arbitrary
# command in an unguarded child while holding a lock; its file/fd operand + -c forms
# make the command word hard to resolve, so block the wrapper outright.
"flock",
}
)
# Language interpreters that run inline / file / stdin code in a FRESH child process.
# The runtime filesystem backstop only patches the current interpreter, so a spawned
# `python -c '...'`, `perl -e '...'`, `node -e '...'`, etc. runs with none of the guard
# monkeypatches and can write/delete outside the session workdir. Blocking the
# interpreter at shell command position closes that child-process escape; the sandbox's
# own python_execute tool is the supported way to run Python (it IS guarded). Argument
# position (`echo python`, `ls /usr/bin/python3`) is unaffected by the command-position
# scanner.
_INTERPRETER_COMMANDS = frozenset(
{
"python",
"python2",
"python3",
"pythonw",
"perl",
"ruby",
"node",
"nodejs",
"php",
"deno",
"lua",
"luajit",
"rscript",
# awk variants run an inline program that can write files (print > "/path")
# in an unguarded child without any shell redirection token the scanner sees.
"awk",
"gawk",
"mawk",
"nawk",
}
)
# Python console-script entry points that START A FRESH, UNGUARDED Python interpreter (their
# shebang is the same interpreter whose bin dir the safe env prepends). Running a workdir file
# through one -- subprocess.run(['pytest', 'test_evil.py']) / pip install <local sdist> -- is
# the same child-process escape as a bare `python foo.py`, which is already blocked above, so
# the launcher entry points are denied for consistency. In-workdir Python belongs in the
# guarded python_execute tool. (This is deliberately tight to well-known launchers; a broader
# allowlisted-tooling relaxation is tracked separately.)
_PYTHON_LAUNCHER_COMMANDS = frozenset(
{
"pip",
"pip2",
"pip3",
"pipx",
"pytest",
"py.test",
"ipython",
"ipython3",
}
)
# Recipe / task runners that execute shell commands read from a workdir control file (a
# Makefile recipe, etc.) in an unguarded child, the same escape as the Python launchers: a
# sandboxed snippet can write a Makefile whose recipe runs `echo x > /tmp/p` and then run
# `make`. Deny the runner; in-workdir work belongs in the guarded tools. (Kept tight to the
# common ones; a broader allowlisted-tooling relaxation is tracked separately.)
_RECIPE_RUNNER_COMMANDS = frozenset({"make", "gmake"})
# File-creating / writing coreutils. Same rationale as the interpreters: a spawned child
# runs without the in-process realpath backstop, so subprocess.run(['touch', '/tmp/x']),
# tee, cp, mv, ... write / create / delete outside the session workdir. In-workdir file
# work should go through the guarded Python file APIs. (dd / ln / rm are already denied
# above.) Native / unknown binaries the sandbox cannot enumerate remain an OS-isolation
# residual.
_CHILD_WRITE_COMMANDS = frozenset(
{
"touch",
"tee",
"cp",
"mv",
"mkdir",
"install",
"truncate",
"mkfifo",
"mknod",
"shred",
"unlink",
# patch applies a diff in an unguarded child; patch -o /tmp/x writes the result outside
# the workdir, and a patch targeting ../../tmp/x escapes even without -o. Same native
# writer class as touch / cp / tar.
"patch",
# rmdir removes (empty) directories; a bash child gets no realpath guard, so
# rmdir /tmp/some-empty-dir deletes a host directory outside the workdir.
"rmdir",
# split / csplit slice a file into PREFIXaa, PREFIXab, ... at an arbitrary prefix
# path, creating files outside the workdir in an unguarded child.
"split",
"csplit",
# Archive / compression tools create files in an unguarded child (tar -cf out,
# zip out, unzip extracts, gzip file). In-workdir archiving should go through the
# guarded Python APIs.
"tar",
"zip",
"unzip",
"gzip",
"gunzip",
"bzip2",
"bunzip2",
"xz",
"unxz",
"zstd",
"7z",
"7za",
"rar",
"unrar",
"cpio",
"rsync",
# mktemp creates a file / dir at a caller-chosen template path (mktemp
# /tmp/x.XXXXXX, mktemp -d), writing outside the workdir in an unguarded child.
"mktemp",
# sponge (moreutils) soaks up stdin and writes it to a file argument
# (printf x | sponge /tmp/probe), an unguarded-child write outside the workdir.
"sponge",
}
)
_BLOCKED_COMMANDS_COMMON = (
_BLOCKED_COMMANDS_COMMON
| _INTERPRETER_COMMANDS
| _PYTHON_LAUNCHER_COMMANDS
| _RECIPE_RUNNER_COMMANDS
| _CHILD_WRITE_COMMANDS
)
_BLOCKED_COMMANDS_WIN = frozenset(
{
"rmdir",
"takeown",
"icacls",
"runas",
"powershell",
"pwsh",
}
)
# Commands that take a path as DATA but never read its CONTENTS: echo / printf print their args,
# the no-ops do nothing, and test / [ only stat. A literal sensitive path handed to one of these
# (echo /etc/passwd) is not an exfiltration, so the literal-sensitive-path scan skips it. Any
# OTHER (unknown) command word still fails closed -- only this explicit allowlist is exempt.
_SHELL_NON_READER_COMMANDS = frozenset(
{
"echo",
"printf",
":",
"true",
"false",
"test",
"[",
"[[",
}
)
_BLOCKED_COMMANDS = (
_BLOCKED_COMMANDS_COMMON | _BLOCKED_COMMANDS_WIN
if sys.platform == "win32"
else _BLOCKED_COMMANDS_COMMON
)
_SHELL_SEPARATORS = frozenset({";", "&&", "||", "|", "&", "\n", "(", ")", "`", "{", "}"})
# Bash keywords whose FOLLOWING word is a new command position: the compound-statement
# headers (if / while / until / elif run their CONDITION command) and the body markers
# (then / do / else). `if touch x; then :; fi` executes `touch` as the condition command, so
# these must reset command position -- otherwise the header word is mistaken for the command
# and the real command it precedes is skipped as an argument.
_SHELL_KEYWORDS_AS_SEP = frozenset({"then", "do", "else", "elif", "if", "while", "until", "coproc"})
# POSIX / common shell binaries. A shell without an inline `-c` payload runs unscanned
# code (a script file, -s / stdin, or a bare stdin-reading shell), so it is denied.
_SHELL_BINARIES = frozenset({"bash", "sh", "zsh", "dash", "ksh", "csh", "tcsh", "fish"})
# Utilities whose LATER argv elements are actions / write flags, not inert arguments
# (find -exec/-delete, sed -i / w, sort -o). A non-shell argv resolving to one of these is
# re-scanned as a reconstructed command line so those dangerous flags are caught.
_ARGV_TAIL_SCAN_COMMANDS = frozenset(
{"find", "sed", "gsed", "ssed", "perl", "sort", "git", "openssl", "sqlite3"}
)
# openssl option flags whose VALUE is an output file the unguarded openssl child writes (rand
# -out, req -keyout, ca -CAout / -CAserial, ...). A value that escapes the workdir writes a host
# file the realpath guard never sees; a workdir-local -out and the no-output forms stay allowed.
_OPENSSL_WRITE_FLAGS = frozenset(
{"-out", "-writerand", "-keyout", "-CAout", "-CAkeyout", "-CAserial"}
)
# sqlite3 CLI dot-commands that WRITE (or read) an arbitrary file argument in the unguarded
# child: `.output FILE` / `.once FILE` redirect query output to FILE, `.excel` / `.import` /
# `.backup FILE` / `.save FILE` / `.dump FILE` / `.clone FILE` create files, `.log FILE` writes
# a log, and `.read FILE` sources SQL from FILE. A FILE that escapes the workdir writes / reads a
# host path the realpath guard never sees. The group captures the FILE operand for a path check.
_SQLITE_DOTFILE_RE = re.compile(
r"(?m)^\s*\.(?:output|once|excel|import|backup|save|dump|clone|log|read)\b\s+(?:-{1,2}\S+\s+)*"
r"(?P<f>(?:'[^']*'|\"[^\"]*\"|\S+))"
)
# sqlite3 dot-commands that RUN a system shell command in the unguarded child: `.shell CMD` /
# `.system CMD` ("Run CMD ARGS... in a system shell"), and `.excel` (opens the result in a
# system program). These execute regardless of any path check, so match the command itself.
_SQLITE_SHELL_RE = re.compile(r"(?m)^\s*\.(?:shell|system|excel)\b")
# sqlite3 CLI options that consume a SEPARATED operand (so the value after them is NOT the
# database filename). Only -init also reads a file (its value is path-checked at the call site).
_SQLITE_OPERAND_OPTS = frozenset(
{
"-init",
"-cmd",
"-mode",
"-separator",
"-newline",
"-nullvalue",
"-lookaside",
"-mmap",
"-maxsize",
}
)
def _is_versioned_interpreter(base: str) -> bool:
"""True when ``base`` is a version-suffixed interpreter name (python3.14, python3.11,
perl5.36, ruby3.0) whose unversioned stem is a blocked interpreter. Those binaries are
commonly on the sandbox PATH and start the same unguarded child as the bare name."""
stem = re.sub(r"[0-9][0-9.]*$", "", base)
return stem != base and stem in _INTERPRETER_COMMANDS
# Absolute paths under a standard system bin dir are trusted as real system commands (their
# basename is still interpreter-checked separately); every OTHER explicit path is a local file.
_SYSTEM_BIN_PREFIXES = ("/bin/", "/usr/bin/", "/usr/local/bin/", "/sbin/", "/usr/sbin/")
def _is_local_executable_path(tok: str) -> bool:
"""True when a command word is an explicit path to a LOCAL executable file (./evil, ../x,
subdir/tool, /tmp/x). Running such a file executes whatever its shebang names in an
UNGUARDED child -- a sandboxed snippet can create + chmod ./evil with `#!/usr/bin/python3`
and run it, starting an interpreter the argv basename scan never sees. A bare command name
resolved via PATH (no slash) and an absolute system-bin path are not treated as local."""
t = tok.replace("\\", "/")
if "/" not in t:
return False
# Collapse .. before the system-bin exemption so a workdir shebang cannot masquerade as a
# trusted binary via /usr/bin/../../<workdir>/evil (normpath -> /<workdir>/evil, not exempt).
# normpath keeps the leading ./ -> bare-name collapse harmless: the "/" check above already
# ran on the original token, so ./evil (has a slash) still reaches here and stays local.
norm = os.path.normpath(t)
return not norm.startswith(_SYSTEM_BIN_PREFIXES)
def _split_path_entries(value: str):
"""Split a PATH value on ':' separators, but NOT on a ':' inside a ${...} expansion (so a
${VAR:-default} default operator is not mistaken for a list separator)."""
entries = []
cur = []
depth = 0
i = 0
v = value.replace("\\", "/")
while i < len(v):
c = v[i]
if c == "$" and i + 1 < len(v) and v[i + 1] == "{":
depth += 1
cur.append("${")
i += 2
continue
if c == "}" and depth > 0:
depth -= 1
cur.append(c)
i += 1
continue
if c == ":" and depth == 0:
entries.append("".join(cur))
cur = []
i += 1
continue
cur.append(c)
i += 1
entries.append("".join(cur))
return entries
def _path_var_resolves_unsafe(var, assignments):
"""Whether a PATH component expanded from shell variable ``var`` can resolve to the workdir.
HOME / PWD are the session workdir; PATH is the trusted search list; a var assigned a
relative / cwd value earlier in the same command (P=.; PATH=$P) is unsafe; an unknown
external var (CONDA_PREFIX) is assumed to expand to a trusted absolute path."""
if var in ("HOME", "PWD"):
return True
if var == "PATH":
return False
if assignments and var in assignments:
return _path_value_is_unsafe(assignments[var], assignments)
return False
def _path_var_is_unknown_external(var, assignments):
"""True when ``var`` is neither a workdir alias (HOME/PWD), the trusted inherited PATH, nor a
variable assigned earlier in the same command. In the sandbox such a variable is UNSET, so its
expansion is EMPTY -- not a trusted absolute path."""
return var not in ("HOME", "PWD", "PATH") and not (assignments and var in assignments)
def _path_entry_empty_expansion_unsafe(entry: str, var: str) -> bool:
"""Model an unknown/unset ``$var`` in a PATH ENTRY as EMPTY (the sandbox reality) and report
whether the entry then collapses to an empty or RELATIVE path (both search the cwd). A bare
``$EVIL`` -> ``''`` and ``${X}bin`` -> ``bin`` are unsafe; ``$CONDA_PREFIX/bin`` -> ``/bin``
stays absolute and is safe."""
blanked = re.sub(r"\$\{?" + re.escape(var) + r"\}?", "", entry)
return blanked == "" or not blanked.startswith(("/", "%"))
def _path_value_is_unsafe(value: str, assignments = None) -> bool:
"""True when a PATH search list would let a BARE (no-slash) command resolve to a workdir
executable: any entry that is ``.``, empty (``:`` = cwd), a relative directory, or one that
expands to the session workdir. In the sandbox ``HOME`` and the child cwd ARE the workdir,
so ``~`` / ``~user``, ``$HOME`` / ``$PWD``, a ``${VAR:-.}`` default that is relative, and a
``$VAR`` bound to a relative value earlier in the same command (``P=.; PATH=$P``) are unsafe.
An absolute (``/...``), ``%VAR%``, or unknown external ``$VAR`` entry (``$PATH``,
``$CONDA_PREFIX/bin``, assumed to expand to a trusted absolute path) is safe. ``assignments``
maps local shell VAR=value bindings so a locally-controlled expansion can be resolved."""
for entry in _split_path_entries(value):
e = entry.strip()
if e in ("", "."):
return True
# ~ / ~user expand to HOME, which is the session workdir in the sandbox.
if e.startswith("~"):
return True
if e.startswith("${"):
inner = e[2:]
if inner.endswith("}"):
inner = inner[:-1]
# ${VAR-def} / ${VAR:-def} / ${VAR=def} / ${VAR:=def}: def applies when VAR is unset/
# empty, so a relative default is unsafe. ${VAR:+alt} / ${VAR:?msg} carry no path.
m = re.match(r"([A-Za-z_][A-Za-z0-9_]*)(:?[-=?+])(.*)$", inner)
if m:
var, op, default = m.group(1), m.group(2), m.group(3)
if op in (":-", "-", ":=", "=") and _path_value_is_unsafe(default, assignments):
return True
if _path_var_resolves_unsafe(var, assignments):
return True
continue
var = re.split(r"[/}]", inner, maxsplit = 1)[0]
if _path_var_resolves_unsafe(var, assignments):
return True
if _path_var_is_unknown_external(
var, assignments
) and _path_entry_empty_expansion_unsafe(e, var):
return True
continue
if e.startswith("$"):
m = re.match(r"\$([A-Za-z_][A-Za-z0-9_]*)", e)
if m and _path_var_resolves_unsafe(m.group(1), assignments):
return True
# An unknown/unset $VAR expands to EMPTY in the sandbox, so a bare `$EVIL` (or one that
# leaves a relative remainder, `${X}bin`) collapses the entry to the cwd; only an entry
# that stays ABSOLUTE with the var blanked ($CONDA_PREFIX/bin -> /bin) is trusted.
if (
m
and _path_var_is_unknown_external(m.group(1), assignments)
and _path_entry_empty_expansion_unsafe(e, m.group(1))
):
return True
continue # $PATH / $CONDA_PREFIX/bin / $1: a trusted absolute expansion
if e.startswith(("/", "%")):
continue
return True # a relative directory (relbin, ./tools)
return False
def _dynamic_path_value_unsafe(value_node, env) -> bool:
"""A NON-literal PATH assignment value (os.environ['PATH'] = '.:' + os.environ['PATH'],
f'.:{x}') is unsafe when a COMPLETE, fully-literal PATH entry it contributes is a relative /
cwd / empty entry. Operands are const-folded; an OPAQUE segment (os.environ['PATH'], a
variable) taints only the entry that spans it, so a dynamic ABSOLUTE extension
(venv + ':' + $PATH, '/usr/local/bin:' + $PATH) stays allowed. Returns True only for a
provable unsafe entry -- the folded literal case is handled by the caller."""
segments: list = [] # ("lit", str) or ("opaque",)
def _flatten(n):
folded = _const_fold(n, env)
if isinstance(folded, str):
segments.append(("lit", folded))
return
if isinstance(n, ast.BinOp) and isinstance(n.op, ast.Add):
_flatten(n.left)
_flatten(n.right)
return
if isinstance(n, ast.JoinedStr):
for _p in n.values:
if isinstance(_p, ast.Constant) and isinstance(_p.value, str):
segments.append(("lit", _p.value))
else:
_fv = _const_fold(getattr(_p, "value", _p), env)
segments.append(("lit", _fv) if isinstance(_fv, str) else ("opaque",))
return
segments.append(("opaque",))
_flatten(value_node)
entries: list = [] # (text, complete, tainted)
cur = ""
tainted = False
for seg in segments:
if seg[0] == "opaque":
tainted = True
continue
parts = seg[1].split(":")
for j, part in enumerate(parts):
if j == 0:
cur += part
else:
entries.append((cur, True, tainted))
cur = part
tainted = False
_last_opaque = bool(segments) and segments[-1][0] == "opaque"
entries.append((cur, not _last_opaque, tainted))
for text, complete, taint in entries:
if complete and not taint and _path_value_is_unsafe(text):
return True
return False
def _arg_escapes_workdir(tok: str) -> bool:
"""True when a path-like argument can point OUTSIDE the session workdir: an absolute path
(``/tmp/x``), a ``~`` / ``~user`` home path (home == workdir, but a shell child follows the
real HOME), or any path with a ``..`` component that can traverse above the workdir. A
workdir-relative name (``sub/out``, ``repo``) stays inside and returns False. Used to confine
file-creating child commands (git init/clone <dir>, ...) that the runtime guard cannot see."""
t = tok.replace("\\", "/")
if t.startswith("/") or t.startswith("~"):
return True
return ".." in t.split("/")
def _git_operand_escapes(tok: str, assigns = None) -> bool:
"""As _arg_escapes_workdir, but resolves a ``$VAR`` / ``${VAR}`` bound to an escaping value
earlier in the SAME command, as the WHOLE token (``OUT=/tmp/repo; git init $OUT``) OR as a
PREFIX (``P=/tmp; git init $P/repo``, ``openssl rand -out $P/key``). An unknown external
expansion is left to the literal check (so ``git clone $REPO_URL`` is not a false positive)."""
m = re.match(r"\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?(.*)$", tok)
if m and assigns and m.group(1) in assigns:
return _arg_escapes_workdir(assigns[m.group(1)] + m.group(2))
return _arg_escapes_workdir(tok)
# git options whose VALUE is a path that a native git child writes to / operates in (the runtime
# realpath backstop never sees a native git process). A value that escapes the workdir lets git
# write outside the session: -C / --git-dir / --work-tree / --separate-git-dir (repo location),
# and -o / --output / -O / --output-directory (git archive / format-patch write their output
# file there). Handled for `-x val`, `--opt val`, and inline `--opt=val` forms.
_GIT_PATH_VALUE_OPTIONS = frozenset(
{
"-C",
"--git-dir",
"--work-tree",
"--separate-git-dir",
"-o",
"--output",
"-O",
"--output-directory",
}
)
# git config keys whose value is a COMMAND git runs in an unguarded child (git -c KEY=CMD ... /
# git config KEY CMD). core.fsmonitor / sshCommand / pager / editor / credential.helper /
# diff.external / gpg.program / sequence.editor / uploadpack.packObjectsHook run their value;
# core.hooksPath / init.templateDir re-point hooks (undoing the sandbox hook suppression).
_GIT_EXEC_CONFIG_KEYS = frozenset(
{
"core.fsmonitor",
"core.sshcommand",
"core.pager",
"core.editor",
"core.hookspath",
"core.askpass",
"sequence.editor",
"diff.external",
"gpg.program",
"credential.helper",
"init.templatedir",
"uploadpack.packobjectshook",
"ssh.variant",
}
)
def _git_config_key_is_exec(key: str) -> bool:
"""True for a git config key whose value git executes as a command (or that re-points hooks)."""
k = key.strip().lower()
if k in _GIT_EXEC_CONFIG_KEYS:
return True
# include.path / includeIf.<cond>.path pull in another config file whose contents git then
# honors, so an included workdir config can set core.hooksPath / core.fsmonitor (re-enabling a
# planted hook) even though the direct key is blocked. Treat any include*.path key as exec.
if k == "include.path" or (k.startswith("includeif.") and k.endswith(".path")):
return True
# filter.<name>.clean/smudge/process, diff.<name>.command, merge.<name>.driver take commands.
parts = k.split(".")
if len(parts) == 3:
section, _, leaf = parts
if section == "filter" and leaf in ("clean", "smudge", "process"):
return True
if section == "diff" and leaf == "command":
return True
if section == "merge" and leaf == "driver":
return True
return False
# The only shell redirection targets trusted without a realpath check: standard device
# sinks that cannot escape the workdir. Every other target (relative or absolute) fails
# closed, because the unguarded child follows symlinks and resolves relative names against a
# cwd the static scanner cannot verify (a pre-existing `out -> /tmp/host` symlink escapes).
_SAFE_REDIRECT_TARGETS = frozenset(
{"/dev/null", "/dev/zero", "/dev/full", "/dev/stdout", "/dev/stderr", "/dev/tty"}
)
# Coreutils that read + print file contents. A shell-expanded ($VAR / `cmd`) path passed
# to one of these can exfiltrate a host secret whose name the static scan cannot resolve.
_SHELL_READ_COMMANDS = frozenset(
{
"cat",
"head",
"tail",
"less",
"more",
"od",
"xxd",
"hexdump",
"strings",
"nl",
"tac",
"cut",
"sort",
"uniq",
"wc",
"base64",
"base32",
"sed",
"grep",
"egrep",
"fgrep",
"rev",
"fold",
"paste",
"comm",
"tr",
"dd",
"readlink",
"realpath",
# diff-style utilities print file contents in their output: `diff SECRET /dev/null`
# (or `cmp -l SECRET /dev/null`) leaks the file line-by-line / byte-by-byte, so a
# shell-expanded ($VAR / glob / `cmd`) path handed to one exfiltrates a host secret.
"diff",
"sdiff",
"diff3",
"colordiff",
"cmp",
# directory / file enumerators: an EXPANDED root (find ${P:-/root/.ssh} -exec cat {} \;,
# ls $SECRET) enumerates a host path the static scan cannot resolve, and find's -exec
# can then read every match. Literal find / ls (find . -name '*.py', ls -la) carry no
# expansion and stay allowed; only a $ / backtick / escaping-glob operand fails closed.
"find",
"ls",
# openssl can READ + print a file's contents (openssl base64 -in SECRET, openssl enc -d
# -in SECRET, openssl x509 -in SECRET), so an EXPANDED / sensitive -in path exfiltrates a
# host secret. Literal in-workdir input (openssl base64 -in data.txt) carries no expansion
# and stays allowed; only a $ / backtick / escaping-glob / sensitive operand fails closed.
"openssl",
}
)
# Wrappers whose next non-flag argument is the command Bash will exec.
_COMMAND_PREFIXES = frozenset(
{
"env",
"command",
"builtin",
"exec",
"time",
"nohup",
"nice",
"setsid",
"stdbuf",
"timeout",
"ionice",
"chroot",
"sudo",
"doas",
"su",
"xargs",
# chrt [options] <priority> <command> [<arg>...]: util-linux scheduler wrapper that
# execs the following command, so chrt -o 0 touch /tmp/x must resolve to touch.
"chrt",
# watch [options] command: repeatedly runs command (via sh -c, or exec with -x), so
# watch -x touch /tmp/x / watch -n 2 rm -rf / must resolve to the wrapped command.
"watch",
}
)
# A shell assignment prefix: NAME=value or NAME+=value (bash append). The optional `+` is part
# of the operator, so `PATH+=:. cmd` is recognized as an assignment prefix, not a command word.
_ASSIGNMENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*\+?=")
# Per-wrapper option flags that take a SEPARATED operand (the NEXT token is the flag's value,
# not the command). Anything not listed -- a no-operand flag (env -i, xargs -0), a GLUED short
# flag (stdbuf -oL), or a --long=value -- does NOT consume the next token, so the real command
# after it is still analysed. Wrappers absent from the map default to no operand-taking flags
# (their numeric args, nice -n 5 / timeout 5, are skipped separately).
_WRAPPER_OPERAND_FLAGS = {
"env": frozenset({"-u", "--unset", "-C", "--chdir"}),
"nice": frozenset({"-n", "--adjustment"}),
"timeout": frozenset({"-s", "--signal", "-k", "--kill-after"}),
"stdbuf": frozenset({"-i", "--input", "-o", "--output", "-e", "--error"}),
"ionice": frozenset({"-c", "--class", "-n", "--classdata", "-p", "--pid"}),
"sudo": frozenset(
{
"-u",
"--user",
"-g",
"--group",
"-C",
"--close-from",
"-h",
"--host",
"-p",
"--prompt",
"-r",
"--role",
"-t",
"--type",
"-U",
"--other-user",
"-T",
"--command-timeout",
"-R",
"--chroot",
"-D",
"--chdir",
}
),
"xargs": frozenset(
{
"-n",
"--max-args",
"-P",
"--max-procs",
"-L",
"--max-lines",
"-s",
"--max-chars",
"-I",
"--replace",
"-E",
"-d",
"--delimiter",
"-a",
"--arg-file",
# --process-slot-var VAR sets an env var for the child; the separated operand VAR
# would otherwise be mistaken for the command word (xargs --process-slot-var V touch).
"--process-slot-var",
}
),
"time": frozenset({"-f", "--format", "-o", "--output"}),
"chrt": frozenset({"-T", "--sched-runtime", "-P", "--sched-period", "-D", "--sched-deadline"}),
"watch": frozenset({"-n", "--interval"}),
}
def _wrapper_flag_takes_operand(wrapper, flag: str) -> bool:
"""True when a wrapper option FLAG consumes the NEXT token as a separated operand
(env -u NAME, nice -n 5, stdbuf -o L). A glued short flag (-oL), a --long=value, or any
flag not listed for the wrapper does NOT, so the command word after it is still analysed
(stdbuf -oL sed -i ..., xargs -0 sed ...)."""
if "=" in flag:
return False
if not flag.startswith("--") and len(flag) > 2:
return False # glued short flag: -oL already carries its value
return flag in _WRAPPER_OPERAND_FLAGS.get(wrapper, frozenset())
# GNU sed can WRITE files (`w FILE`, `W FILE`, `s///w FILE`) or EXECUTE shell commands
# (`e COMMAND`, `s///e`) straight from its SCRIPT even without -i, escaping the workdir in an
# unguarded child. The filename/command may follow immediately (GNU accepts `w/tmp/x`) or
# after whitespace. A plain `s/word/x/` has `w`/`e` inside the pattern/replacement (a letter or
# closing delimiter follows), so these patterns are shaped to skip that.
_SED_WRITE_RE = re.compile(r"(?<![A-Za-z])[wW](?:[ \t]|/|~)")
_SED_EXEC_RE = re.compile(r"(?:^|[;\n{}]|[0-9$])[ \t]*e(?:[ \t;}\n]|$)")
# GNU sed runs `e COMMAND` after an ADDRESS too (`/regex/e cmd`, `/a/,/b/e cmd`, `1,/x/e cmd`).
# Match a `/regex/` that sits at a command boundary (start / `;` / `{` / `}`) OR right after a
# range comma, followed by an optional `!` and the `e` command with a trailing separator. The `e`
# must be followed by whitespace / `;` / `}` / EOL, so a substitution replacement that merely ends
# in `.../e/` (e before the closing delimiter) does not match; and requiring a boundary before the
# opening `/` means `s/a/e /` (its `/` preceded by `s`) is not mistaken for an address. A
# line-number address then `e` (`1e`, `1,/x/` handled via the comma) is already covered above.
_SED_ADDR_EXEC_RE = re.compile(
r"(?:^|[;\n{},])[ \t]*!?[ \t]*/(?:[^/\\]|\\.)*/[ \t]*!?[ \t]*e(?:[ \t;}\n]|$)"
)
# A completed s/PATTERN/REPL/FLAGS whose FLAGS include w (write) or e (execute).
_SED_SFLAG_RE = re.compile(r"s(.)(?:(?!\1).)*\1(?:(?!\1).)*\1[A-Za-z0-9]*[we]")
_FIND_EXEC_FLAGS = frozenset({"-exec", "-execdir", "-ok", "-okdir"})
# find's -exec / -ok command runs up to a `;` or `+` terminator; everything between
# is a full command line (may itself begin with a wrapper like env/timeout or sh -c).
_FIND_EXEC_TERMINATORS = frozenset({";", "+"})
def _is_wrapper_numeric_arg(token: str) -> bool:
"""A wrapper's numeric argument (`nice -n 5`, `timeout 5m`, `timeout 0.5`).
Accepts a plain int/float, optionally with a single trailing GNU ``timeout``
duration unit (s/m/h/d). Used only to decide whether to skip a token while a
command-prefix wrapper is still awaiting its real command, so being permissive
keeps the scan on the following command rather than dropping out of command
position.
"""
t = token.lstrip("-")
if not t:
return False
if len(t) > 1 and t[-1] in "smhd":
t = t[:-1]
try:
float(t)
return True
except ValueError:
return False
_ANSI_C_ESCAPES = {
"a": "\a",
"b": "\b",
"e": "\x1b",
"E": "\x1b",
"f": "\f",
"n": "\n",
"r": "\r",
"t": "\t",
"v": "\v",
"\\": "\\",
"'": "'",
'"': '"',
"?": "?",
}
def _decode_ansi_c(body: str) -> str:
"""Decode the escape sequences bash resolves inside a $'...' word (\\n, \\t, \\xHH,
octal \\NNN, \\uHHHH, ...) so the resulting command word matches what actually runs."""
out = []
i, n = 0, len(body)
while i < n:
c = body[i]
if c != "\\" or i + 1 >= n:
out.append(c)
i += 1
continue
d = body[i + 1]
if d in _ANSI_C_ESCAPES:
out.append(_ANSI_C_ESCAPES[d])
i += 2
elif d == "x":
j, h = i + 2, ""
while j < n and len(h) < 2 and body[j] in "0123456789abcdefABCDEF":
h += body[j]
j += 1
if h:
out.append(chr(int(h, 16)))
i = j
else:
out.append(c)
out.append(d)
i += 2
elif d in "01234567":
j, o = i + 1, ""
while j < n and len(o) < 3 and body[j] in "01234567":
o += body[j]
j += 1
out.append(chr(int(o, 8) & 0xFF))
i = j
elif d in ("u", "U"):
width = 4 if d == "u" else 8
j, h = i + 2, ""
while j < n and len(h) < width and body[j] in "0123456789abcdefABCDEF":
h += body[j]
j += 1
if h:
out.append(chr(int(h, 16)))
i = j
else:
out.append(c)
out.append(d)
i += 2
else:
out.append(c)
out.append(d)
i += 2
return "".join(out)
def _normalize_ansi_c_quotes(command: str) -> str:
"""Rewrite bash ANSI-C ($'...') and locale ($"...") quoted words to plain quoted words
so shlex sees the token bash actually executes. shlex leaves `$'touch'` as the literal
`$touch`, so a writer/interpreter hidden behind ANSI-C quoting (`$'touch' x`,
`$'\\x74ouch' x`) never matches the command blocklist otherwise."""
if "$'" not in command and '$"' not in command:
return command
res = []
i, n = 0, len(command)
while i < n:
if command[i] == "$" and i + 1 < n and command[i + 1] == '"':
res.append('"') # locale translation: bash just strips the leading $
i += 2
continue
if command[i] == "$" and i + 1 < n and command[i + 1] == "'":
j, buf = i + 2, []
while j < n:
if command[j] == "\\" and j + 1 < n:
buf.append(command[j])
buf.append(command[j + 1])
j += 2
continue
if command[j] == "'":
break
buf.append(command[j])
j += 1
decoded = _decode_ansi_c("".join(buf))
# Re-emit as a single-quoted shlex token (escaping embedded single quotes).
res.append("'" + decoded.replace("'", "'\\''") + "'")
i = j + 1 # skip the closing quote
continue
res.append(command[i])
i += 1
return "".join(res)
_IFS_RE = re.compile(r"\$\{IFS[^}]*\}|\$IFS\b")
def _expand_ifs(command: str) -> str:
"""bash expands ${IFS} / $IFS to whitespace (default space/tab/newline) BEFORE word
splitting, so cat${IFS}/etc/shadow runs `cat /etc/shadow` in the child. Replace an IFS
reference with a space so the scanner tokenizes the command bash actually executes."""
if "IFS" not in command:
return command
return _IFS_RE.sub(" ", command)
def _rewrite_unquoted_newlines(command: str) -> str:
"""Rewrite only UNQUOTED newline runs to ` ; ` (a bash command separator). A newline INSIDE
quotes is data (echo "ok\\nrm" is one argument), so a blanket regex would split a quoted
multiline string into a spurious command position and mis-block the later line."""
out = []
q = None
esc = False
prev_nl = False
for ch in command:
if esc:
if ch in ("\n", "\r"):
# A backslash immediately before a newline is a bash LINE CONTINUATION: both are
# removed before command lookup, so `tou\<nl>ch` runs `touch`. Drop the backslash
# we already emitted and the newline so the joined word is tokenized (outside
# single quotes; single-quoted text never sets esc, so it stays literal).
if out and out[-1] == "\\":
out.pop()
esc = False
prev_nl = False
continue
out.append(ch)
esc = False
prev_nl = False
continue
if q == "'":
out.append(ch)
if ch == "'":
q = None
prev_nl = False
continue
if q == '"':
out.append(ch)
if ch == "\\":
esc = True
elif ch == '"':
q = None
prev_nl = False
continue
if ch == "\\":
out.append(ch)
esc = True
prev_nl = False
continue
if ch in ("'", '"'):
out.append(ch)
q = ch
prev_nl = False
continue
if ch in ("\r", "\n"):
if not prev_nl:
out.append(" ; ")
prev_nl = True
continue
out.append(ch)
prev_nl = False
return "".join(out)
def _mask_quoted_separators(command: str) -> str:
"""Neutralize command-boundary characters that are DATA inside quotes (blank them to a
space) so the regex command-position scan does not treat a quoted separator -- echo
"ok\\nrm" or 'a;rm' -- as a fresh command word. Command substitution ($(...) / backticks)
still runs inside DOUBLE quotes, so those are preserved; single-quoted text is fully
literal. The result is used only for the boundary regex, not for tokenization."""
out = []
q = None
esc = False
i = 0
n = len(command)
while i < n:
ch = command[i]
if esc:
out.append(ch)
esc = False
i += 1
continue
if q == "'":
out.append(" " if ch in ";&|(\n\r`$" else ch)
if ch == "'":
q = None
i += 1
continue
if q == '"':
if ch == "\\":
out.append(ch)
esc = True
i += 1
continue
if ch == '"':
out.append(ch)
q = None
i += 1
continue
if ch == "$" and i + 1 < n and command[i + 1] == "(":
out.append("$(") # command substitution runs inside double quotes; keep it
i += 2
continue
if ch == "`":
out.append("`")
i += 1
continue
out.append(" " if ch in ";&|(\n\r" else ch)
i += 1
continue
if ch == "\\":
out.append(ch)
esc = True
i += 1
continue
if ch in ("'", '"'):
out.append(ch)
q = ch
i += 1
continue
out.append(ch)
i += 1
return "".join(out)
def _iter_unquoted_chars(s):
"""Yield (index, char) for every character OUTSIDE single / double quotes (a backslash
escape and the char it escapes are skipped inside double quotes / unquoted text). Used to
locate brace-expansion syntax that bash would act on, ignoring quoted braces."""
q = None
esc = False
for i, ch in enumerate(s):
if esc:
esc = False
continue
if q == "'":
if ch == "'":
q = None
continue
if q == '"':
if ch == "\\":
esc = True
elif ch == '"':
q = None
continue
if ch == "\\":
esc = True
yield i, ch
continue
if ch in ("'", '"'):
q = ch
continue
yield i, ch
def _brace_first_comma_group(s):
"""Return (open, close) indices of the first UNQUOTED ``{...}`` that contains a top-level
comma (the shape bash expands), else None. ``{}`` / ``${x}`` / ``{1..5}`` have no top-level
comma and are left untouched, as are quoted braces."""
idxset = {i: ch for i, ch in _iter_unquoted_chars(s)}
for o, ch in list(idxset.items()):
if ch != "{":
continue
depth = 0
has_comma = False
for i in range(o, len(s)):
c = idxset.get(i)
if c is None:
continue
if c == "{":
depth += 1
elif c == "}":
depth -= 1
if depth == 0:
if has_comma:
return o, i
break
elif c == "," and depth == 1:
has_comma = True
return None
def _brace_split_top_commas(content):
"""Split a brace group's inner text on top-level (unnested, unquoted) commas."""
parts = []
cur = []
depth = 0
q = None
esc = False
for ch in content:
if esc:
cur.append(ch)
esc = False
continue
if q == "'":
cur.append(ch)
if ch == "'":
q = None
continue
if q == '"':
cur.append(ch)
if ch == "\\":
esc = True
elif ch == '"':
q = None
continue
if ch == "\\":
cur.append(ch)
esc = True
continue
if ch in ("'", '"'):
cur.append(ch)
q = ch
continue
if ch == "{":
depth += 1
cur.append(ch)
continue
if ch == "}":
depth -= 1
cur.append(ch)
continue
if ch == "," and depth == 0:
parts.append("".join(cur))
cur = []
continue
cur.append(ch)
parts.append("".join(cur))
return parts
def _brace_expand_word(word, budget):
"""Recursively expand a single word's comma brace groups (bash-style, quote-aware,
cartesian across multiple groups), returning the list of expansions. Bounded by budget."""
grp = _brace_first_comma_group(word)
if grp is None:
return [word]
o, c = grp
pre, content, post = word[:o], word[o + 1 : c], word[c + 1 :]
out = []
for opt in _brace_split_top_commas(content):
for opt_exp in _brace_expand_word(opt, budget):
for post_exp in _brace_expand_word(post, budget):
out.append(pre + opt_exp + post_exp)
if len(out) >= budget[0]:
return out
return out
def _split_words_unquoted_ws(s):
"""Split ``s`` into words on UNQUOTED space / tab; emit an unquoted newline as its own
token so it survives as a command separator. Quotes and their contents stay intact."""
words = []
cur = []
q = None
esc = False
for ch in s:
if esc:
cur.append(ch)
esc = False
continue
if q == "'":
cur.append(ch)
if ch == "'":
q = None
continue
if q == '"':
cur.append(ch)
if ch == "\\":
esc = True
elif ch == '"':
q = None
continue
if ch == "\\":
cur.append(ch)
esc = True
continue
if ch in ("'", '"'):
cur.append(ch)
q = ch
continue
if ch == "\n":
if cur:
words.append("".join(cur))
cur = []
words.append("\n")
continue
if ch in " \t":
if cur:
words.append("".join(cur))
cur = []
continue
cur.append(ch)
if cur:
words.append("".join(cur))
return words
def _expand_braces(command: str) -> str:
"""Model bash brace expansion (comma lists) before the block / read scans so a payload such
as ``{touch,/tmp/escape}`` or ``{python3,-c} '...'`` is seen as the writer / interpreter bash
would actually run, instead of a single opaque ``{...}`` token. Only unquoted groups with a
top-level comma are expanded; ``{}`` (find -exec), ``${VAR}`` parameter expansion, numeric
``{1..5}`` sequences and quoted braces are left intact. Expansion is bounded to avoid blowup;
if the bound is hit the (partial) expansion is still scanned."""
if "{" not in command:
return command
budget = [4096]
out = []
for w in _split_words_unquoted_ws(command):
if "{" in w and "}" in w and "," in w:
out.extend(_brace_expand_word(w, budget))
else:
out.append(w)
if len(out) >= 8192:
break
return " ".join(out)
def _find_blocked_commands(command: str) -> set[str]:
"""Detect blocked commands at shell command position only.
A token is at command position if it is the first token, or follows a
shell separator / brace-group opener / new-command keyword (`then`, `do`,
etc.), or a command-prefix wrapper like `env` / `time` / `xargs` (next
token is the real command). Tokens in argument position (`grep -r curl .`,
`echo source the data`, `ls /usr/bin/curl`) pass through. Also scans
`find ... -exec CMD` and recurses into bash -c / cmd /c.
"""
blocked: set[str] = set()
# Normalize bash ANSI-C ($'...') / locale ($"...") quoting first: shlex leaves
# `$'touch'` as `$touch`, so a writer/interpreter hidden behind ANSI-C quoting would
# never match the blocklist even though bash decodes and runs it. Then expand ${IFS} to
# whitespace so a separator-obfuscated command (rm${IFS}-rf${IFS}/) is tokenized.
command = _expand_ifs(_normalize_ansi_c_quotes(command))
# bash treats an unquoted newline as a command separator, but shlex's whitespace_split
# folds it into ordinary whitespace, so `echo ok\nsed -i ...` would read `sed` as an
# argument of `echo` and miss the write. Rewrite UNQUOTED newlines to `;` so each line
# starts a fresh command position; a newline inside quotes stays data (echo "ok\nrm" is one
# argument), so it is not turned into a spurious `; rm` command position.
command = _rewrite_unquoted_newlines(command)
# bash performs brace expansion before command lookup, so `{touch,/tmp/x}` /
# `{python3,-c} '...'` run the writer / interpreter even though the raw string has no
# blocked token. Expand comma brace groups so the produced command words are scanned.
command = _expand_braces(command)
# punctuation_chars splits separators into their own tokens, so command
# position is detected even in `echo done; rm -rf x` (no whitespace) or
# quote-split names (`r''m` collapses to `rm` after `;`). Including `<` splits an INPUT
# redirect / here-string glued to the command word (sh<<<'...', cat</etc/passwd) so the
# shell / reader is still recognized. (`>` is left out so the regex-based output-redirect
# scan keeps seeing `>&` as one operator.)
try:
if sys.platform == "win32":
tokens = shlex.split(command, posix = False)
else:
lexer = shlex.shlex(command, posix = True, punctuation_chars = ";&|()`<")
lexer.whitespace_split = True
tokens = list(lexer)
except ValueError:
tokens = command.split()
def _token_basename(tok: str) -> str:
# Strip glued-on meta-chars (`rm;`) so the basename still matches `rm`.
tok = tok.strip(";&|()`{}")
base = os.path.basename(tok).lower()
stem, ext = os.path.splitext(base)
if ext in {".exe", ".com", ".bat", ".cmd"}:
base = stem
return base
expect_command = True # start of string is a command position
prefix_pending = False # last cmd-position token was a wrapper (env/time/xargs/...)
prev_was_flag = False # previous token (while a wrapper is pending) takes an operand
cur_wrapper = None # the active wrapper's basename (drives per-wrapper option arity)
for token in tokens:
if token in _SHELL_SEPARATORS:
expect_command = True
prefix_pending = False
prev_was_flag = False
cur_wrapper = None
continue
if token in _SHELL_KEYWORDS_AS_SEP:
# if / while / until / then / do / else / elif start a NEW command position ONLY when
# they appear at command position (the compound-statement header: `if touch x; then`).
# After a command word they are ordinary arguments -- bash does not run the next word as
# a command in `echo if touch`, so only reset there. Real separators (; | && ...) above
# always reset regardless of position.
if expect_command:
prefix_pending = False
prev_was_flag = False
cur_wrapper = None
continue
if token.startswith("-"):
# Flags belong to the active command, but keep expect_command while a
# wrapper prefix awaits its command. Only a flag that actually takes a SEPARATED
# operand (env -u NAME) marks the next token as its value; a glued / no-operand
# flag (stdbuf -oL sed, xargs -0 sed) does not, so the command that follows is
# still analysed.
if not prefix_pending:
expect_command = False
elif _wrapper_flag_takes_operand(cur_wrapper, token):
prev_was_flag = True
continue
if not expect_command:
continue
# A leading `!` negates the pipeline exit status, but the following word is still the
# command bash executes (`! touch x`, `! python3 -c ...`). Keep command position so the
# real command is scanned, rather than mistaking `!` for the command and its command for
# an argument.
if token == "!":
continue
# FOO=bar assignment prefix; next non-assignment token is the command.
if _ASSIGNMENT_RE.match(token):
continue
# Numeric wrapper arg: `timeout 1 cmd` / `nice -n 5 cmd`, plus GNU `timeout`
# duration forms (`5m`, `0.5`, `2h`). Skipping it keeps prefix_pending so the
# real command that follows is still analysed at command position; over-
# accepting a numeric-looking token is safe (we only skip, never stop scanning),
# whereas the old int-only check let `timeout 5m rm -rf /` slip through.
if prefix_pending and _is_wrapper_numeric_arg(token):
prev_was_flag = False
continue
base = _token_basename(token)
# A wrapper's separated option ARGUMENT (`stdbuf -o L cmd`, `ionice -c 2 cmd`):
# an operand right after a wrapper flag that is NOT itself a blocked command /
# prefix is the flag's value, so skip it and keep scanning for the real command
# instead of mistaking it for the command and stopping. If it IS a blocked
# command / prefix it is treated as the command below (never miss `env -i rm`).
if (
prefix_pending
and prev_was_flag
and base not in _BLOCKED_COMMANDS
and base not in _COMMAND_PREFIXES
):
prev_was_flag = False
continue
prev_was_flag = False
# An expansion sitting AT the resolved command word -- behind a wrapper
# (env $CMD -c ...) or after a leading assignment -- runs whatever it expands to as
# the command name and cannot be proven safe, so fail closed. The separator-anchored
# regex below misses the wrapper case because $CMD is not right after a separator.
if "$" in token or "`" in token:
blocked.add("command-expansion")
# Glob metacharacters in a command NAME (/bin/s?, touc?, /bin/[bd]ash) are expanded by
# the shell to a matching path BEFORE command lookup, so the literal basename compared
# against the blocklist (s?, touc?) never matches the shell / writer it resolves to.
# The resolved binary cannot be proven safe, so fail closed. A bare `[` is the test
# builtin (not a glob), so exclude it.
if "*" in token or "?" in token or (token != "[" and "[" in token):
blocked.add("command-glob")
if base in _BLOCKED_COMMANDS or _is_versioned_interpreter(base):
blocked.add(base)
# The `.` builtin is bash's `source`: `. evil.sh` runs an unscanned script in the
# shell, the same escape as `source`, but its basename is not a blocklist word.
if base == ".":
blocked.add("source")
# An explicit path to a LOCAL executable at command position (./evil, subdir/tool) runs
# whatever its shebang names in an unguarded child, so treat it like a blocked command.
if _is_local_executable_path(token):
blocked.add("local-exec:" + base)
# Wrappers (env/time/xargs/sudo) consume one command; the next non-flag,
# non-numeric token is the real command. sudo is also in _BLOCKED_COMMANDS.
if base in _COMMAND_PREFIXES:
prefix_pending = True
cur_wrapper = base
continue
expect_command = False
prefix_pending = False
cur_wrapper = None
# `find ... -exec CMD ... ;` / `-execdir CMD ... +` invoke CMD directly. CMD may
# itself be a wrapper (`env rm`, `timeout 5 rm`) or a nested shell (`sh -c '...'`),
# so rescan the whole slice up to the `;`/`+` terminator through the full command-
# position analyzer instead of only basename-matching the immediate next token.
for i, tok in enumerate(tokens):
if tok in _FIND_EXEC_FLAGS:
seg = []
j = i + 1
while j < len(tokens) and tokens[j] not in _FIND_EXEC_TERMINATORS:
seg.append(tokens[j])
j += 1
if seg:
blocked |= _find_blocked_commands(" ".join(seg))
# Regex catches blocked words at command boundaries shlex misses: inside
# $(rm -rf), <(rm), backtick chains, or "foo;rm". Anchored to command-position
# delimiters, so it doesn't match in argument position. Quoted separators are neutralized
# first so a quoted multiline string (echo "ok\nrm") is not read as a command boundary.
lowered = _mask_quoted_separators(command).lower()
if _BLOCKED_COMMANDS:
words_alt = "|".join(re.escape(w) for w in sorted(_BLOCKED_COMMANDS))
pattern = (
rf"(?:^|[;&|`\n(]\s*|[$]\(\s*|<\(\s*)"
rf"(?:[\w./\\-]*/|[a-zA-Z]:[/\\][\w./\\-]*)?"
rf"({words_alt})(?:\.(?:exe|com|bat|cmd))?\b"
)
blocked.update(re.findall(pattern, lowered))
# Nested shell invocations (bash -c '...', bash -lc '...', cmd /c '...'):
# on a -c/-/c flag, look back for a shell name (skipping flags) and
# recursively scan the nested command string.
_SHELLS = _SHELL_BINARIES
_SHELLS_WIN = {"cmd", "cmd.exe"}
for i, token in enumerate(tokens):
tok_lower = token.lower()
# Match -c exactly, or combined flags ending in c (e.g. -lc, -xc)
is_unix_c = tok_lower == "-c" or (
tok_lower.startswith("-") and tok_lower.endswith("c") and not tok_lower.startswith("--")
)
is_win_c = tok_lower == "/c"
if not (is_unix_c or is_win_c) or i < 1 or i + 1 >= len(tokens):
continue
# Look back past flags for the shell binary. Windows flags and absolute
# paths both start with /, so only skip short /X flags (not /bin/bash).
for j in range(i - 1, -1, -1):
prev = tokens[j]
if prev.startswith("-"):
continue # skip Unix flags like --login, -l
if is_win_c and prev.startswith("/") and len(prev) <= 3:
continue # skip Windows flags like /s, /q (not /bin/bash)
prev_base = os.path.basename(prev).lower()
if is_unix_c and prev_base in _SHELLS:
blocked |= _find_blocked_commands(tokens[i + 1])
elif is_win_c and prev_base in _SHELLS_WIN:
blocked |= _find_blocked_commands(tokens[i + 1])
break # stop at first non-flag token
# `env -S 'cmd ...'` / `env --split-string='cmd'` splits the string and runs it as a
# fresh command, so a bare `env -S` operand is NOT just a flag value -- recurse into
# it (it can invoke an unguarded interpreter or another blocked command).
for i, token in enumerate(tokens):
tl = token.lower()
payload = None
if tl in ("-s", "--split-string") and i + 1 < len(tokens):
payload = tokens[i + 1]
elif tl.startswith("-s") and tl != "-s" and not tl.startswith("--"):
payload = token[2:] # glued short form: env -S'cmd' / -Scmd
elif tl.startswith("--split-string="):
payload = token[len("--split-string=") :]
if not payload:
continue
for j in range(i - 1, -1, -1):
prev = tokens[j]
if prev.startswith("-"):
continue
if os.path.basename(prev).lower() == "env":
blocked |= _find_blocked_commands(payload)
break
def _command_word_indices():
# Indices of the REAL command word at each command position, skipping FOO=bar
# assignments and wrapper prefixes (env / nice / timeout / xargs / ...) plus their
# numeric / separated-option arguments, so `env sed`, `timeout 5 bash` resolve to
# sed / bash. Mirrors the main command-position scan above.
out = []
expect = True
pending = False
prev_flag = False
wrapper = None
for _i, _tok in enumerate(tokens):
if _tok in _SHELL_SEPARATORS:
expect = True
pending = False
prev_flag = False
wrapper = None
continue
if _tok in _SHELL_KEYWORDS_AS_SEP:
# if / while / until / then / do (etc.) begin a new command position ONLY at
# command position (the compound-statement header); after a command word they are
# ordinary arguments, so `echo if sed -i ...` must not record sed as a command.
# Mirrors the round-44 fix in the main scanner above.
if expect:
pending = False
prev_flag = False
wrapper = None
continue
if _tok.startswith("-"):
if not pending:
expect = False
elif _wrapper_flag_takes_operand(wrapper, _tok):
prev_flag = True
continue
if not expect:
continue
if _tok == "!":
continue # pipeline negation keeps command position (! bash s.sh)
if _ASSIGNMENT_RE.match(_tok):
continue
if pending and _is_wrapper_numeric_arg(_tok):
prev_flag = False
continue
_base = _token_basename(_tok)
if (
pending
and prev_flag
and _base not in _BLOCKED_COMMANDS
and _base not in _COMMAND_PREFIXES
):
prev_flag = False
continue
prev_flag = False
if _base in _COMMAND_PREFIXES:
pending = True
wrapper = _base
continue
out.append(_i)
expect = False
pending = False
wrapper = None
return out
_cmd_word_idx = _command_word_indices()
def _wrapper_prefix_indices():
# Indices where a _COMMAND_PREFIXES wrapper (env / xargs / watch / ...) sits AT command
# position. _command_word_indices SKIPS these (it records the RESOLVED command), but the
# watch / xargs handlers below key off the wrapper token itself, so track them here with
# the same command-position rules -- so `echo watch rm` (watch in ARGUMENT position) is
# not mistaken for a wrapper.
out = []
expect = True
pending = False
prev_flag = False
wrapper = None
for _i, _tok in enumerate(tokens):
if _tok in _SHELL_SEPARATORS:
expect = True
pending = False
prev_flag = False
wrapper = None
continue
if _tok in _SHELL_KEYWORDS_AS_SEP:
if expect:
pending = False
prev_flag = False
wrapper = None
continue
if _tok.startswith("-"):
if not pending:
expect = False
elif _wrapper_flag_takes_operand(wrapper, _tok):
prev_flag = True
continue
if not expect:
continue
if _tok == "!":
continue
if _ASSIGNMENT_RE.match(_tok):
continue
if pending and _is_wrapper_numeric_arg(_tok):
prev_flag = False
continue
_base = _token_basename(_tok)
if (
pending
and prev_flag
and _base not in _BLOCKED_COMMANDS
and _base not in _COMMAND_PREFIXES
):
prev_flag = False
continue
prev_flag = False
if _base in _COMMAND_PREFIXES:
out.append(_i)
pending = True
wrapper = _base
continue
expect = False
pending = False
wrapper = None
return out
_wrapper_prefix_idx = _wrapper_prefix_indices()
# trap 'CMD' SIGSPEC registers CMD to run (in the unguarded shell) on EXIT / a signal, so
# the quoted handler is unscanned shell code. Scan the handler operand of a command-position
# `trap` recursively; a reset (trap - EXIT) / ignore (trap '' EXIT) has nothing to run.
for i in _cmd_word_idx:
if _token_basename(tokens[i]) != "trap":
continue
# Skip trap options / the -- terminator (trap -- 'CMD' EXIT, trap -p) so the handler
# operand is not mistaken for -- and left unscanned.
_j = i + 1
while _j < len(tokens) and tokens[_j].startswith("-") and len(tokens[_j]) > 1:
_j += 1
if _j >= len(tokens):
continue
_h = tokens[_j]
if _h and _h != "-" and _h not in _SHELL_SEPARATORS and _h not in _SHELL_KEYWORDS_AS_SEP:
blocked |= _find_blocked_commands(_h)
# A shell binary invoked with a SCRIPT FILE (`bash s.sh`) or `-s` (read the script from
# stdin) runs unscanned shell code in the same unguarded environment; only the inline
# `-c '...'` form is statically analyzable (handled above). Block a command-position
# shell whose operands include a non-flag argument (the script) and no -c/-lc flag. Using
# the wrapper-aware command-word indices so `env bash s.sh` / `timeout 5 bash s.sh` are
# not hidden behind the wrapper prefix.
for i in _cmd_word_idx:
tok = tokens[i]
if os.path.basename(tok).lower() not in _SHELLS:
continue
_has_c = False
_script = None
_interactive = False
for k in range(i + 1, len(tokens)):
t = tokens[k]
if t in _SHELL_SEPARATORS or t in _SHELL_KEYWORDS_AS_SEP:
break
tl = t.lower()
# An interactive shell (bash -i, sh -i, or a combined short flag like -ic) SOURCES
# the user's rc files (.bashrc / ENV) before running any -c payload, executing
# unscanned workdir startup code in the unguarded child. Treat -i as unscanned
# startup like BASH_ENV.
if tl.startswith("-") and not tl.startswith("--") and "i" in tl[1:]:
_interactive = True
if tl == "-c" or (tl.startswith("-") and not tl.startswith("--") and tl.endswith("c")):
_has_c = True
break
if tl in ("-s", "--"): # -s reads the script from stdin (unscanned)
_script = t
break
if t.startswith("-"):
continue # other shell flags: -l, -x, --login, --norc, ...
_script = t # first non-flag operand is the script file
break
if _interactive:
blocked.add("shell-interactive-rc:" + _token_basename(tok))
# Any command-position shell WITHOUT an inline `-c` payload runs unscanned code:
# a script file (bash s.sh), stdin via -s, or a bare shell that reads stdin
# (`printf 'evil' | bash`). Only the `-c '...'` form is statically analyzable, so
# block everything else.
if not _has_c:
blocked.add("shell-script:" + (_script or _token_basename(tok)))
# BASH_ENV=script / ENV=script assignment prefix before a shell makes bash / sh SOURCE
# that workdir file before the scanned -c payload runs, executing unscanned commands in
# the unguarded child (BASH_ENV=env.sh bash -c 'echo ok', env BASH_ENV=env.sh bash -c).
# Scan the command segment before this shell word for a non-empty startup-env assignment.
for k in range(i - 1, -1, -1):
pk = tokens[k]
if pk in _SHELL_SEPARATORS or pk in _SHELL_KEYWORDS_AS_SEP:
break
if _ASSIGNMENT_RE.match(pk):
_an, _, _av = pk.partition("=")
if _an in ("BASH_ENV", "ENV") and _av != "":
blocked.add("shell-startup-env:" + _an)
# Local VAR=value bindings in this command, so a PATH component expanded from a locally-set
# variable (P=.; PATH=$P evil) can be resolved to its (unsafe) value.
_local_assigns = {}
for _et in tokens:
if _ASSIGNMENT_RE.match(_et):
_n, _, _v = _et.partition("=")
_local_assigns[_n.rstrip("+")] = _v
# Assignment prefixes that persist for the command's child: a non-empty BASH_ENV / ENV (sourced
# by a later shell), a PATH with a cwd entry (a bare command resolves to a workdir shebang), and
# git path / config environment variables -- GIT_DIR / GIT_WORK_TREE / GIT_INDEX_FILE point
# git's writes outside the workdir, and GIT_CONFIG_* override the sandbox's env-based hook
# suppression. Handle both NAME=value and NAME+=value (append).
for _ei, _et in enumerate(tokens):
if not _ASSIGNMENT_RE.match(_et):
continue
_an, _, _av = _et.partition("=")
_append = _an.endswith("+")
_an = _an.rstrip("+")
if _an in ("BASH_ENV", "ENV") and _av != "":
blocked.add("shell-startup-env:" + _an)
# PATH=. cmd / PATH+=:. cmd: a relative / cwd entry lets a bare command word resolve to a
# workdir shebang. For += the value is APPENDED to the existing PATH, so evaluate
# "$PATH" + value (a trailing / doubled separator or . entry is then the unsafe one).
elif _an == "PATH":
_pval = ("$PATH" + _av) if _append else _av
if _path_value_is_unsafe(_pval, _local_assigns):
blocked.add("unsafe-path-assign")
# GIT_DIR / GIT_WORK_TREE / GIT_INDEX_FILE set git's repo / tree / index path directly, so
# an escaping value writes outside the workdir (GIT_DIR=/tmp/x git init) with no --git-dir.
elif _an in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE") and _arg_escapes_workdir(_av):
blocked.add("git-write-outside")
# GIT_CONFIG[_GLOBAL/_SYSTEM/_COUNT/_KEY_*/_VALUE_*] re-point git config or drop the
# sandbox's env-based hook suppression (GIT_CONFIG_COUNT=0 git ...), re-enabling a planted
# .git/hooks/* in an unguarded git child.
elif _an == "GIT_CONFIG" or _an.startswith("GIT_CONFIG_"):
blocked.add("git-config-env-override")
# git -c alias.X='!CMD' X / git config alias.X '!CMD': a git alias whose value starts with
# `!` runs CMD through an unguarded shell, but the scanner sees only `git`. Flag the shell-
# dispatch alias form (the ! marker) so the aliased writer / reader is not smuggled past.
for i in _cmd_word_idx:
if _token_basename(tokens[i]) != "git":
continue
# An env -C DIR / --chdir DIR wrapper BEFORE git changes git's cwd, so even a bare or
# relative write subcommand (env -C /tmp git init) resolves under DIR. Scan back to the
# previous separator for such a wrapper; if DIR escapes the workdir, git operates outside.
_git_cwd_escapes = False
_env_suppress_dropped = False
_seg_has_env = False
for _bk in range(i - 1, -1, -1):
_bt = tokens[_bk]
if _bt in _SHELL_SEPARATORS or _bt in _SHELL_KEYWORDS_AS_SEP:
break
if _bt in ("-C", "--chdir") and _bk + 1 < len(tokens):
if _arg_escapes_workdir(tokens[_bk + 1]):
_git_cwd_escapes = True
elif _bt.startswith("--chdir=") and _arg_escapes_workdir(_bt.split("=", 1)[1]):
_git_cwd_escapes = True
# GNU env glues the short chdir operand directly onto the flag (env -C/tmp git init),
# which the separated / --chdir= forms above miss. Only -C takes a dir here.
elif _bt.startswith("-C") and len(_bt) > 2 and _arg_escapes_workdir(_bt[2:]):
_git_cwd_escapes = True
# env -i / --ignore-environment / a bare `-` start git with an EMPTY environment, and
# env -u NAME / --unset NAME / --unset=NAME strip just the suppression var; either
# removes the injected core.hooksPath suppression so a planted .git/hooks/* runs in
# the unguarded git child. Handle the separated and glued long forms and the bare `-`.
if _bt in ("-i", "--ignore-environment", "-"):
_env_suppress_dropped = True
elif (
_bt in ("-u", "--unset")
and _bk + 1 < len(tokens)
and tokens[_bk + 1].startswith("GIT_CONFIG")
):
_env_suppress_dropped = True
elif _bt.startswith("--unset=") and _bt.split("=", 1)[1].startswith("GIT_CONFIG"):
_env_suppress_dropped = True
# GNU env glues the short unset operand onto the flag (env -uGIT_CONFIG git ...).
elif _bt.startswith("-u") and len(_bt) > 2 and _bt[2:].startswith("GIT_CONFIG"):
_env_suppress_dropped = True
elif _token_basename(_bt) == "env":
_seg_has_env = True
if _git_cwd_escapes:
blocked.add("git-write-outside")
if _env_suppress_dropped and _seg_has_env:
blocked.add("git-config-env-override")
_seg = []
for k in range(i + 1, len(tokens)):
if tokens[k] in _SHELL_SEPARATORS or tokens[k] in _SHELL_KEYWORDS_AS_SEP:
break
_seg.append(tokens[k])
_joined = " ".join(_seg)
if re.search(r"alias\.[^=\s]+=\s*!", _joined):
blocked.add("git-shell-alias")
else:
for _k, _t in enumerate(_seg):
if _t.startswith("alias.") and _k + 1 < len(_seg) and _seg[_k + 1].startswith("!"):
blocked.add("git-shell-alias")
break
# git init /tmp/x, git clone url /tmp/x, git worktree add /tmp/x, git -C /outside ...
# all create / operate on files outside the workdir in an unguarded native git child.
# Flag a path OPERAND (bare, non-flag) or a -C / --git-dir / --work-tree value that
# escapes the workdir. Workdir-relative git usage (git init, git clone url, git -C sub)
# and non-path operands (a clone URL, a config name=value) stay allowed.
_gk = 0
while _gk < len(_seg):
_gt = _seg[_gk]
# git -c KEY=VALUE: an execution-capable config (core.fsmonitor / sshCommand / ...)
# runs VALUE in an unguarded child; core.hooksPath / init.templateDir re-enable
# planted hooks. Block the exec-capable configs (alias.*=! handled above).
if _gt == "-c" and _gk + 1 < len(_seg):
if _git_config_key_is_exec(_seg[_gk + 1].split("=", 1)[0]):
blocked.add("git-exec-config")
_gk += 2
continue
# git --exec-path=<path> re-points where git looks for its git-<cmd> helpers, so
# `git --exec-path=. evil` runs a workdir git-evil in an unguarded child. Any value
# redirects the core path (the no-value form just prints it), so flag it.
if _gt.startswith("--exec-path=") and _gt.split("=", 1)[1]:
blocked.add("git-exec-config")
_gk += 1
continue
# git --config-env=KEY=ENVVAR sets a config KEY from an env var, so an execution-
# capable / alias KEY (git --config-env=alias.x=P with P='!cmd') runs a command.
if _gt.startswith("--config-env="):
_cekey = _gt.split("=", 1)[1].split("=", 1)[0]
if _git_config_key_is_exec(_cekey) or _cekey.startswith("alias."):
blocked.add("git-exec-config")
_gk += 1
continue
if _gt in _GIT_PATH_VALUE_OPTIONS and _gk + 1 < len(_seg):
if _git_operand_escapes(_seg[_gk + 1], _local_assigns):
blocked.add("git-write-outside")
_gk += 2
continue
# Stuck short form: git archive -o/tmp/x (and -O.. / -C/outside) glue the path value
# directly onto the short option with no space, which the separated / --opt=val scans
# above miss. Only the path-valued SHORT options take a glued value; a non-escaping
# value (-oout.tar, -C90 for find-copies) is left alone by _git_operand_escapes.
if len(_gt) > 2 and _gt[:2] in ("-C", "-o", "-O"):
if _git_operand_escapes(_gt[2:], _local_assigns):
blocked.add("git-write-outside")
_gk += 1
continue
_oeq = None
for _opt in _GIT_PATH_VALUE_OPTIONS:
if _gt.startswith(_opt + "="):
_oeq = _gt.split("=", 1)[1]
break
if _oeq is not None:
if _git_operand_escapes(_oeq, _local_assigns):
blocked.add("git-write-outside")
elif not _gt.startswith("-") and _git_operand_escapes(_gt, _local_assigns):
blocked.add("git-write-outside")
_gk += 1
# git apply --unsafe-paths lets a patch write to targets OUTSIDE the working tree (a
# +++ ../../tmp/x hunk), which the native git child applies with no realpath guard. The
# patch body is not statically visible, so deny the unsafe mode outright; a plain
# git apply p.patch (in-tree targets) stays allowed.
if "apply" in _seg and "--unsafe-paths" in _seg:
blocked.add("git-write-outside")
# git config [options] KEY [VALUE]: setting an execution-capable config key (git config
# core.pager 'sh -c ...') runs its value on later git operations, like the -c form; and
# git config --file <path> / -f <path> writes the config to an arbitrary file, escaping
# the workdir (git config --file=/tmp/gitcfg ...).
for _ci, _ct in enumerate(_seg):
if _ct == "config":
_cj = _ci + 1
# --system / --global select the host system / user config file (/etc/gitconfig,
# ~/.gitconfig), both OUTSIDE the workdir. A WRITE there (KEY VALUE, or a write
# flag / --edit) escapes the sandbox; a pure read (--get* / --list / -l / a bare
# KEY) does not, so only writes are blocked.
_host_scope = False
_write_flag = False
while _cj < len(_seg):
_cw = _seg[_cj]
if _cw in ("--file", "-f") and _cj + 1 < len(_seg):
if _arg_escapes_workdir(_seg[_cj + 1]):
blocked.add("git-write-outside")
_cj += 2
continue
if _cw.startswith("--file="):
if _arg_escapes_workdir(_cw.split("=", 1)[1]):
blocked.add("git-write-outside")
_cj += 1
continue
if _cw in ("--system", "--global"):
_host_scope = True
_cj += 1
continue
if _cw in (
"--add",
"--unset",
"--unset-all",
"--replace-all",
"--remove-section",
"--rename-section",
"-e",
"--edit",
):
_write_flag = True
_cj += 1
continue
if not _cw.startswith("-"):
if _git_config_key_is_exec(_cw.split("=", 1)[0]):
blocked.add("git-exec-config")
# A host-scope write: an explicit write flag, or a KEY followed by a VALUE
# operand (git config --global user.name x). A bare KEY read is left alone.
if _host_scope and (
_write_flag
or (_cj + 1 < len(_seg) and not _seg[_cj + 1].startswith("-"))
):
blocked.add("git-write-outside")
break
_cj += 1
if _host_scope and _write_flag:
blocked.add("git-write-outside") # --global --edit / --unset with no inline KEY
break
# hash -p PATHNAME NAME binds the command NAME to PATHNAME in the shell's hash table, so a
# later bare `NAME` runs PATHNAME. With a local executable (hash -p ./evil ls; ls) that
# launches an unguarded workdir shebang under a benign-looking command word. Block hash -p
# when its pathname operand is a local executable path.
for i in _cmd_word_idx:
if _token_basename(tokens[i]) != "hash":
continue
for k in range(i + 1, len(tokens)):
t = tokens[k]
if t in _SHELL_SEPARATORS or t in _SHELL_KEYWORDS_AS_SEP:
break
if t == "-p" and k + 1 < len(tokens) and _is_local_executable_path(tokens[k + 1]):
blocked.add("hash-p-local-exec")
break
# alias x='touch /tmp/p'; ...; x (with expand_aliases) runs the alias BODY at execution
# time, but the command word `x` is unknown to the scanner. Scan the body of each alias
# definition so a blocked writer / interpreter in it is caught at the definition site.
for i in _cmd_word_idx:
if _token_basename(tokens[i]) != "alias":
continue
for k in range(i + 1, len(tokens)):
t = tokens[k]
if t in _SHELL_SEPARATORS or t in _SHELL_KEYWORDS_AS_SEP:
break
if t.startswith("-"):
continue # alias -p (print)
if "=" in t:
_body = t.split("=", 1)[1]
if _body:
blocked |= _find_blocked_commands(_body)
# openssl <subcmd> ... -out FILE writes FILE in an unguarded openssl child (openssl rand
# -out /tmp/p 4), which the realpath guard never sees. Block when an output-file flag names a
# path that escapes the workdir; a workdir-local -out (openssl rand -out key.bin) and the
# no-output forms (openssl rand -hex 16, openssl dgst file) stay allowed.
for i in _cmd_word_idx:
if _token_basename(tokens[i]) != "openssl":
continue
for k in range(i + 1, len(tokens)):
t = tokens[k]
if t in _SHELL_SEPARATORS or t in _SHELL_KEYWORDS_AS_SEP:
break
if (
t in _OPENSSL_WRITE_FLAGS
and k + 1 < len(tokens)
and _git_operand_escapes(tokens[k + 1], _local_assigns)
):
blocked.add("openssl-write-outside")
# sqlite3 <DBFILE> creates / opens a database in an unguarded child (no realpath guard), and
# its dot-commands (.output / .backup / .dump / .read ...) read + write arbitrary files. Flag
# a DBFILE operand that escapes the workdir, and any dot-file target that escapes. A local DB
# (sqlite3 local.db 'create ...'), :memory:, and an in-memory URI carry no escape and stay
# allowed. -init / -cmd option values are option operands, not the DBFILE.
for i in _cmd_word_idx:
if _token_basename(tokens[i]) != "sqlite3":
continue
_seen_db = False
_sk = i + 1
while _sk < len(tokens):
t = tokens[_sk]
if t in _SHELL_SEPARATORS or t in _SHELL_KEYWORDS_AS_SEP:
break
# sqlite3 options that consume a SEPARATED operand; skip the value so it is not
# mistaken for the DBFILE (only -init reads a file, checked via its own value here).
if t in _SQLITE_OPERAND_OPTS:
if (
t == "-init"
and _sk + 1 < len(tokens)
and _git_operand_escapes(tokens[_sk + 1], _local_assigns)
):
blocked.add("sqlite3-write-outside")
_sk += 2
continue
# Any dot-command file target that escapes the workdir (.output /tmp/leak, .backup
# ../x, .read $P) writes / reads a host path; scan the (possibly quoted, multi-line
# SQL) operand for one.
_unq = t
if len(_unq) >= 2 and _unq[0] == _unq[-1] and _unq[0] in ("'", '"'):
_unq = _unq[1:-1]
# .shell CMD / .system CMD run an arbitrary command in the unguarded child shell.
if _SQLITE_SHELL_RE.search(_unq):
blocked.add("sqlite3-shell")
for _m in _SQLITE_DOTFILE_RE.finditer(_unq):
_dot_f = _m.group("f")
if len(_dot_f) >= 2 and _dot_f[0] == _dot_f[-1] and _dot_f[0] in ("'", '"'):
_dot_f = _dot_f[1:-1]
# .output |CMD / .once |CMD open CMD as a PIPE (a shell command), not a file.
if _dot_f.startswith("|"):
blocked.add("sqlite3-shell")
elif _dot_f not in ("stdout", "stderr", "off") and _git_operand_escapes(
_dot_f, _local_assigns
):
blocked.add("sqlite3-write-outside")
if t.startswith("-"):
_sk += 1
continue
# First bare operand is the DBFILE. :memory: / '' / file::memory: never touch disk.
if not _seen_db:
_seen_db = True
_dbn = t
if len(_dbn) >= 2 and _dbn[0] == _dbn[-1] and _dbn[0] in ("'", '"'):
_dbn = _dbn[1:-1]
_dblow = _dbn.lower()
_is_mem = (
_dbn in ("", ":memory:")
or _dblow.startswith("file::memory:")
or "mode=memory" in _dblow
)
if not _is_mem and _git_operand_escapes(_dbn, _local_assigns):
blocked.add("sqlite3-write-outside")
_sk += 1
# watch runs its command via `sh -c '<operands joined>'` UNLESS -x/--exec is given (then it
# execs argv directly, resolved by the wrapper handling above). So a quoted payload
# (watch 'python3 -c ...', watch -n 0.1 'rm -rf /') is shell CODE, not one inert command
# word; scan it recursively. A bare `watch date` / `watch -n 1 date` just re-scans `date`.
for i in _wrapper_prefix_idx:
if _token_basename(tokens[i]) != "watch":
continue
_has_x = False
_ops = []
_skip_val = False
_wk = i + 1
while _wk < len(tokens):
t = tokens[_wk]
if t in _SHELL_SEPARATORS or t in _SHELL_KEYWORDS_AS_SEP:
break
if _skip_val:
_skip_val = False
_wk += 1
continue
if t in ("-x", "--exec"):
_has_x = True
elif t in ("-n", "--interval"):
_skip_val = True
elif not t.startswith("-"):
_ops.append(t)
_wk += 1
if not _has_x and _ops:
_payload = " ".join(
(o[1:-1] if len(o) >= 2 and o[0] == o[-1] and o[0] in ("'", '"') else o)
for o in _ops
)
blocked |= _find_blocked_commands(_payload)
# xargs -I{} / -i / --replace substitutes UNSCANNED stdin into the command at runtime. When
# the replacement token becomes the command word (xargs -I{} {}) or flows into an interpreter
# code string (xargs -I{} sh -c '{}', xargs -I% python3 -c %), stdin executes as code -- the
# `{}` payload the scanner sees is inert. Fail closed on those forms; a replacement used only
# as a data ARGUMENT to a non-interpreter (xargs -I{} cp {} dir/) is left to the normal
# command-word scan, and xargs without a replace flag (xargs echo hi) is unaffected.
_XARGS_INTERP = _SHELL_BINARIES | _INTERPRETER_COMMANDS
for i in _wrapper_prefix_idx:
if _token_basename(tokens[i]) != "xargs":
continue
_xseg = []
_xk = i + 1
while _xk < len(tokens):
t = tokens[_xk]
if t in _SHELL_SEPARATORS or t in _SHELL_KEYWORDS_AS_SEP:
break
_xseg.append(t)
_xk += 1
_repl = None
_xj = 0
while _xj < len(_xseg):
t = _xseg[_xj]
if t == "-I" and _xj + 1 < len(_xseg):
_repl = _xseg[_xj + 1]
_xj += 2
continue
if t.startswith("-I") and len(t) > 2:
_repl = t[2:]
elif t in ("-i", "--replace"):
_repl = "{}"
elif t.startswith("--replace="):
_repl = t.split("=", 1)[1] or "{}"
elif t.startswith("-i") and len(t) > 2:
_repl = t[2:]
_xj += 1
if not _repl:
continue
# Resolve the wrapped command word (skip xargs flags + their separated operands).
_cwidx = None
_cwj = 0
while _cwj < len(_xseg):
t = _xseg[_cwj]
if t.startswith("-"):
_cwj += 2 if _wrapper_flag_takes_operand("xargs", t) else 1
continue
_cwidx = _cwj
break
if _cwidx is None:
continue
_cw = os.path.basename(_xseg[_cwidx]).lower()
if _repl in _xseg[_cwidx]:
blocked.add("xargs-replace-exec") # stdin becomes the command itself
elif _cw in _XARGS_INTERP:
for _ci2 in range(_cwidx + 1, len(_xseg)):
_ct2 = _xseg[_ci2].lower()
_is_code_flag = _ct2 in ("-c", "-e", "--eval") or (
_ct2.startswith("-") and not _ct2.startswith("--") and _ct2.endswith("c")
)
if _is_code_flag and _ci2 + 1 < len(_xseg) and _repl in _xseg[_ci2 + 1]:
blocked.add("xargs-replace-exec") # stdin flows into interpreter code
break
# Output redirection (> / >> / &> / N>) runs in an unguarded child shell that follows
# symlinks before any Python guard, so no filename target can be trusted: a relative
# single-component name (> out) may be a pre-existing symlink to an outside file, a
# relative multi-component name (> sub/out) may traverse a symlinked subdir, an absolute
# / ~ / .. target is plainly outside, and a $ / backtick target can expand anywhere.
# Fail closed on every real-file target; only fd duplications (>&2) and the standard
# device sinks (/dev/null, ...) are allowed. Scanning tokens (not the raw string) avoids
# matching a `>` inside a quoted argument.
for i, tok in enumerate(tokens):
rm = re.search(r">{1,2}([^\s>]*)$", tok)
if rm is None:
continue
tgt = rm.group(1)
j = i
# `>|` (noclobber override) and `>&` (stdout+stderr / fd-or-file redirect) tokenize
# as `>` then `|` / `&`, so that punctuation is part of the redirect operator, not a
# pipeline / background op; skip it and take the real target after.
if not tgt and j + 1 < len(tokens) and tokens[j + 1] in ("|", "&"):
j += 1
if not tgt and j + 1 < len(tokens):
tgt = tokens[j + 1]
if not tgt:
continue
tn = tgt.replace("\\", "/")
# Allowed: a pure fd duplication (>&2, >&1 -> `&2` / a bare digit) and the safe
# device sinks. Everything else is a file target that fails closed.
if tgt.startswith("&") or tgt.isdigit() or tn in _SAFE_REDIRECT_TARGETS:
continue
blocked.add("redirect:" + tgt)
# `cd` / `pushd` to a dir OUTSIDE the workdir moves the child shell's cwd so a later
# relative redirect / write escapes (`cd /tmp; echo x > p`, `pushd /tmp; echo x > p`).
# Block a command-position cwd change to an absolute / .. / ~ / variable target; a
# relative in-workdir `cd data` stays allowed.
_at_cmd = True
for i, tok in enumerate(tokens):
if tok in _SHELL_SEPARATORS or tok in _SHELL_KEYWORDS_AS_SEP:
_at_cmd = True
continue
if _at_cmd and _token_basename(tok) in ("command", "builtin"):
# `command` / `builtin` run the following shell builtin with its args, so a
# `command cd /tmp` still changes the cwd. Stay at command position so the cd
# behind the wrapper is inspected (bash `help command`/`help builtin`).
continue
if _at_cmd and _token_basename(tok) in ("cd", "pushd"):
_cwd_kw = _token_basename(tok)
for k in range(i + 1, len(tokens)):
t = tokens[k]
if t.startswith("-") or t.startswith("+"):
continue # cd flags (-P/-L/-e/-@) and pushd rotation (+N/-N)
tnn = t.replace("\\", "/")
if (
t.startswith("~")
or tnn.startswith("/")
or ".." in tnn.split("/")
or "$" in t
or "`" in t
):
blocked.add(_cwd_kw + ":" + t)
break
_at_cmd = False
continue
if not tok.startswith("-"):
_at_cmd = False
# An EXPANSION in COMMAND POSITION runs whatever it expands to as the command name and
# cannot be proven safe: a command substitution ($(printf touch) / `printf touch`), a
# variable-expanded command word (p=python3; $p -c ...), or a ${VAR} parameter expansion.
# Fail closed. (An argument-position expansion -- echo $(date), echo $HOME, x=$(cmd) -- is
# not at command position, so it stays allowed. ${IFS} is already expanded to whitespace
# above, so a `cat${IFS}x` command word is not misread as an expansion here.)
if re.search(r"(?:^|[\n;&|(])\s*(?:\$|`)", command):
blocked.add("command-expansion")
# Some normally read-only utilities MUTATE files with certain flags (sed -i, sort -o
# FILE, find ... -delete, dd of=FILE, tee FILE, truncate), writing/deleting OUTSIDE the
# workdir in an unguarded child that no redirect token exposes. Treat the mutating
# invocation as a child writer. Uses the wrapper-aware command-word indices so a wrapper
# prefix (env sed -i ..., nice sed -i ...) does not hide the mutating utility.
for i in _cmd_word_idx:
tok = tokens[i]
_base = _token_basename(tok)
if _base not in (
"sed",
"gsed",
"ssed",
"perl",
"sort",
"find",
"dd",
"tee",
"truncate",
"history",
):
continue
if _base == "truncate":
blocked.add("mutating:truncate")
continue
for k in range(i + 1, len(tokens)):
a = tokens[k]
if a in _SHELL_SEPARATORS or a in _SHELL_KEYWORDS_AS_SEP:
break
al = a.lower()
_short = al.startswith("-") and not al.startswith("--")
if _base in ("sed", "gsed", "ssed", "perl"):
if al.startswith("--in-place") or (_short and "i" in al[1:]):
blocked.add("mutating:" + _base)
break
# A sed SCRIPT can write files (`w FILE` / `W FILE` / `s///w`) or execute shell
# commands (`e CMD` / `s///e`) even without -i: sed -n '1w /tmp/escape' file,
# sed -n 'w/tmp/probe' file (no space), sed '1e touch /tmp/x' file. The script may
# be a bare positional OR provided via -e / --expression (sed -e'w /tmp/x' /dev/null,
# sed --expression='w /tmp/x'). Detect the write / execute commands and flags in the
# script text; a plain s/word/x/ is not matched.
if _base in ("sed", "gsed", "ssed"):
_sed_script = None
if a in ("-e", "--expression") and k + 1 < len(tokens):
_sed_script = tokens[k + 1] # -e SCRIPT (separated)
elif al.startswith("-e") and not al.startswith("--") and len(a) > 2:
_sed_script = a[2:] # glued -e'w /tmp/x'
elif a.startswith("--expression="):
_sed_script = a.split("=", 1)[1]
elif not a.startswith("-"):
_sed_script = a # bare positional script
if _sed_script is not None and (
_SED_WRITE_RE.search(_sed_script)
or _SED_EXEC_RE.search(_sed_script)
or _SED_ADDR_EXEC_RE.search(_sed_script)
or _SED_SFLAG_RE.search(_sed_script)
):
blocked.add("mutating:" + _base)
break
elif _base == "sort":
if al.startswith("--output") or (_short and "o" in al[1:]):
blocked.add("mutating:sort")
break
elif _base == "find":
# -delete removes; -fprint/-fprintf/-fprint0 and -fls write their listing to a
# named FILE (find . -fls /tmp/escape truncates/creates it in an unguarded child).
if al == "-delete" or al.startswith("-fprint") or al == "-fls":
blocked.add("mutating:find")
break
elif _base == "dd":
if al.startswith("of="):
blocked.add("mutating:dd")
break
elif _base == "tee" and not a.startswith("-"):
blocked.add("mutating:tee")
break
elif _base == "history" and _short and any(_c in al[1:] for _c in "warn"):
# bash's history builtin reads/writes an arbitrary file: `history -w FILE`
# (or -a append) creates/overwrites an absolute host path, and `-r` / `-n`
# read a file into the history buffer. Even without a FILE operand it targets
# $HISTFILE, which the caller can point outside the workdir. -c / -d / -p / -s
# do not touch a file, so only w / a / r / n are blocked.
blocked.add("mutating:history")
break
return blocked
def _blocked_in_argv(str_elts: list[str | None]) -> tuple[set[str], int | None]:
"""Scan the command WORDS of a non-shell argv vector (subprocess.run(['rm', '-rf', '/'])).
Only element 0 -- and the real command after any wrapper prefix (env / nice / timeout /
xargs / ...) -- is executed by the OS; every later element is a literal argument that is
never run. Scanning just the command word keeps `env rm -rf /` blocked (rm resolved through
the wrapper) while a benign argument such as subprocess.run(['echo', 'python']) is not
misread as invoking `python`.
Returns (blocked_basenames, cmd_index): cmd_index is the position of the resolved command
word (or None), so the caller can hand a wrapper-hidden shell binary (env bash s.sh) to the
shell-argv analyzer."""
blocked: set[str] = set()
idx, n = 0, len(str_elts)
prefix_pending = False # a wrapper is awaiting its real command word
prev_was_flag = False # last token (under a wrapper) was an option flag with an operand
cur_wrapper = None # the active wrapper's basename (env / nice / timeout / ...)
while idx < n:
tok = str_elts[idx]
if tok is None:
return blocked, None # a non-literal element hides the command word; stop
# env FOO=bar assignments precede the command word.
if _ASSIGNMENT_RE.match(tok):
idx += 1
continue
if prefix_pending and tok.startswith("-"):
# env -S CMD / --split-string=CMD splits its operand into a command line, so
# scan that operand with the full command scanner (env -S 'bash -c ...').
if cur_wrapper == "env":
if tok in ("-S", "--split-string"):
_nxt = str_elts[idx + 1] if idx + 1 < n else None
if _nxt is not None:
blocked |= _find_blocked_commands(_nxt)
return blocked, None
if tok.startswith("--split-string="):
blocked |= _find_blocked_commands(tok[len("--split-string=") :])
return blocked, None
if tok.startswith("-S") and len(tok) > 2:
blocked |= _find_blocked_commands(tok[2:])
return blocked, None
# Only a flag that takes a SEPARATED operand (env -u NAME, nice -n 5) marks the
# next token as its value; a no-operand flag (env -i, xargs -0) or a glued short
# flag (stdbuf -oL) does not, so the real command after it is still analysed.
prev_was_flag = _wrapper_flag_takes_operand(cur_wrapper, tok)
idx += 1
continue
# A wrapper's numeric arg (`timeout 5 cmd`).
if prefix_pending and _is_wrapper_numeric_arg(tok):
prev_was_flag = False
idx += 1
continue
base = os.path.basename(tok).lower()
stem, ext = os.path.splitext(base)
if ext in {".exe", ".com", ".bat", ".cmd"}:
base = stem
# A wrapper flag's SEPARATED operand (`env -u FOO python3`, `env -C DIR cmd`): the
# token after a wrapper option flag that is not itself a blocked command / prefix /
# shell is the flag's value -- skip it and keep scanning so the real command (python3,
# bash) is not missed. A blocked command / prefix / shell is treated as the command.
if (
prefix_pending
and prev_was_flag
and base not in _BLOCKED_COMMANDS
and base not in _COMMAND_PREFIXES
and base not in _SHELL_BINARIES
):
prev_was_flag = False
idx += 1
continue
prev_was_flag = False
if base in _BLOCKED_COMMANDS or _is_versioned_interpreter(base):
blocked.add(base)
if base in _COMMAND_PREFIXES:
prefix_pending = True
cur_wrapper = base
idx += 1
continue # wrapper consumes one command; the next word is the real one
if _is_local_executable_path(tok):
blocked.add("local-exec:" + base) # runs an unguarded shebang interpreter
return blocked, idx # reached the executed command word
return blocked, None
def _build_safe_env(workdir: str) -> dict[str, str]:
"""Build a minimal, credential-free environment for sandboxed subprocesses.
Whitelist-built from scratch (parent env NOT inherited): only PATH/HOME/
TMPDIR/LANG/TERM/PYTHONIOENCODING (+VIRTUAL_ENV or Windows SystemRoot) reach
the child; all credential vars (HF_TOKEN, AWS_*, etc.) are absent. HOME
points at the sandbox workdir so SDKs can't read the operator's cached creds.
"""
# Start from the running interpreter's dir so 'python'/'pip' resolve to the
# same environment the Studio server runs in.
exe_dir = os.path.dirname(sys.executable)
path_entries = [exe_dir] if exe_dir else []
# If a virtualenv is active, include its bin/Scripts directory.
venv = os.environ.get("VIRTUAL_ENV")
if venv:
venv_bin = os.path.join(venv, "Scripts" if sys.platform == "win32" else "bin")
if venv_bin not in path_entries:
path_entries.append(venv_bin)
if sys.platform == "win32":
sysroot = os.environ.get("SystemRoot", r"C:\Windows")
path_entries.extend([os.path.join(sysroot, "System32"), sysroot])
else:
path_entries.extend(["/usr/local/bin", "/usr/bin", "/bin"])
# Deduplicate, preserving order.
deduped = list(dict.fromkeys(p for p in path_entries if p))
env = {
"PATH": os.pathsep.join(deduped),
"HOME": workdir,
"TMPDIR": workdir,
"LANG": os.environ.get("LANG", "C.UTF-8"),
"TERM": "dumb",
"PYTHONIOENCODING": "utf-8",
# HOME points at the workdir, so a prior run could plant
# .local/.../site-packages/usercustomize.py that runs (unguarded) at the next child's
# startup. Disable the per-user site directory here too (belt-and-suspenders with the
# interpreter's -s flag) so a sandboxed child never imports it.
"PYTHONNOUSERSITE": "1",
# git runs repository hooks (.git/hooks/pre-commit, post-checkout, ...) as executable
# files in an UNGUARDED child; a sandboxed snippet could plant one and trigger it via a
# benign-looking git commit / merge / checkout. Point core.hooksPath at a non-directory
# (via git's env-config mechanism) so NO repository hook runs, for every git subcommand,
# without having to block git itself. Neutralizing hooks is the sandbox-correct default.
"GIT_CONFIG_COUNT": "1",
"GIT_CONFIG_KEY_0": "core.hooksPath",
"GIT_CONFIG_VALUE_0": os.devnull,
}
if venv:
env["VIRTUAL_ENV"] = venv
# Windows needs SystemRoot for Python/subprocess to work.
if sys.platform == "win32":
env["SystemRoot"] = os.environ.get("SystemRoot", r"C:\Windows")
return env
# Credential env vars dropped even in bypass mode so tool code cannot read the
# operator's keys. Over-strips on purpose (a benign var is harmless to lose).
_BYPASS_ENV_SECRET_NAMES = frozenset(
{
"HF_TOKEN",
"HF_HUB_TOKEN",
"HUGGING_FACE_HUB_TOKEN",
"HUGGINGFACE_TOKEN",
"HUGGINGFACEHUB_API_TOKEN",
"WANDB_API_KEY",
"GH_TOKEN",
"GITHUB_TOKEN",
"OPENAI_API_KEY",
"ANTHROPIC_API_KEY",
"GEMINI_API_KEY",
"GOOGLE_API_KEY",
"GROQ_API_KEY",
"OPENROUTER_API_KEY",
"REPLICATE_API_TOKEN",
"COHERE_API_KEY",
"MISTRAL_API_KEY",
"NGC_API_KEY",
"KAGGLE_KEY",
"MYSQL_PWD", # exact name: markers use PASSWD, not PWD (PWD is the cwd var)
"LD_PRELOAD",
# Auth brokers / capability handles: not secrets by value, but they
# hand the child the operator's live agent (ssh/gpg), kube config, or
# docker daemon. Names are listed because there is no value signal to
# key off. URL config vars (HTTP_PROXY, PIP_INDEX_URL, DATABASE_URL,
# ...) are intentionally NOT name-listed: a benign proxy/index without
# credentials must keep working in bypass mode, while a credentialed
# value is dropped by _is_secret_env_value() regardless of its name.
"SSH_AUTH_SOCK",
"SSH_AGENT_PID",
"GPG_AGENT_INFO",
"GNUPGHOME",
"KUBECONFIG",
"DOCKER_HOST",
}
)
_BYPASS_ENV_SECRET_PREFIXES = ("AWS_", "AZURE_", "GOOGLE_", "GCP_", "GCLOUD_", "DYLD_")
_BYPASS_ENV_SECRET_MARKERS = (
"TOKEN",
"API_KEY",
"APIKEY",
"SECRET",
"PASSWORD",
"PASSWD",
"CREDENTIAL",
"PRIVATE_KEY",
"AUTH", # e.g. NPM_CONFIG__AUTH (npm _auth), REDISCLI_AUTH
# Azure App Service connection strings: SQLCONNSTR_/CUSTOMCONNSTR_/... and
# WEBSITE_CONTENTAZUREFILECONNECTIONSTRING carry DB/storage credentials.
"CONNSTR",
"CONNECTIONSTRING",
)
# Non-secret hardening flags that match a secret prefix/marker but must be KEPT
# so bypass mode does not silently undo an operator's opt-out. AWS_EC2_METADATA_
# DISABLED tells the AWS SDK/CLI not to pull instance-role creds from IMDS;
# dropping it would re-open that path for a bypassed tool.
_BYPASS_ENV_KEEP_NAMES = frozenset(
{
"AWS_EC2_METADATA_DISABLED",
"AWS_EC2_METADATA_V1_DISABLED",
}
)
# Matches a URL that embeds userinfo before the host, covering both
# "scheme://user:pass@host" and token-only "scheme://token@host" (and
# percent-encoded variants). The userinfo must precede the first '/', so an '@'
# in a path or query does not false-positive. Used to scrub credential-bearing
# URL values regardless of the variable's name.
_URL_USERINFO_RE = re.compile(r"://[^/\s@]+@")
# Connection-string credential fields (ADO.NET / Azure storage / Service Bus):
# "...;Password=...", "...;AccountKey=...", "...;SharedAccessKey=...". Catches
# credential-bearing values whose names dodge the name classifier. "accesskey"
# also covers Shared/Secret AccessKey via substring; the Name fields (e.g.
# SharedAccessKeyName=) do not match since "=" must follow the keyword.
_SECRET_VALUE_RE = re.compile(r"(?i)(?:password|pwd|accountkey|accesskey)\s*=\s*[^\s;]")
# Names that hold no secret value but point SDKs at the operator's real
# home/cache/config (cached tokens, cred files), defeating the HOME repoint.
# Startup always sets HF_HOME (-> $HF_HOME/token), so this is the live leak.
# Dropped in bypass mode so tools fall back to the empty repointed HOME.
_BYPASS_ENV_CRED_LOCATION_NAMES = frozenset(
{
# HF cache roots (token lives under $HF_HOME/token)
"HF_HOME",
"HF_HUB_CACHE",
"HUGGINGFACE_HUB_CACHE",
"HF_XET_CACHE",
"TRANSFORMERS_CACHE",
"HF_DATASETS_CACHE",
"HF_ASSETS_CACHE",
# XDG base dirs (resolved before $HOME)
"XDG_CONFIG_HOME",
"XDG_CACHE_HOME",
"XDG_DATA_HOME",
# explicit cred/config file pointers honoured before $HOME
"NETRC",
"PGPASSFILE",
"BOTO_CONFIG",
"PIP_CONFIG_FILE",
"CLOUDSDK_CONFIG",
"KAGGLE_CONFIG_DIR",
"DOCKER_CONFIG",
"WANDB_DIR",
"WANDB_CONFIG_DIR",
"WANDB_CACHE_DIR",
# package-manager / git / cloud config pointers to real cred files
"NPM_CONFIG_USERCONFIG",
"NPM_CONFIG_GLOBALCONFIG",
"YARN_RC_FILENAME",
"GIT_CONFIG_GLOBAL",
"GIT_CONFIG_SYSTEM",
"CARGO_HOME",
"RCLONE_CONFIG",
# auth-helper scripts that hand creds to git/ssh
"GIT_ASKPASS",
"SSH_ASKPASS",
# shell startup hook: bash -c sources $BASH_ENV (can re-export secrets)
"BASH_ENV",
# Windows: HOMEDRIVE+HOMEPATH compose a home that bypasses HOME
"HOMEDRIVE",
"HOMEPATH",
}
)
# Windows profile dirs SDKs read creds under; repointed (not dropped) since
# callers expect them present.
_BYPASS_ENV_WINDOWS_PROFILE_VARS = ("USERPROFILE", "APPDATA", "LOCALAPPDATA")
def _is_secret_env_name(name: str) -> bool:
"""True if an env var name looks like it carries a credential."""
upper = name.upper()
if upper in _BYPASS_ENV_KEEP_NAMES:
return False # non-secret hardening flag; keep it
if upper in _BYPASS_ENV_SECRET_NAMES:
return True
if any(upper.startswith(p) for p in _BYPASS_ENV_SECRET_PREFIXES):
return True
return any(marker in upper for marker in _BYPASS_ENV_SECRET_MARKERS)
def _is_cred_location_env_name(name: str) -> bool:
"""True for vars that point SDKs at the real home/cache/config (cached creds)."""
return name.upper() in _BYPASS_ENV_CRED_LOCATION_NAMES
def _is_secret_env_value(value: str) -> bool:
"""True if a value embeds credentials regardless of its name.
Catches URL userinfo (``scheme://user:token@host`` in DATABASE_URL /
PIP_INDEX_URL / HTTP_PROXY) and connection-string credential fields
(``...;Password=...`` / ``...;AccountKey=...``) whose names dodge the name
classifier.
"""
if not value:
return False
return _URL_USERINFO_RE.search(value) is not None or _SECRET_VALUE_RE.search(value) is not None
def _build_bypass_env(workdir: str) -> dict[str, str]:
"""Env for bypass exec: full host env (unrestricted) minus credential vars,
with HOME/TMPDIR repointed at the workdir so SDKs cannot read cached creds.
Note: stripping the child env is necessary but not sufficient on its own -
a same-UID child can still read the parent's environment via procfs, so
callers also harden the parent (see _harden_parent_against_proc_env_leak).
"""
env = {
k: v
for k, v in os.environ.items()
if not _is_secret_env_name(k)
and not _is_secret_env_value(v)
and not _is_cred_location_env_name(k)
}
env["HOME"] = workdir
env["TMPDIR"] = workdir
# Windows tempfile / SDKs honour TEMP/TMP, not TMPDIR; repoint all three so
# the bypassed tool writes under the per-session sandbox dir on every OS.
env["TEMP"] = workdir
env["TMP"] = workdir
# Windows SDKs read creds under the profile dirs, not $HOME; repoint set
# ones to the workdir (HOMEDRIVE/HOMEPATH are dropped above).
for var in _BYPASS_ENV_WINDOWS_PROFILE_VARS:
if var in os.environ:
env[var] = workdir
return env
def _sandbox_preexec():
"""Best-effort sandbox setup for sandboxed subprocesses (modules are
resolved at import time so the forked child runs no imports)."""
try:
os.setsid()
except OSError:
pass
try:
os.umask(0o077)
except OSError:
pass
if _libc is not None:
try:
_libc.prctl(38, 1, 0, 0, 0) # PR_SET_NO_NEW_PRIVS
except (OSError, AttributeError):
pass
try:
_libc.prctl(1, 9, 0, 0, 0) # PR_SET_PDEATHSIG = SIGKILL
except (OSError, AttributeError):
pass
# CLONE_NEWNET not applied: with userns enabled it blocks all egress,
# including allowlisted hosts. Network policy is enforced by the AST
# host check and the bash blocklist.
if _resource is not None:
# RLIMIT_NPROC is per-real-UID, so the cap is well above normal usage.
try:
nproc = int(os.environ.get("UNSLOTH_STUDIO_SANDBOX_NPROC", "10000"))
_resource.setrlimit(_resource.RLIMIT_NPROC, (nproc, nproc))
except (ValueError, OSError, AttributeError):
pass
try:
_resource.setrlimit(_resource.RLIMIT_FSIZE, (100 * 1024 * 1024, 100 * 1024 * 1024))
except (ValueError, OSError):
pass
try:
as_bytes = int(os.environ.get("UNSLOTH_STUDIO_SANDBOX_AS_GB", "8")) * 1024 * 1024 * 1024
_resource.setrlimit(_resource.RLIMIT_AS, (as_bytes, as_bytes))
except (ValueError, OSError, AttributeError):
pass
try:
cpu_s = int(os.environ.get("UNSLOTH_STUDIO_SANDBOX_CPU_S", "600"))
_resource.setrlimit(_resource.RLIMIT_CPU, (cpu_s, cpu_s))
except (ValueError, OSError, AttributeError):
pass
try:
# High enough for multi-shard safetensors mmaps; tunable via env.
# Clamp to the inherited hard limit so setrlimit doesn't ValueError
# when the parent's hard cap is below the request.
nofile = int(os.environ.get("UNSLOTH_STUDIO_SANDBOX_NOFILE", "16384"))
_soft_cur, hard_cur = _resource.getrlimit(_resource.RLIMIT_NOFILE)
target = nofile if hard_cur == _resource.RLIM_INFINITY else min(nofile, hard_cur)
_resource.setrlimit(_resource.RLIMIT_NOFILE, (target, target))
except (ValueError, OSError, AttributeError):
pass
def _bypass_preexec():
"""Minimal pre-exec for bypass exec: os.setsid() only.
Required, not a restriction: _kill_process_tree does killpg(getpgid(child)),
so without a new session a timeout/cancel would kill the Studio server too.
"""
try:
os.setsid()
except OSError:
pass
# Hardening the Studio parent is done once (PR_SET_DUMPABLE is process-global
# and sticky); guarded so repeated bypass calls do not re-issue the prctl.
_parent_proc_hardened = False
def _harden_parent_against_proc_env_leak() -> bool:
"""Make the Studio process's /proc/<pid>/environ unreadable to its children.
Stripping the child env is not enough on Linux: a bypassed same-UID child
runs unsandboxed and can read /proc/<getppid()>/environ to recover the
tool-executing process's *unfiltered* secrets (HF_TOKEN, cloud keys, ...).
Clearing the dumpable flag (PR_SET_DUMPABLE=0) reparents this process's
/proc entries to root, so a same-UID child can no longer read its environ.
Returns True when the process is hardened or hardening is unnecessary (no
/proc leak off Linux), and False when it is needed but could not be applied
(e.g. prctl denied by a seccomp policy). Callers must fail closed - refuse
the unsandboxed exec - when this returns False, rather than running with the
parent environ still readable.
Scope: this closes the direct parent read (the demonstrated leak). It is a
mitigation, not a full boundary - a bypassed tool is unsandboxed by design,
so it can still walk /proc to a same-UID *ancestor* (e.g. the launching
shell) or read on-disk credentials by absolute path. Complete isolation
needs a separate uid / PID+mount namespace, which is out of scope here; the
UI already warns the mode is dangerous. Applied lazily on first bypass exec
so non-bypass operation is unchanged.
"""
global _parent_proc_hardened
if _parent_proc_hardened:
return True
if sys.platform != "linux":
return True # no /proc/<pid>/environ same-UID leak to close
if _libc is None:
return False # on Linux but cannot issue prctl -> cannot harden
try:
# prctl(PR_SET_DUMPABLE=4, SUID_DUMP_DISABLE=0). ctypes returns the
# syscall result (-1 on failure) and does NOT raise, so check it.
ret = _libc.prctl(4, 0, 0, 0, 0)
except (OSError, AttributeError):
return False
if ret != 0:
return False
_parent_proc_hardened = True
return True
def _get_shell_cmd(command: str) -> list[str]:
"""Return the platform-appropriate shell invocation for a command string."""
if sys.platform == "win32":
return ["cmd", "/c", command]
return ["bash", "-c", command]
# Per-session working directories so each chat thread gets its own sandbox.
# Falls back to ~/studio_sandbox/_default for callers without a session_id.
_workdirs: dict[str, str] = {}
# Non-matching session_ids collapse to ``_invalid`` to block cross-session escapes.
_SESSION_ID_RE = re.compile(r"\A[A-Za-z0-9_\-]{1,64}\Z")
_PROJECT_SESSION_PREFIX = "project-"
def _get_project_workdir(session_id: str) -> str | None:
if not session_id.startswith(_PROJECT_SESSION_PREFIX):
return None
project_id = session_id[len(_PROJECT_SESSION_PREFIX) :]
if not project_id or not _SESSION_ID_RE.match(project_id):
return None
try:
from storage.studio_db import ensure_chat_project_workspace
project = ensure_chat_project_workspace(project_id)
except Exception:
logger.warning("Failed to resolve project sandbox for %s", session_id, exc_info = True)
return None
if not project:
return None
root_path = project.get("rootPath")
sandbox_path = project.get("sandboxPath")
if not root_path or not sandbox_path:
return None
root_real = os.path.realpath(root_path)
sandbox_real = os.path.realpath(sandbox_path)
if sandbox_real != root_real and not sandbox_real.startswith(root_real + os.sep):
return None
return sandbox_real
def _get_workdir(session_id: str | None = None) -> str:
"""Return a per-session sandbox dir at mode 0o700."""
global _workdirs
key = session_id or "_default"
if key not in _workdirs or not os.path.isdir(_workdirs[key]):
home = os.path.expanduser("~")
sandbox_root = os.path.join(home, "studio_sandbox")
project_workdir = (
_get_project_workdir(session_id)
if session_id and _SESSION_ID_RE.match(session_id)
else None
)
if project_workdir:
workdir = project_workdir
elif session_id and _SESSION_ID_RE.match(session_id):
workdir = os.path.join(sandbox_root, session_id)
if not os.path.realpath(workdir).startswith(os.path.realpath(sandbox_root) + os.sep):
workdir = os.path.join(sandbox_root, "_invalid")
elif session_id:
workdir = os.path.join(sandbox_root, "_invalid")
else:
workdir = os.path.join(sandbox_root, "_default")
os.makedirs(workdir, exist_ok = True)
try:
os.chmod(sandbox_root, 0o700)
except OSError:
pass
try:
os.chmod(workdir, 0o700)
except OSError:
pass
_workdirs[key] = workdir
return _workdirs[key]
def get_sandbox_workdir(session_id: str | None = None) -> str:
return _get_workdir(session_id)
WEB_SEARCH_TOOL = {
"type": "function",
"function": {
"name": "web_search",
"description": (
"Search the web and fetch page content. Returns snippets for all results. "
"Use the url parameter to fetch full page text from a specific URL."
),
"parameters": {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "The search query",
},
"url": {
"type": "string",
"description": "A URL to fetch full page content from (instead of searching). Use this to read a page found in search results.",
},
},
"required": [],
},
},
}
PYTHON_TOOL = {
"type": "function",
"function": {
"name": "python",
"description": "Execute Python code in a sandbox and return stdout/stderr.",
"parameters": {
"type": "object",
"properties": {
"code": {
"type": "string",
"description": "The Python code to run",
}
},
"required": ["code"],
},
},
}
TERMINAL_TOOL = {
"type": "function",
"function": {
"name": "terminal",
"description": "Execute a terminal command and return stdout/stderr.",
"parameters": {
"type": "object",
"properties": {
"command": {
"type": "string",
"description": "The command to run",
}
},
"required": ["command"],
},
},
}
RENDER_HTML_TOOL = {
"type": "function",
"function": {
"name": "render_html",
"description": (
"Render a self-contained HTML/CSS/JavaScript canvas for the user. "
"Call this at most once per assistant response unless the user "
"explicitly asks for changes in that response. Future user requests "
"for new canvases may call render_html once. Put the entire document "
"in code, including any CSS in <style> tags and JavaScript in <script> tags."
),
"parameters": {
"type": "object",
"properties": {
"code": {
"type": "string",
"description": "A complete self-contained HTML document.",
},
"title": {
"type": "string",
"description": "Short display title for the canvas.",
},
},
"required": ["code"],
},
},
}
# Duplicated (not imported from core.rag.tool) so the registry never pulls in
# the RAG stack; dispatch imports it lazily.
SEARCH_KNOWLEDGE_BASE_TOOL = {
"type": "function",
"function": {
"name": "search_knowledge_base",
"description": (
"Search the user's uploaded documents and knowledge bases for "
"relevant passages. Use this whenever the question may be answered "
"by the attached documents, then cite the returned chunks."
),
"parameters": {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "Natural-language search query.",
},
"top_k": {
"type": "integer",
"description": "Max chunks to return.",
},
},
"required": ["query"],
},
},
}
ALL_TOOLS = [
WEB_SEARCH_TOOL,
PYTHON_TOOL,
TERMINAL_TOOL,
RENDER_HTML_TOOL,
SEARCH_KNOWLEDGE_BASE_TOOL,
]
# OpenAI's function.name regex ^[a-zA-Z0-9_-]{1,64}$, enforced before streaming.
# MCP tool names with '.', '/', spaces, etc. would 400 the whole request, so we
# validate up front and skip with a warning.
_OPENAI_FN_NAME_RE = re.compile(r"^[a-zA-Z0-9_-]{1,64}$")
def _mcp_specs_for_server(server: dict, mcp_tools: list[dict]) -> list[dict]:
"""Convert an MCP server's tool list into OpenAI function specs."""
display = server.get("display_name") or server["id"]
specs: list[dict] = []
seen_names: set[str] = set()
for tool in mcp_tools:
raw_name = tool.get("name") or ""
if not raw_name:
logger.warning("Skipping MCP tool on '%s': empty name.", display)
continue
name = f"{MCP_TOOL_PREFIX}{server['id']}__{raw_name}"
# Bad chars or oversized names would 400 the whole request; skip + warn
# so the rest of the tools still ship.
if not _OPENAI_FN_NAME_RE.fullmatch(name):
logger.warning(
"Skipping MCP tool '%s' on '%s': composed name '%s' is not "
"valid OpenAI function.name (regex ^[a-zA-Z0-9_-]{1,64}$).",
raw_name,
display,
name,
)
continue
# Duplicate tool names would also 400 OpenAI; drop dupes.
if name in seen_names:
logger.warning("Skipping duplicate MCP tool '%s' on '%s'.", raw_name, display)
continue
seen_names.add(name)
specs.append(
{
"type": "function",
"function": {
"name": name,
"description": f"[{display}] {tool.get('description') or ''}".strip(),
"parameters": tool.get("inputSchema") or {"type": "object", "properties": {}},
},
}
)
return specs
async def get_enabled_mcp_tools() -> list[dict]:
servers = [s for s in mcp_servers_db.list_servers() if s.get("is_enabled")]
# Never spawn stdio servers when stdio is disabled on this host (e.g. a DB
# carried from a desktop install onto a Colab/network deployment).
if not stdio_mcp_enabled():
servers = [s for s in servers if not is_stdio(s["url"])]
if not servers:
return []
# Skip servers still in their post-failure cool-off, otherwise a down
# server gets re-probed -- and blocks the send for the full timeout -- on
# every message.
uncached = [
s for s in servers if get_cached_tools(s["id"]) is None and not in_failure_cooloff(s["id"])
]
if uncached:
results = await asyncio.gather(
*(
list_tools_async(
url = s["url"],
headers = parse_server_headers(s),
timeout = probe_timeout(s["url"], bool(s.get("use_oauth"))),
use_oauth = bool(s.get("use_oauth")),
)
for s in uncached
),
return_exceptions = True,
)
# An edit/delete can land while we await a probe (up to 305 s for
# OAuth); its cache eviction is a no-op against an entry we haven't
# written yet. Re-read and drop a result whose server changed or
# was removed mid-probe, else a stale tool list caches indefinitely.
current = {s["id"]: s for s in mcp_servers_db.list_servers()}
for server, payload in zip(uncached, results):
# Guard the failure branch too: a stale failure must not park a
# cool-off on the fresh config, or the server the user just fixed
# is skipped for the whole window.
fresh = current.get(server["id"])
if fresh is None or any(
fresh.get(k) != server.get(k) for k in TOOL_CACHE_INVALIDATING_FIELDS
):
continue
if isinstance(payload, BaseException):
logger.warning(
"MCP server '%s' (%s) discovery failed: %s",
server.get("display_name") or server["id"],
server.get("url"),
payload,
)
# Failures aren't cached, but record one so a down server
# isn't re-probed every send during the cool-off.
record_probe_failure(server["id"], bool(fresh.get("use_oauth")))
continue
cache_tools(server["id"], payload)
specs: list[dict] = []
for server in servers:
payload = get_cached_tools(server["id"])
if payload is None:
continue
specs.extend(_mcp_specs_for_server(server, payload))
return specs
_TIMEOUT_UNSET = object()
def _render_html_result(arguments: dict) -> str:
code = arguments.get("code")
if not isinstance(code, str) or not code.strip():
return "Error: render_html requires a non-empty code string."
title = arguments.get("title")
if isinstance(title, str) and title.strip():
safe_title = title.strip()[:120]
return (
f"Rendered HTML canvas: {safe_title}. Do not call render_html "
"again in this response unless the user asks for changes. For a later "
"user request for a new canvas, call render_html once."
)
return (
"Rendered HTML canvas. Do not call render_html again in this response "
"unless the user asks for changes. For a later user request for a new "
"canvas, call render_html once."
)
def execute_tool(
name: str,
arguments: dict,
cancel_event = None,
timeout: int | None = _TIMEOUT_UNSET,
session_id: str | None = None,
rag_scope: dict | None = None,
disable_sandbox: bool = False,
) -> str:
"""Execute a tool by name with the given arguments; returns a string.
``timeout``: int seconds, ``None`` = no limit, unset = ``_EXEC_TIMEOUT``.
``session_id``: optional ID for per-conversation sandbox isolation.
``rag_scope``: hidden per-request RAG context the model never sees; consumed
by ``search_knowledge_base``.
``disable_sandbox``: Bypass Permissions; run python/terminal without the
safety checks, blocklist, or resource caps (secrets still stripped). Only
affects local code tools; web_search / MCP are unchanged.
"""
logger.info(f"execute_tool: name={name}, session_id={session_id}, timeout={timeout}")
effective_timeout = _EXEC_TIMEOUT if timeout is _TIMEOUT_UNSET else timeout
if name == "search_knowledge_base":
return _search_knowledge_base(arguments, rag_scope)
if name == "render_html":
return _render_html_result(arguments)
if name.startswith(MCP_TOOL_PREFIX):
try:
_, server_id, tool_name = name.split("__", 2)
except ValueError:
return f"Error: malformed MCP tool name '{name}'"
server = mcp_servers_db.get_server(server_id)
if not server:
return f"Error: MCP server '{server_id}' not found"
if not server.get("is_enabled"):
return f"Error: MCP server '{server_id}' is disabled"
if is_stdio(server["url"]) and not stdio_mcp_enabled():
return f"Error: stdio MCP server '{server_id}' is disabled on this host"
return call_tool_sync(
url = server["url"],
headers = parse_server_headers(server),
name = tool_name,
args = arguments,
timeout = effective_timeout,
use_oauth = bool(server.get("use_oauth")),
cancel_event = cancel_event,
)
if name == "web_search":
return _web_search(
arguments.get("query", ""),
url = arguments.get("url"),
timeout = effective_timeout,
)
if name == "python":
return _python_exec(
arguments.get("code", ""),
cancel_event,
effective_timeout,
session_id,
disable_sandbox = disable_sandbox,
)
if name == "terminal":
return _bash_exec(
arguments.get("command", ""),
cancel_event,
effective_timeout,
session_id,
disable_sandbox = disable_sandbox,
)
return f"Unknown tool: {name}"
def _opt_int(v) -> int | None:
try:
return int(v) if v is not None else None
except (TypeError, ValueError):
return None
def _scope_retrieval_kwargs(scope: dict) -> dict:
"""Retrieval mode from rag_scope; candidate pools and RRF come from config."""
mode = scope.get("mode")
return {"mode": mode if mode in ("hybrid", "dense", "lexical") else "hybrid"}
def _search_knowledge_base(arguments: dict, rag_scope: dict | None) -> str:
"""Run the RAG search bound to the hidden per-request ``rag_scope`` (the model
supplies only ``query``/``top_k``). Lazy import; missing sqlite-vec degrades
to a friendly message."""
scope = rag_scope or {}
query = (arguments or {}).get("query", "")
if not query or not str(query).strip():
return "Error: query is empty."
try:
from storage import rag_db
if not rag_db.RAG_AVAILABLE:
return "Knowledge base search is unavailable on this server."
from core.rag.tool import search_knowledge_base_with_sources
except Exception as exc: # noqa: BLE001
logger.warning("RAG tool unavailable: %s", exc)
return "Knowledge base search is unavailable on this server."
top_k = _opt_int((arguments or {}).get("top_k") or scope.get("default_top_k"))
text, sources = search_knowledge_base_with_sources(
query = str(query),
scope_kb_id = scope.get("kb_id"),
scope_thread_id = scope.get("thread_id"),
scope_project_id = scope.get("project_id"),
top_k = top_k,
**_scope_retrieval_kwargs(scope),
)
# Append the UI source-map after the sentinel; loops strip it before the model.
if sources:
import json as _json
return text + RAG_SOURCES_SENTINEL + _json.dumps(sources, ensure_ascii = False)
return text
# Forced first-pass RAG retrieval: a high cosine floor keeps it precise (fires on
# on-topic queries, skips weak ones) and helps small models that under-call the tool.
# Tunable via RAG_AUTOINJECT_MIN_SCORE.
_AUTOINJECT_DEFAULT_FLOOR = 0.70
def _autoinject_enabled() -> bool:
return os.environ.get("RAG_AUTOINJECT", "1").strip().lower() not in (
"0",
"false",
"no",
"off",
)
def _autoinject_floor() -> float:
raw = os.environ.get("RAG_AUTOINJECT_MIN_SCORE")
if raw is not None:
try:
return float(raw)
except ValueError:
pass
return _AUTOINJECT_DEFAULT_FLOOR
# Lean: injecting the full top_k every turn prefills thousands of tokens.
_AUTOINJECT_DEFAULT_TOP_K = 4
def _autoinject_top_k() -> int:
raw = os.environ.get("RAG_AUTOINJECT_TOP_K")
if raw is not None:
try:
return max(1, int(raw))
except ValueError:
pass
return _AUTOINJECT_DEFAULT_TOP_K
def _thread_whole_doc_enabled(scope: dict) -> bool:
"""Whether a thread-attached file should be injected in full rather than
retrieved top-K. ``rag_scope.whole_doc=False`` disables it for this request."""
override = scope.get("whole_doc")
if override is False:
return False
try:
from core.rag import config as _rag_config
except Exception: # noqa: BLE001
return True
return _rag_config.THREAD_WHOLE_DOC
_IMAGE_PART_TOKEN_ESTIMATE = 1024
def _message_token_estimate(conversation: list[dict]) -> int:
"""Cheap prompt-size estimate for budget guards; exact tokenization happens later."""
total = 0
for msg in conversation:
content = msg.get("content")
if isinstance(content, str):
total += max(1, len(content) // 4)
elif isinstance(content, list):
for part in content:
if isinstance(part, dict):
if part.get("type") in ("image_url", "input_image"):
total += _IMAGE_PART_TOKEN_ESTIMATE
else:
total += max(1, len(str(part.get("text") or "")) // 4)
total += 4 # chat-template role / separator overhead estimate
return total
def _whole_doc_budget(scope: dict | None = None, conversation: list[dict] | None = None) -> int:
try:
from core.rag import config as _rag_config
except Exception: # noqa: BLE001
budget = 6000
else:
budget = _rag_config.WHOLE_DOC_MAX_TOKENS
if not scope:
return budget
context = _opt_int(scope.get("context_length") or scope.get("max_context_tokens"))
if context is None or context <= 0:
return budget
headroom = _opt_int(scope.get("response_headroom"))
if headroom is None:
headroom = max(1024, context // 4)
used = _message_token_estimate(conversation or [])
# Leave room for tool XML wrappers, citation metadata, and chat-template overhead.
available = context - headroom - used - 512
return min(budget, max(0, available))
def _last_user_text(conversation: list[dict]) -> str:
"""Plain text of the most recent user turn (text parts only)."""
for msg in reversed(conversation):
if msg.get("role") != "user":
continue
content = msg.get("content")
if isinstance(content, str):
return content.strip()
if isinstance(content, list):
parts = [
p.get("text", "")
for p in content
if isinstance(p, dict) and p.get("type") in ("text", "input_text")
]
return " ".join(t for t in parts if t).strip()
return ""
return ""
def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> dict | None:
"""Pre-retrieve the latest user turn; if a hit clears the cosine floor return
``{"events": [...], "messages": [...]}`` to splice into the loop, else ``None``.
Toggle via ``rag_scope.autoinject`` (else env ``RAG_AUTOINJECT``); floor via
``rag_scope.autoinject_min_score`` (else env ``RAG_AUTOINJECT_MIN_SCORE``).
Also the small-model fallback: models below ~4B often answer from memory
instead of calling ``search_knowledge_base``, so forcing retrieval here keeps
attachments consulted regardless of model size."""
if not rag_scope:
return None
enabled = rag_scope.get("autoinject")
if enabled is None:
enabled = _autoinject_enabled()
thread_id = rag_scope.get("thread_id")
whole_doc_requested = (
bool(thread_id) and not rag_scope.get("kb_id") and _thread_whole_doc_enabled(rag_scope)
)
if not enabled and not whole_doc_requested:
return None
query = _last_user_text(conversation)
if not query:
return None
try:
from storage import rag_db
if not rag_db.RAG_AVAILABLE:
return None
from core.rag.tool import render_sources, search_for_autoinject, whole_document_context
except Exception as exc: # noqa: BLE001
logger.warning("RAG auto-inject unavailable: %s", exc)
return None
text: str | None = None
sources: list[dict] = []
floor_override = rag_scope.get("autoinject_min_score")
floor = float(floor_override) if floor_override is not None else _autoinject_floor()
# Cap at the lean top_k, but honor a lower user setting.
lean_k = _autoinject_top_k()
sidebar_k = _opt_int(rag_scope.get("default_top_k"))
top_k = min(sidebar_k, lean_k) if sidebar_k is not None else lean_k
# Whole-document mode: a thread-attached file under budget is injected in full so
# the model reads everything. A KB selection is exclusive, so whole-doc never
# preempts it; in a project chat the project sources are still retrieved top-K and
# appended under one citation numbering. Oversized files (or no thread doc) fall
# through to the combined top-K retrieval below.
if whole_doc_requested:
try:
budget = _whole_doc_budget(rag_scope, conversation)
whole = whole_document_context(
scope_thread_id = thread_id,
max_tokens = budget,
)
except Exception as exc: # noqa: BLE001
logger.warning("RAG whole-document context failed: %s", exc)
whole = None
if whole is not None:
text, sources = whole
project_id = rag_scope.get("project_id")
if project_id:
try:
proj = search_for_autoinject(
query = query,
scope_project_id = project_id,
top_k = top_k,
min_dense_score = floor,
**_scope_retrieval_kwargs(rag_scope),
)
except Exception as exc: # noqa: BLE001
logger.warning("RAG project retrieval (whole-doc companion) failed: %s", exc)
proj = None
if proj is not None:
merged = sources + proj[1]
merged_text = render_sources(merged)
if max(1, len(merged_text) // 4) <= budget:
sources = merged
text = merged_text
logger.info("RAG auto-inject: whole-document context (%d chunk(s))", len(sources))
if text is None and enabled:
try:
found = search_for_autoinject(
query = query,
scope_kb_id = rag_scope.get("kb_id"),
scope_thread_id = rag_scope.get("thread_id"),
scope_project_id = rag_scope.get("project_id"),
top_k = top_k,
min_dense_score = floor,
**_scope_retrieval_kwargs(rag_scope),
)
except Exception as exc: # noqa: BLE001
logger.warning("RAG auto-inject retrieval failed: %s", exc)
return None
if not found:
logger.info("RAG auto-inject: no passage >= %.2f; skipping", floor)
return None
text, sources = found
if text is None:
return None
import json as _json
import uuid as _uuid
call_id = "rag_auto_" + _uuid.uuid4().hex[:12]
args = {"query": query}
full_result = text + RAG_SOURCES_SENTINEL + _json.dumps(sources, ensure_ascii = False)
events = [
{"type": "status", "text": f"Searching documents: {query[:60]}"},
{
"type": "tool_start",
"tool_name": "search_knowledge_base",
"tool_call_id": call_id,
"arguments": args,
},
{
"type": "tool_end",
"tool_name": "search_knowledge_base",
"tool_call_id": call_id,
"result": full_result,
},
{"type": "status", "text": ""},
]
messages = [
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": call_id,
"type": "function",
"function": {
"name": "search_knowledge_base",
"arguments": _json.dumps(args, ensure_ascii = False),
},
}
],
},
{
"role": "tool",
"name": "search_knowledge_base",
"tool_call_id": call_id,
"content": text,
},
]
logger.info("RAG auto-inject: %d passage(s) for %r", len(sources), query[:80])
return {"events": events, "messages": messages}
_MAX_PAGE_CHARS = 16000 # cap fetched page text (after HTML-to-MD conversion)
# Raw download cap > _MAX_PAGE_CHARS because SSR pages embed large <head>
# sections stripped during conversion; 512 KB reaches article content even
# where <head> alone is ~200 KB.
_MAX_FETCH_BYTES = 512 * 1024
_USER_AGENTS = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15",
)
_tls_ctx = ssl.create_default_context()
class _NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
return None
class _PinnedHTTPSConnection(http.client.HTTPSConnection):
"""HTTPS connection to a pinned IP, using a different hostname for SNI and
cert verification.
SSRF IP-pinning rewrites URLs to raw IPs; a normal HTTPSConnection would then
send no SNI and verify the cert against the IP (both fail). This splits the
concerns: TCP connects to the pinned IP (``host``), TLS uses ``sni_hostname``.
"""
def __init__(self, host: str, *, sni_hostname: str, **kwargs):
super().__init__(host, **kwargs)
self._sni_hostname = sni_hostname
def connect(self):
# TCP connect to the pinned IP in self.host.
http.client.HTTPConnection.connect(self)
# TLS handshake with the real hostname for SNI + cert verification.
self.sock = self._context.wrap_socket(
self.sock,
server_hostname = self._sni_hostname,
)
class _SNIHTTPSHandler(urllib.request.HTTPSHandler):
"""HTTPS handler sending the correct SNI hostname during TLS handshake.
SSRF IP-pinning breaks SNI and cert verification; this returns a
``_PinnedHTTPSConnection`` that connects to the pinned IP but verifies TLS
against the original hostname.
"""
def __init__(self, hostname: str):
super().__init__(context = _tls_ctx)
self._sni_hostname = hostname
def https_open(self, req):
return self.do_open(self._sni_connection, req)
def _sni_connection(self, host, **kwargs):
kwargs["context"] = _tls_ctx
return _PinnedHTTPSConnection(host, sni_hostname = self._sni_hostname, **kwargs)
def _validate_and_resolve_host(hostname: str, port: int) -> tuple[bool, str, str]:
"""Resolve *hostname*, reject non-public IPs, return a pinned IP string.
Returns ``(ok, reason_or_empty, resolved_ip)``. The caller should connect
to *resolved_ip* (with a ``Host`` header) to prevent DNS rebinding between
validation and the actual fetch.
"""
import ipaddress
import socket
try:
infos = socket.getaddrinfo(hostname, port, type = socket.SOCK_STREAM)
except OSError as e:
return False, f"Failed to resolve host: {e}", ""
if not infos:
return False, f"Failed to resolve host: no addresses for {hostname!r}", ""
for *_, sockaddr in infos:
ip = ipaddress.ip_address(sockaddr[0])
# `not ip.is_global` is the source of truth: it rejects every category
# below PLUS shared/CGNAT (100.64.0.0/10) and benchmarking/doc ranges
# Python marks is_private=False and is_global=False. The explicit
# predicates only give human-readable categories in the error message.
if (
not ip.is_global
or ip.is_private
or ip.is_loopback
or ip.is_link_local
or ip.is_multicast
or ip.is_reserved
or ip.is_unspecified
):
return False, f"Blocked: refusing to fetch non-public address {ip}.", ""
# Return the first resolved address for pinning.
first_ip = infos[0][4][0]
return True, "", first_ip
def _fetch_page_text(
url: str,
max_chars: int = _MAX_PAGE_CHARS,
timeout: int = 30,
) -> str:
"""Fetch a URL and return plain text content (HTML tags stripped).
Blocks private/loopback/link-local targets (SSRF protection) and caps
the download size to avoid unbounded memory usage.
"""
from urllib.parse import urlparse
parsed = urlparse(url)
if parsed.scheme not in ("http", "https"):
return f"Blocked: only http/https URLs are allowed (got {parsed.scheme!r})."
if not parsed.hostname:
return "Blocked: URL is missing a hostname."
port = parsed.port or (443 if parsed.scheme == "https" else 80)
ok, reason, pinned_ip = _validate_and_resolve_host(parsed.hostname, port)
if not ok:
return reason
try:
from urllib.error import HTTPError as _HTTPError
from urllib.parse import urljoin, urlunparse
max_bytes = _MAX_FETCH_BYTES
current_url = url
current_host = parsed.hostname
ua = random.choice(_USER_AGENTS)
for _hop in range(5):
# Pin to the validated IP (prevents DNS rebinding): rewrite URL to
# the IP, set the Host header.
cp = urlparse(current_url)
# Bracket IPv6 addresses so the netloc is valid in a URL.
ip_str = f"[{pinned_ip}]" if ":" in pinned_ip else pinned_ip
ip_netloc = f"{ip_str}:{cp.port}" if cp.port else ip_str
pinned_url = urlunparse(cp._replace(netloc = ip_netloc))
opener = urllib.request.build_opener(
_NoRedirect,
_SNIHTTPSHandler(current_host),
)
req = urllib.request.Request(
pinned_url,
headers = {
"User-Agent": ua,
"Host": current_host,
},
)
try:
resp = opener.open(req, timeout = timeout)
except _HTTPError as e:
if e.code not in (301, 302, 303, 307, 308):
return f"Failed to fetch URL: HTTP {e.code} {getattr(e, 'reason', '')}"
location = e.headers.get("Location")
if not location:
return "Failed to fetch URL: redirect missing Location header."
current_url = urljoin(current_url, location)
rp = urlparse(current_url)
if rp.scheme not in ("http", "https") or not rp.hostname:
return "Blocked: redirect target is not a valid http/https URL."
rp_port = rp.port or (443 if rp.scheme == "https" else 80)
ok2, reason2, pinned_ip = _validate_and_resolve_host(
rp.hostname,
rp_port,
)
if not ok2:
return reason2
current_host = rp.hostname
continue
# Success: read capped body.
raw_bytes = resp.read(max_bytes)
break
else:
return "Failed to fetch URL: too many redirects."
charset = resp.headers.get_content_charset() or "utf-8"
raw_html = raw_bytes.decode(charset, errors = "replace")
except _HTTPError as e:
return f"Failed to fetch URL: HTTP {e.code} {getattr(e, 'reason', '')}"
except Exception as e:
return f"Failed to fetch URL: {e}"
# Convert HTML to Markdown with the builtin converter (no external deps).
from ._html_to_md import html_to_markdown
text = html_to_markdown(raw_html)
if not text:
return "(page returned no readable text)"
if len(text) > max_chars:
text = text[:max_chars] + f"\n\n... (truncated, {len(text)} chars total)"
return text
def _web_search(
query: str,
max_results: int = 5,
timeout: int = _EXEC_TIMEOUT,
url: str | None = None,
) -> str:
"""Search the web using DuckDuckGo and return formatted results.
If ``url`` is provided, fetches that page directly instead of searching.
"""
# Direct URL fetch mode.
if url and url.strip():
fetch_timeout = 60 if timeout is None else min(timeout, 60)
return _fetch_page_text(url.strip(), timeout = fetch_timeout)
if not query or not query.strip():
return "No query provided."
try:
from ddgs import DDGS
results = DDGS(timeout = timeout).text(query, max_results = max_results)
if not results:
return "No results found."
parts = []
for r in results:
parts.append(
f"Title: {r.get('title', '')}\n"
f"URL: {r.get('href', '')}\n"
f"Snippet: {r.get('body', '')}"
)
text = "\n\n---\n\n".join(parts)
text += (
"\n\n---\n\nIMPORTANT: These are only short snippets. "
"To get the full page content, call web_search with "
'the url parameter (e.g. {"url": "<URL>"}).'
)
return text
except Exception as e:
return f"Search failed: {e}"
# ==========================================================================
# Sandbox static-analysis hardening (feature-flagged; see UNSLOTH_STUDIO_SINK_ANALYZER)
#
# A pure, whitelist-only constant folder plus a filesystem-confinement path
# resolver back the eval/exec payload recursion and the destructive-op gate.
# Everything here recomputes pure transforms on *literals only* and never runs,
# imports, or reflects on user code. All limits are bounded so the analyzer can
# never be slower or crashier than the legacy syntactic checks; on any breach a
# folder returns None (opaque) and the caller fails safe.
# ==========================================================================
# Folder bounds (Stage 1). Breaching any of these yields None ("un-foldable").
_FOLD_DEPTH = 24
_FOLD_MAXLEN = 65536
_FOLD_OPS = 4000
_FOLD_MAX_SEQ = 4096
_FOLD_MAXINT = 1 << 64
_UNKNOWN = object() # sentinel: "not statically decidable"
class _ConstEnv(dict):
"""A const-prop env (name -> RHS node) that also carries the set of names REBOUND away from
their canonical builtin / stdlib module in the snippet, so the folder can refuse to fold a
shadowed helper (str = lambda _: '...'; eval(str(1))) as the real builtin."""
__slots__ = ("shadowed",)
def __init__(
self,
*a,
shadowed = None,
**k,
):
super().__init__(*a, **k)
self.shadowed = shadowed or frozenset()
class _FoldState:
"""Shared op counter + single-assignment const-prop environment."""
__slots__ = ("ops", "names", "shadowed")
def __init__(self, names = None):
self.ops = 0
self.names = names or {}
self.shadowed = getattr(names, "shadowed", None) or frozenset()
def _fold_cap(value):
"""Return value unless a str/bytes exceeds the size cap or an int the magnitude cap."""
if isinstance(value, (str, bytes, bytearray)) and len(value) > _FOLD_MAXLEN:
return None
if isinstance(value, int) and not isinstance(value, bool) and abs(value) > _FOLD_MAXINT:
return None
return value
def _too_wide(n):
"""A format width / precision / size arg large enough to OOM the folder."""
return isinstance(n, int) and not isinstance(n, bool) and n > _FOLD_MAXLEN
# Format-spec mini-language: reject an oversized width or precision BEFORE format()
# allocates the padded string. format()/str.format()/f-strings all run in the Studio
# process during static analysis, ahead of the child-subprocess rlimits.
_FMT_SPEC_RE = re.compile(r"^(?:.?[<>=^])?[+\- ]?z?#?0?(\d+)?[,_]?(?:\.(\d+))?[a-zA-Z%]?$")
def _format_spec_ok(spec):
if not isinstance(spec, str) or not spec:
return True
m = _FMT_SPEC_RE.match(spec)
if not m:
return True # unrecognized spec: let format() itself decide at runtime
return not any(g and _too_wide(int(g)) for g in m.groups())
def _format_template_ok(template):
"""Every replacement field of a str.format template has a bounded width."""
if not isinstance(template, str):
return True
try:
import string as _string
for _lit, _field, _spec, _conv in _string.Formatter().parse(template):
if _spec and not _format_spec_ok(_spec):
return False
except Exception:
return True
return True
def _format_has_nested_spec(template):
"""A replacement field whose spec itself contains a field ({:{}}): the width is
supplied by an argument, so a large numeric arg drives the allocation."""
if not isinstance(template, str):
return False
try:
import string as _string
for _lit, _field, _spec, _conv in _string.Formatter().parse(template):
if _spec and "{" in _spec:
return True
except Exception:
return False
return False
_PRINTF_WIDTH_RE = re.compile(r"%[-+ #0]*(\d+)?(?:\.(\d+))?[hlL]?[diouxXeEfFgGcrsab%]")
def _printf_ok(fmt):
"""Percent-format string with no oversized (or dynamic '*') width / precision."""
if isinstance(fmt, (bytes, bytearray)):
try:
fmt = fmt.decode("latin-1")
except Exception:
return True
if not isinstance(fmt, str):
return True
# A '*' width or precision ('%*s', '%.*f') pulls its size from a runtime argument,
# so it cannot be bounded statically -- refuse rather than risk a large allocation.
for m in re.finditer(r"%[-+ #0]*(\*)?(?:\.(\*)?\d*)?", fmt):
if m.group(1) == "*" or m.group(2) == "*":
return False
for m in _PRINTF_WIDTH_RE.finditer(fmt):
if any(g and _too_wide(int(g)) for g in m.groups()):
return False
return True
def _replace_output_ok(recv, call_args):
"""Bound str.replace/bytes.replace output before it allocates: replacing many
occurrences with a long replacement can build a multi-gigabyte string."""
if len(call_args) < 2:
return True
old, new = call_args[0], call_args[1]
if not isinstance(new, (str, bytes, bytearray)):
return True
lo = len(old) if isinstance(old, (str, bytes, bytearray)) else 1
n_repl = (len(recv) + 1) if lo == 0 else (len(recv) // max(lo, 1) + 1)
if len(call_args) >= 3 and isinstance(call_args[2], int) and call_args[2] >= 0:
n_repl = min(n_repl, call_args[2])
return len(recv) + n_repl * len(new) <= _FOLD_MAXLEN
def _join_output_ok(sep, call_args):
"""Bound str.join/bytes.join output before it allocates."""
if not call_args or not isinstance(call_args[0], (list, tuple)):
return True
items = call_args[0]
total = len(sep) * max(len(items) - 1, 0)
for x in items:
if not isinstance(x, (str, bytes, bytearray)):
return True # a real join would TypeError; not an allocation concern
total += len(x)
if total > _FOLD_MAXLEN:
return False
return True
def _fold_apply_codec(name, data):
"""Pure data transforms only (rot13/hex/base64/zlib/text codecs). Bounded zlib."""
name = name.lower().replace("-", "_")
try:
if name in ("rot_13", "rot13"):
text = data if isinstance(data, str) else data.decode("latin-1")
return codecs.decode(text, "rot_13")
if name == "hex":
return codecs.decode(data, "hex")
if name in ("base64", "base_64"):
return base64.b64decode(data if isinstance(data, (bytes, bytearray)) else data.encode())
if name == "zlib":
payload = data if isinstance(data, (bytes, bytearray)) else str(data).encode()
d = zlib.decompressobj()
out = d.decompress(payload, _FOLD_MAXLEN)
if d.unconsumed_tail: # would exceed the cap -> refuse
return None
return out
if name in ("utf_8", "utf8", "latin_1", "latin1", "ascii"):
if isinstance(data, (bytes, bytearray)):
return data.decode(name)
return data.encode(name)
except Exception:
return None
return None # bz2/lzma/gzip and unknowns: bomb-unsafe / opaque -> refuse
_FOLD_PURE_BUILTINS = frozenset(
{"chr", "ord", "str", "int", "bytes", "bytearray", "hex", "oct", "bin", "bool", "float", "len"}
)
_FOLD_STR_METHODS = frozenset(
{
"join",
"replace",
"upper",
"lower",
"strip",
"lstrip",
"rstrip",
"swapcase",
"title",
"capitalize",
"format",
"zfill",
"ljust",
"rjust",
"center",
"encode",
"decode",
}
)
_FOLD_B64_FUNCS = frozenset(
{
"b64decode",
"b64encode",
"urlsafe_b64decode",
"standard_b64decode",
"b32decode",
"b16decode",
"a85decode",
"b85decode",
}
)
def _const_fold(
node,
env = None,
_state = None,
_depth = 0,
):
"""Fold an AST expression to a concrete str/bytes/int/list value, else None.
Whitelist-only and pure: it never executes user code, never imports, never
reflects. Only a fixed set of pure transforms over already-folded literals
(concat/repeat/join/format/slice/reverse, base64/hex/rot13/zlib decode, and
a handful of pure builtins/str methods) is supported; anything else returns
None. ``env`` maps single-assignment module-level names to their RHS nodes.
"""
if _state is None:
_state = _FoldState(env)
_state.ops += 1
if node is None or _depth > _FOLD_DEPTH or _state.ops > _FOLD_OPS:
return None
if isinstance(node, ast.Constant):
v = node.value
if isinstance(v, (str, bytes, bytearray, int, float)) or v is None:
return _fold_cap(v)
return None
if isinstance(node, ast.Name):
rhs = _state.names.get(node.id)
if rhs is None:
return None
return _const_fold(rhs, None, _state, _depth + 1)
if isinstance(node, (ast.List, ast.Tuple)):
if len(node.elts) > _FOLD_MAX_SEQ:
return None
vals = []
for e in node.elts:
v = _const_fold(e, None, _state, _depth + 1)
if v is None and not (isinstance(e, ast.Constant) and e.value is None):
return None
vals.append(v)
return vals
if isinstance(node, ast.JoinedStr):
out = []
for part in node.values:
if isinstance(part, ast.Constant):
out.append(str(part.value))
elif isinstance(part, ast.FormattedValue):
v = _const_fold(part.value, None, _state, _depth + 1)
if v is None:
return None
spec = ""
if part.format_spec is not None:
spec = _const_fold(part.format_spec, None, _state, _depth + 1)
if spec is None:
return None
if part.conversion and part.conversion != -1:
try:
v = {114: repr, 115: str, 97: ascii}[part.conversion](v)
except Exception:
return None
if not _format_spec_ok(spec if isinstance(spec, str) else ""):
return None # oversized f-string width/precision: refuse pre-format
try:
out.append(format(v, spec if isinstance(spec, str) else ""))
except Exception:
return None
else:
return None
return _fold_cap("".join(out))
if isinstance(node, ast.BinOp):
left = _const_fold(node.left, None, _state, _depth + 1)
right = _const_fold(node.right, None, _state, _depth + 1)
if left is None or right is None:
return None
op = node.op
try:
if isinstance(op, ast.Mult):
if isinstance(left, (str, bytes, bytearray)) and isinstance(right, int):
if len(left) * max(right, 0) > _FOLD_MAXLEN:
return None
if isinstance(right, (str, bytes, bytearray)) and isinstance(left, int):
if len(right) * max(left, 0) > _FOLD_MAXLEN:
return None
# list/tuple repetition allocates len(seq)*n elements before _fold_cap
# (which only sizes str/bytes) can reject it -- cap it here too.
if isinstance(left, (list, tuple)) and isinstance(right, int):
if len(left) * max(right, 0) > _FOLD_MAX_SEQ:
return None
if isinstance(right, (list, tuple)) and isinstance(left, int):
if len(right) * max(left, 0) > _FOLD_MAX_SEQ:
return None
return _fold_cap(left * right)
if isinstance(op, ast.Add):
# str/bytes concat is sized by _fold_cap, but list/tuple concatenation is
# not, so a chain (a + a + a + ...) materializes an oversized sequence in the
# parent process before child rlimits apply. Cap the combined length.
if isinstance(left, (list, tuple)) and isinstance(right, (list, tuple)):
if len(left) + len(right) > _FOLD_MAX_SEQ:
return None
return _fold_cap(left + right)
if isinstance(op, ast.Mod):
if isinstance(left, (str, bytes, bytearray)) and not _printf_ok(left):
return None # oversized %-format width/precision: refuse pre-format
return _fold_cap(left % right)
if isinstance(op, ast.Sub):
return _fold_cap(left - right)
if isinstance(op, ast.FloorDiv):
return _fold_cap(left // right)
if isinstance(op, ast.Div):
return _fold_cap(left / right)
if isinstance(op, ast.BitXor):
return _fold_cap(left ^ right)
if isinstance(op, ast.BitOr):
return _fold_cap(left | right)
if isinstance(op, ast.BitAnd):
return _fold_cap(left & right)
if isinstance(op, ast.LShift) and isinstance(right, int) and 0 <= right < 64:
return _fold_cap(left << right)
if isinstance(op, ast.RShift) and isinstance(right, int) and 0 <= right < 64:
return _fold_cap(left >> right)
except Exception:
return None
return None # Pow and others: refuse (bignum DoS)
if isinstance(node, ast.UnaryOp):
v = _const_fold(node.operand, None, _state, _depth + 1)
if v is None:
return None
try:
return {
ast.USub: lambda x: -x,
ast.UAdd: lambda x: +x,
ast.Invert: lambda x: ~x,
ast.Not: lambda x: not x,
}[type(node.op)](v)
except Exception:
return None
if isinstance(node, ast.Subscript):
base = _const_fold(node.value, None, _state, _depth + 1)
if base is None or not isinstance(base, (str, bytes, bytearray, list, tuple)):
return None
sl = node.slice
try:
if isinstance(sl, ast.Slice):
lo = _const_fold(sl.lower, None, _state, _depth + 1) if sl.lower else None
hi = _const_fold(sl.upper, None, _state, _depth + 1) if sl.upper else None
st = _const_fold(sl.step, None, _state, _depth + 1) if sl.step else None
if (
(sl.lower is not None and lo is None)
or (sl.upper is not None and hi is None)
or (sl.step is not None and st is None)
):
return None
return _fold_cap(base[lo:hi:st])
idx = _const_fold(sl, None, _state, _depth + 1)
if not isinstance(idx, int):
return None
return _fold_cap(base[idx])
except Exception:
return None
if isinstance(node, ast.Call):
return _fold_call(node, _state, _depth)
return None
def _is_path_join_owner(nv):
"""AST for ``os.path`` (Attribute) or ``posixpath`` / ``ntpath`` (Name)."""
if (
isinstance(nv, ast.Attribute)
and nv.attr == "path"
and isinstance(nv.value, ast.Name)
and nv.value.id == "os"
):
return True
return isinstance(nv, ast.Name) and nv.id in ("posixpath", "ntpath")
def _fold_call(node, _state, _depth):
"""Fold a whitelisted pure builtin / method / decode call, else None."""
f = node.func
args = []
for a in node.args:
v = _const_fold(a, None, _state, _depth + 1)
if v is None and not (isinstance(a, ast.Constant) and a.value is None):
return None
args.append(v)
if isinstance(f, ast.Name):
name = f.id
if name not in _FOLD_PURE_BUILTINS:
return None
# A snippet that rebinds the builtin name (str = lambda _: '...'; eval(str(1))) makes the
# real-builtin fold diverge from runtime; refuse so the payload stays opaque (fail closed).
if name in _state.shadowed:
return None
try:
if name == "chr":
if len(args) == 1 and isinstance(args[0], int) and 0 <= args[0] <= 0x10FFFF:
return chr(args[0])
return None
if name == "ord":
if (
len(args) == 1
and isinstance(args[0], (str, bytes, bytearray))
and len(args[0]) == 1
):
return ord(args[0])
return None
fn = {
"str": str,
"bytes": bytes,
"bytearray": bytearray,
"int": int,
"hex": hex,
"oct": oct,
"bin": bin,
"bool": bool,
"float": float,
"len": len,
}[name]
if name in ("bytes", "bytearray") and len(args) == 1:
# bytes(n) / bytearray(n) allocate n zero bytes; refuse an oversized
# integer size before constructing it so static analysis of e.g.
# bytes(2_000_000_000) cannot OOM the Studio process (the child
# sandbox rlimits never get a chance to help during analysis).
if (
isinstance(args[0], int)
and not isinstance(args[0], bool)
and args[0] > _FOLD_MAXLEN
):
return None
return _fold_cap(fn(*args))
except Exception:
return None
if isinstance(f, ast.Attribute):
attr = f.attr
owner = f.value
# os.path.join('/etc', 'passwd') / posixpath.join(...) / ntpath.join(...):
# fold literal path builders so the sensitive-read scanner sees the concrete
# path (open(os.path.join('/etc','passwd')) must not be treated as opaque).
if attr == "join" and _is_path_join_owner(owner):
if args and all(isinstance(x, str) for x in args):
try:
return _fold_cap(os.path.join(*args))
except Exception:
return None
return None
# os.path.normpath / abspath on a literal reveal the same sensitive / traversal
# path a direct string would (open(os.path.normpath('a/../../etc/passwd')) must
# not stay opaque). normpath is a pure string transform; abspath is only foldable
# for an already-absolute arg (a relative abspath depends on the runtime cwd,
# which is the in-workdir sandbox cwd, so it need not be folded).
if attr in ("normpath", "abspath") and _is_path_join_owner(owner):
if len(args) == 1 and isinstance(args[0], str):
try:
if attr == "normpath" or os.path.isabs(args[0]):
return _fold_cap(os.path.normpath(args[0]))
except Exception:
return None
return None
if isinstance(owner, ast.Name):
mod = owner.id
# A rebound module receiver (base64 = <fake>; eval(base64.b64decode('...'))) would fold
# through the real stdlib module while runtime uses the user binding; refuse the fold.
if mod in _state.shadowed:
return None
try:
if mod == "base64" and attr in _FOLD_B64_FUNCS and len(args) >= 1:
return _fold_cap(getattr(base64, attr)(args[0]))
if (
mod == "codecs"
and attr in ("decode", "encode")
and len(args) >= 2
and isinstance(args[1], str)
):
return _fold_cap(_fold_apply_codec(args[1], args[0]))
if mod == "binascii" and attr in ("unhexlify", "a2b_hex") and len(args) >= 1:
return _fold_cap(binascii.unhexlify(args[0]))
if (
mod in ("bytes", "bytearray")
and attr == "fromhex"
and len(args) >= 1
and isinstance(args[0], str)
):
return _fold_cap(bytes.fromhex(args[0]))
except Exception:
return None
recv = _const_fold(owner, None, _state, _depth + 1)
if isinstance(recv, (str, bytes, bytearray)) and attr in _FOLD_STR_METHODS:
try:
kwargs = {}
for kw in node.keywords:
if kw.arg is None:
return None
kv = _const_fold(kw.value, None, _state, _depth + 1)
if kv is None:
return None
kwargs[kw.arg] = kv
call_args = []
for a in args:
call_args.append(
list(a) if attr == "join" and isinstance(a, (list, tuple)) else a
)
# Padding methods take a width as their first arg; str.format takes a
# template with per-field widths. Reject an oversized width before the
# method allocates the padded string during folding.
if attr in ("center", "ljust", "rjust", "zfill") and call_args:
if _too_wide(call_args[0]):
return None
if attr == "replace" and not _replace_output_ok(recv, call_args):
return None
if attr == "join" and not _join_output_ok(recv, call_args):
return None
if attr == "format":
if not _format_template_ok(recv):
return None
# Nested-width field ({:{}}) whose width comes from a large numeric
# arg: refuse before format() allocates.
if _format_has_nested_spec(recv) and any(
_too_wide(a) for a in list(call_args) + list(kwargs.values())
):
return None
return _fold_cap(getattr(recv, attr)(*call_args, **kwargs))
except Exception:
return None
return None
def _build_const_prop_env(tree):
"""Names bound exactly once by a module-level ``name = <expr>`` (single Name
target), never re-assigned / aug-assigned / declared global-nonlocal / used as
a loop / comprehension / with / except target. Maps name -> RHS node.
Conservative: any ambiguity excludes the name. Only module-level statements are
considered so a name shadowed inside a def / loop is never folded.
"""
assigned_once: dict[str, ast.expr] = {}
disqualified: set[str] = set()
def _disqualify_targets(target):
for n in ast.walk(target):
if isinstance(n, ast.Name):
disqualified.add(n.id)
# Module-level single assignments.
for stmt in getattr(tree, "body", []):
if (
isinstance(stmt, ast.Assign)
and len(stmt.targets) == 1
and isinstance(stmt.targets[0], ast.Name)
):
name = stmt.targets[0].id
if name in assigned_once or name in disqualified:
disqualified.add(name)
assigned_once.pop(name, None)
else:
assigned_once[name] = stmt.value
elif isinstance(stmt, ast.Assign):
for t in stmt.targets:
_disqualify_targets(t)
elif isinstance(stmt, (ast.AugAssign, ast.AnnAssign)):
if getattr(stmt, "target", None) is not None:
_disqualify_targets(stmt.target)
# Any name that is ALSO written anywhere else (loops, defs, walrus, aug, params,
# comprehension targets, with/except/for) is disqualified.
for n in ast.walk(tree):
if isinstance(n, ast.Name) and isinstance(n.ctx, (ast.Store, ast.Del)):
nm = n.id
if nm in assigned_once:
# It is stored somewhere; allow only if that single store is the
# module-level assign we recorded (identity check below).
pass
if isinstance(n, (ast.AugAssign,)):
_disqualify_targets(n.target)
elif isinstance(n, ast.NamedExpr):
_disqualify_targets(n.target)
elif isinstance(n, (ast.For, ast.AsyncFor)):
_disqualify_targets(n.target)
elif isinstance(n, ast.comprehension):
_disqualify_targets(n.target)
elif isinstance(n, ast.withitem):
if n.optional_vars is not None:
_disqualify_targets(n.optional_vars)
elif isinstance(n, ast.ExceptHandler):
if n.name:
disqualified.add(n.name)
elif isinstance(n, (ast.Global, ast.Nonlocal)):
for nm in n.names:
disqualified.add(nm)
elif isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
disqualified.add(n.name)
args = getattr(n, "args", None)
if args is not None:
for a in list(args.args) + list(args.posonlyargs) + list(args.kwonlyargs):
disqualified.add(a.arg)
for extra in (args.vararg, args.kwarg):
if extra is not None:
disqualified.add(extra.arg)
# A write THROUGH the namespace dict (globals()['x'] = BAD, vars()['x'] = BAD, locals()[...]
# = ..., or globals().update(...) / .setdefault(...) / .__setitem__(...)) mutates a module
# variable with NO Name Store, so a folded constant would be stale and the recovered exec/eval
# payload wrong. Invalidate the affected name (constant key) or, for a dynamic key / bulk
# update, every recorded name -- the snippet is manipulating the namespace opaquely.
def _is_namespace_call(nv):
return (
isinstance(nv, ast.Call)
and isinstance(nv.func, ast.Name)
and nv.func.id in ("globals", "vars", "locals")
)
_ns_write_all = False
_ns_write_names: set[str] = set()
for n in ast.walk(tree):
# globals()[key] = ... (Assign target or AugAssign target).
_subs = []
if isinstance(n, ast.Assign):
_subs = [t for t in n.targets if isinstance(t, ast.Subscript)]
elif isinstance(n, (ast.AugAssign, ast.AnnAssign)):
if isinstance(getattr(n, "target", None), ast.Subscript):
_subs = [n.target]
for _t in _subs:
if not _is_namespace_call(_t.value):
continue
_key = _t.slice.value if isinstance(_t.slice, ast.Constant) else None
if isinstance(_key, str):
_ns_write_names.add(_key)
else:
_ns_write_all = True
# globals().update(...) / .setdefault(...) / .__setitem__(...) -- an opaque bulk write.
if (
isinstance(n, ast.Call)
and isinstance(n.func, ast.Attribute)
and n.func.attr in ("update", "setdefault", "__setitem__", "pop", "clear")
and _is_namespace_call(n.func.value)
):
_ns_write_all = True
# Count how many module-level stores each recorded name really has; if more
# than one Store target references it anywhere, drop it.
store_counts: dict[str, int] = {}
for n in ast.walk(tree):
if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Store):
store_counts[n.id] = store_counts.get(n.id, 0) + 1
# Names REBOUND to something OTHER than their canonical builtin / stdlib module: a fold that
# calls the real builtin (str(...), len(...)) or hard-coded module (base64.b64decode(...))
# would diverge from runtime, which calls the user binding. A plain `import name` keeps the
# canonical module (NOT shadowing); every other binding -- assignment, def/class, param,
# from-import, an aliased import that rebinds the name, or a loop/with/except/comprehension
# target -- is. The folder consults this set before folding a Name builtin / module receiver.
shadowed: set[str] = set()
def _shadow_targets(t):
for nn in ast.walk(t):
if isinstance(nn, ast.Name) and isinstance(nn.ctx, (ast.Store, ast.Del)):
shadowed.add(nn.id)
for n in ast.walk(tree):
if isinstance(n, ast.Assign):
for t in n.targets:
_shadow_targets(t)
elif isinstance(n, (ast.AugAssign, ast.AnnAssign)):
if getattr(n, "target", None) is not None:
_shadow_targets(n.target)
elif isinstance(n, ast.NamedExpr):
_shadow_targets(n.target)
elif isinstance(n, (ast.For, ast.AsyncFor)):
_shadow_targets(n.target)
elif isinstance(n, ast.comprehension):
_shadow_targets(n.target)
elif isinstance(n, ast.withitem):
if n.optional_vars is not None:
_shadow_targets(n.optional_vars)
elif isinstance(n, ast.ExceptHandler):
if n.name:
shadowed.add(n.name)
elif isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
shadowed.add(n.name)
_a = getattr(n, "args", None)
if _a is not None:
for _p in list(_a.args) + list(_a.posonlyargs) + list(_a.kwonlyargs):
shadowed.add(_p.arg)
for _extra in (_a.vararg, _a.kwarg):
if _extra is not None:
shadowed.add(_extra.arg)
elif isinstance(n, ast.ImportFrom):
for _al in n.names:
shadowed.add(_al.asname or _al.name)
elif isinstance(n, ast.Import):
for _al in n.names:
# import os as base64 rebinds `base64` to a different module; a plain
# `import base64` (asname None) keeps the canonical module and does not shadow.
if _al.asname is not None and _al.asname != _al.name:
shadowed.add(_al.asname)
env = _ConstEnv(shadowed = frozenset(shadowed))
if _ns_write_all:
return env # an opaque namespace mutation could rebind any recorded constant
disqualified |= _ns_write_names
for name, rhs in assigned_once.items():
if name in disqualified:
continue
if store_counts.get(name, 0) != 1:
continue
env[name] = rhs
return env
# --------------------------------------------------------------------------
# Stage 2: eval / exec / compile recursive payload analysis.
# --------------------------------------------------------------------------
_MAX_UNWRAP_DEPTH = 5
_MAX_INNER_SRC = 200 * 1024
_MAX_INNER_PARSES = 25
_MAX_TOTAL_INNER_CHARS = 2 * 1024 * 1024
_MAX_BRACKET_DEPTH = 200
_MAX_ANALYZER_NODES = 200_000
_EXEC_BUILTINS = frozenset({"eval", "exec", "compile"})
# Deserialization sinks that reconstruct/execute arbitrary objects from bytes.
_CODE_DESERIALIZE_SINKS = frozenset(
{
"pickle.loads",
"pickle.load",
"marshal.loads",
"marshal.load",
"dill.loads",
"dill.load",
"cloudpickle.loads",
"cloudpickle.load",
"_pickle.loads",
"_pickle.load",
"jsonpickle.decode",
# PyYAML: unsafe_load / full_load construct arbitrary Python objects from
# `!!python/object/apply:os.system [...]`. yaml.load is conditional (safe only with a
# SafeLoader) and is handled separately.
"yaml.unsafe_load",
"yaml.unsafe_load_all",
"yaml.full_load",
"yaml.full_load_all",
}
)
# Modules whose load/loads/decode entry points run a pickle reduce payload; used to
# resolve `import pickle as p; p.loads(x)` and `from pickle import loads as l`.
_DESERIALIZE_MODULES = frozenset(
{"pickle", "marshal", "dill", "cloudpickle", "_pickle", "jsonpickle", "yaml"}
)
# Modules exposing an Unpickler class whose .load() runs the same reduce payload as *.load:
# pickle.Unpickler(f).load() / dill.Unpickler(f).load() bypass the *.load sink-name check.
_UNPICKLER_MODULES = frozenset({"pickle", "_pickle", "dill", "cloudpickle"})
# yaml.load / yaml.load_all construct arbitrary objects UNLESS given a safe loader; flag them
# when the Loader is absent or is one of the unsafe loader classes.
_YAML_SAFE_LOADERS = frozenset({"SafeLoader", "CSafeLoader", "BaseLoader"})
_YAML_LOAD_METHODS = frozenset({"load", "load_all"})
# Pickle-backed loaders whose reduce payload executes arbitrary code, gated by a keyword like
# yaml.load: torch.load runs a pickle unless weights_only is True (torch>=2.6 default), numpy.load
# only unpickles with allow_pickle=True, and joblib.load is always pickle-backed. Tracked
# separately from _CODE_DESERIALIZE_SINKS because the safe default forms must stay allowed.
_PICKLE_LOADER_MODULES = frozenset({"torch", "numpy", "joblib"})
def _kw_constant_truthy(node, name):
"""The literal truthiness of keyword ``name`` in a call: True/False when it is a constant,
None when absent or non-constant. Used for the weights_only / allow_pickle gates."""
for kw in node.keywords:
if kw.arg == name:
if isinstance(kw.value, ast.Constant):
return bool(kw.value.value)
return None
return None
def _kw_present(node, name):
"""True when keyword ``name`` is passed in the call (any value)."""
return any(kw.arg == name for kw in node.keywords)
def _numpy_allow_pickle_unsafe(node):
"""True when numpy.load may unpickle: allow_pickle is a constant truthy, a present-but-non-
constant value (flag=True), or hidden in a **kwargs splat we cannot prove absent/False.
allow_pickle absent (default False) or a constant False stays safe."""
for kw in node.keywords:
if kw.arg == "allow_pickle":
if isinstance(kw.value, ast.Constant):
return bool(kw.value.value)
return True # non-literal value cannot be proven False
if kw.arg is None:
# **{...} / **var splat: inspect a literal dict, else fail closed.
if isinstance(kw.value, ast.Dict):
for _k, _v in zip(kw.value.keys, kw.value.values):
if isinstance(_k, ast.Constant) and _k.value == "allow_pickle":
if not isinstance(_v, ast.Constant) or bool(_v.value):
return True
else:
return True # opaque **var could carry allow_pickle=True
return False
def _pickle_loader_is_unsafe(fq, node):
"""True when a torch.load / numpy.load / joblib.load call runs an UNVERIFIED pickle payload:
joblib.load always does; torch.load when weights_only is EXPLICITLY not-True (torch>=2.6
defaults it to True, so the bare torch.load(f) form relies on that safe default and stays
allowed); numpy.load when allow_pickle is truthy / non-literal / splatted (see
_numpy_allow_pickle_unsafe). So the safe forms (torch.load(f), torch.load(f,
weights_only=True), numpy.load(f), numpy.load(f, allow_pickle=False)) return False."""
if fq == "joblib.load":
return True
if fq == "torch.load":
return (
_kw_present(node, "weights_only")
and _kw_constant_truthy(node, "weights_only") is not True
)
if fq == "numpy.load":
return _numpy_allow_pickle_unsafe(node)
return False
def _yaml_loader_class_name(value):
"""Terminal attribute/name of a Loader= argument (yaml.SafeLoader -> SafeLoader)."""
if isinstance(value, ast.Attribute):
return value.attr
if isinstance(value, ast.Name):
return value.id
return None
def _yaml_call_has_safe_loader(node):
"""True only when a yaml.load(...) call passes an explicit safe loader.
PyYAML's signature is load(stream, Loader), so the loader may be the SECOND POSITIONAL
argument (yaml.load(data, yaml.SafeLoader)) or the Loader= keyword. A missing loader (older
PyYAML defaults to the full, unsafe loader), an unknown/computed loader, or a **kwargs splat
all fail closed so the call is treated as an unsafe sink.
"""
if len(node.args) >= 2:
return _yaml_loader_class_name(node.args[1]) in _YAML_SAFE_LOADERS
for kw in node.keywords:
if kw.arg == "Loader":
return _yaml_loader_class_name(kw.value) in _YAML_SAFE_LOADERS
if kw.arg is None:
# **kwargs unpacking hides the loader; cannot prove it is safe.
return False
return False
# Attribute names of pure decode/decompress primitives used to hide a payload.
_DECODE_ATTRS = frozenset(
{
"b64decode",
"b64encode",
"urlsafe_b64decode",
"standard_b64decode",
"b32decode",
"b16decode",
"a85decode",
"b85decode",
"decodebytes",
"fromhex",
"unhexlify",
"a2b_hex",
"a2b_base64",
"decompress",
}
)
_FETCH_FQ_PREFIXES = (
"requests.",
"urllib.",
"httpx.",
"socket.",
"aiohttp.",
"urllib3.",
"http.client.",
)
# Constant attribute names that, resolved off a sensitive module via getattr, still
# reach shell / process / delete / dynamic-import / code-exec capabilities. A benign
# constant attr (getpid, path, sep, getcwd, ...) is allowed; a dynamic attr blocks.
_DANGEROUS_ATTR_NAMES = frozenset(
{
"system",
"popen",
"popen2",
"popen3",
"popen4",
"execl",
"execle",
"execlp",
"execlpe",
"execv",
"execve",
"execvp",
"execvpe",
"spawnl",
"spawnle",
"spawnlp",
"spawnlpe",
"spawnv",
"spawnve",
"spawnvp",
"spawnvpe",
"posix_spawn",
"posix_spawnp",
"startfile",
"fork",
"forkpty",
"remove",
"unlink",
"rmdir",
"removedirs",
"rename",
"renames",
"replace",
"truncate",
"chmod",
"lchmod",
"chown",
"lchown",
"chflags",
"mkdir",
"makedirs",
"mknod",
"symlink",
"link",
"chdir",
"chroot",
"import_module",
"__import__",
"reload",
"eval",
"exec",
"compile",
"run",
"call",
"check_call",
"check_output",
"Popen",
"getoutput",
"getstatusoutput",
"load_module",
"exec_module",
"loads",
"load",
}
)
class _AnalyzerBudget:
"""Shared, bounded counters across one classification (incl. exec recursion)."""
__slots__ = ("inner_parses", "inner_chars", "nodes")
def __init__(self):
self.inner_parses = 0
self.inner_chars = 0
self.nodes = 0
def _reexport_dangerous_module_name(expr):
"""For a re-export gadget that fetches a submodule by NAME off some object and then calls a
sink on it, return the fetched module name normalized to 'os' / 'subprocess' (or None).
Covers ``getattr(<mod>, 'os')``, ``vars(<mod>)['os']`` and ``<mod>.__dict__['os']`` -- the
call / subscript twins of the plain ``<mod>.os`` attribute form (pathlib.os.system), which
stay reachable off a call-returned module (``getattr(__import__('pathlib'), 'os').system``)."""
def _str_const(n):
return n.value if isinstance(n, ast.Constant) and isinstance(n.value, str) else None
key = None
if (
isinstance(expr, ast.Call)
and isinstance(expr.func, ast.Name)
and expr.func.id == "getattr"
and len(expr.args) >= 2
):
key = _str_const(expr.args[1])
elif isinstance(expr, ast.Subscript):
base = expr.value
if (
isinstance(base, ast.Call)
and isinstance(base.func, ast.Name)
and base.func.id == "vars"
) or (isinstance(base, ast.Attribute) and base.attr == "__dict__"):
key = _str_const(expr.slice)
if key in ("os", "posix"):
return "os"
if key == "subprocess":
return "subprocess"
return None
def _fq_attr_name(node):
"""Return the dotted name for a Name/Attribute chain, else ''."""
parts = []
cur = node
while isinstance(cur, ast.Attribute):
parts.append(cur.attr)
cur = cur.value
if isinstance(cur, ast.Name):
parts.append(cur.id)
return ".".join(reversed(parts))
return ""
def _bracket_depth(s):
"""Linear max bracket-nesting scan; never invokes the C parser (DoS-safe)."""
depth = mx = 0
for ch in s:
if ch in "([{":
depth += 1
if depth > mx:
mx = depth
elif ch in ")]}":
depth = depth - 1 if depth > 0 else 0
return mx
def _to_text(value):
if isinstance(value, (bytes, bytearray)):
try:
return value.decode("utf-8")
except Exception:
return value.decode("latin-1", "replace")
return value
# PEP 263 source-encoding cookie ("# -*- coding: utf-8 -*-", "# coding: utf_7").
_CODING_COOKIE_RE = re.compile(rb"coding[:=]\s*([-\w.]+)")
_CODING_COOKIE_TEXT_RE = re.compile(r"coding[:=]\s*([-\w.]+)")
def _decode_source_bytes(data):
"""Decode an exec/compile *bytes* payload the way CPython would.
exec()/eval()/compile() honor a PEP 263 coding cookie on bytes, so the analyzer
must decode with that cookie's codec (not a fixed UTF-8 view) or a snippet like
``exec(b"# coding: utf_7\\n#+AAo-__import__('os').system('id')")`` reads as pure
comments under UTF-8 while actually running hidden code. Detect the encoding,
decode, then neutralize the cookie so ast.parse(str) does not reject the decoded
text (a str carrying a coding declaration raises SyntaxError), preserving line
numbers so the recursive analysis sees the real source.
"""
data = bytes(data)
enc = "utf-8"
try:
import io as _io_mod
import tokenize as _tok
enc, _ = _tok.detect_encoding(_io_mod.BytesIO(data).readline)
except Exception:
enc = "utf-8"
for _cand in (enc, "utf-8"):
try:
text = data.decode(_cand)
break
except Exception:
text = None
if text is None:
text = data.decode("latin-1", "replace")
lines = text.split("\n")
for _i in range(min(2, len(lines))):
if _CODING_COOKIE_TEXT_RE.search(lines[_i]):
lines[_i] = _CODING_COOKIE_TEXT_RE.sub("coding_neutralized", lines[_i], count = 1)
return "\n".join(lines)
def _recovered_source(v):
"""Text an exec/compile sink actually runs: cookie-aware decode for bytes."""
if isinstance(v, (bytes, bytearray)):
return _decode_source_bytes(v)
return _to_text(v)
def _compile_source_node(node):
"""The SOURCE argument of a compile() call: the 1st positional arg or the ``source=``
keyword. compile() accepts its payload either way, so a keyword-only call
(compile(source='...', filename='<p>', mode='exec')) must still be recovered."""
if node.args:
return node.args[0]
for kw in node.keywords or []:
if kw.arg == "source":
return kw.value
return None
def _compile_mode(node, const_env):
"""Recover a compile()'s literal mode= (3rd positional or keyword), else 'exec'."""
mode_node = None
if len(node.args) >= 3:
mode_node = node.args[2]
for kw in node.keywords or []:
if kw.arg == "mode":
mode_node = kw.value
if mode_node is not None:
v = _const_fold(mode_node, const_env)
if v in ("eval", "exec", "single"):
return "eval" if v == "eval" else "exec"
return "exec"
def _walk_scope_local(scope):
"""Yield descendants of ``scope``'s body that share its namespace, WITHOUT
descending into nested def / lambda / class / comprehension (each of which is a
new scope). Used so single-assignment alias detection is scope-correct.
When ``scope`` is itself a lambda or comprehension, walk its own namespace: a lambda
body is a single expression, and a comprehension's namespace holds its element
expression plus the generator iterables / conditions (target bindings are collected
separately). ``getattr(scope, "body", [])`` only applies to def / class / module."""
if isinstance(scope, ast.Lambda):
stack = [scope.body]
elif isinstance(scope, (ast.ListComp, ast.SetComp, ast.GeneratorExp)):
stack = [scope.elt] + [g for gen in scope.generators for g in [gen.iter, *gen.ifs]]
elif isinstance(scope, ast.DictComp):
stack = [scope.key, scope.value] + [
g for gen in scope.generators for g in [gen.iter, *gen.ifs]
]
else:
stack = list(getattr(scope, "body", []))
_NESTED = (
ast.FunctionDef,
ast.AsyncFunctionDef,
ast.Lambda,
ast.ClassDef,
ast.ListComp,
ast.SetComp,
ast.DictComp,
ast.GeneratorExp,
)
while stack:
n = stack.pop()
# A nested def / lambda / class / comprehension opens its OWN scope: its body
# neither shares this namespace nor should its stores be counted here. Skip it
# entirely -- do not yield it or descend into it. (Checking the popped node,
# not just its children, is what keeps a nested `def f(): s = print` from
# inflating the outer count of an `s = os.system` single-assignment alias.)
if isinstance(n, _NESTED):
continue
yield n
for child in ast.iter_child_nodes(n):
stack.append(child)
class _ScopeAliasIndex:
"""Per-scope single-assignment aliases (shell sink / exec builtin / compiled
source) resolved with Python lexical scoping. Counting and resolution are per
function scope, so two functions binding the same local name neither cancel out
(a real sink would be missed) nor cross-contaminate (a benign call in one function
would be flagged, or a dynamic exec in another wrongly treated as a safe alias)."""
__slots__ = (
"tree",
"node_scope",
"enclosing",
"shell",
"execb",
"compiled",
"compiledany",
"impf",
"deser",
"strconst",
"rhsnode",
"assigned",
"class_shell",
"class_execb",
"class_deser",
"class_bases",
"instance_shell",
"instance_execb",
"instance_deser",
)
def __init__(self, tree):
self.tree = tree
self.node_scope: dict = {tree: tree}
self.enclosing: dict = {tree: None}
self.shell: dict = {}
self.execb: dict = {}
self.compiled: dict = {}
# name -> True: bound to a compile() result (foldable OR dynamic). Used to catch
# a code object executed through types.FunctionType(c, ...) after `c = compile(...)`.
self.compiledany: dict = {}
self.impf: dict = {} # name -> True: alias of __import__ / importlib.import_module
self.deser: dict = {} # name -> fq deserializer sink (pickle.loads, ...)
self.strconst: dict = {} # name -> folded str/bytes constant (for read scanning)
self.rhsnode: dict = {} # name -> single-assignment RHS node (for pathlib reads)
self.assigned: dict = {}
# class NAME -> {attr: sink}: a class-body alias (class C: f = os.system) accessed
# as C.f from outside the class, which lexical scope resolution does not cover.
self.class_shell: dict = {}
self.class_execb: dict = {}
self.class_deser: dict = {}
# class NAME -> [base class names]: literal Name bases, so a subclass access
# (class D(C): pass; D.s) can follow inheritance to a base's class-body sink alias.
self.class_bases: dict = {}
# (receiver_name, attr) -> sink: a simple instance-attribute alias assigned a
# dangerous callable (c.e = exec; c.e(payload) / obj.s = os.system; obj.s('rm -rf /')).
# Tracked tree-wide as a fail-closed over-approximation (attribute values are not
# lexically scoped), so a call through the same receiver name + attr is analyzed.
self.instance_shell: dict = {}
self.instance_execb: dict = {}
self.instance_deser: dict = {}
def resolve_class_attr(
self,
cname,
attr,
kind,
_seen = None,
):
table = getattr(self, "class_" + kind)
m = table.get(cname)
if m and attr in m:
return m[attr]
# Follow literal base classes so an inherited alias (class C: s = os.system;
# class D(C): pass; D.s(...)) resolves through C. Cycle-guarded.
if _seen is None:
_seen = set()
if cname in _seen:
return None
_seen.add(cname)
for _base in self.class_bases.get(cname, ()):
hit = self.resolve_class_attr(_base, attr, kind, _seen)
if hit is not None:
return hit
return None
def resolve_instance_attr(self, recv, attr, kind):
return getattr(self, "instance_" + kind).get((recv, attr))
def _chain(self, node):
s = self.node_scope.get(node, self.tree)
while s is not None:
yield s
s = self.enclosing.get(s)
def resolve(self, name, node, kind):
maps = getattr(self, kind)
for s in self._chain(node):
m = maps.get(s)
if m and name in m:
return m[name]
if name in self.assigned.get(s, ()): # locally shadowed by a non-alias
return None
return None
def effective(self, node, kind):
maps = getattr(self, kind)
result: dict = {}
shadowed: set = set()
for s in self._chain(node):
for k, v in maps.get(s, {}).items():
if k not in shadowed and k not in result:
result[k] = v
shadowed |= self.assigned.get(s, set())
return result
def _build_scope_alias_index(tree, const_env):
idx = _ScopeAliasIndex(tree)
def _rec(node, scope, func_enclose):
# scope: namespace the direct children belong to (for node_scope + counting).
# func_enclose: the scope a nested FUNCTION / class body encloses to. Python skips
# class scope for nested functions, so inside a class body this stays the class's
# own lexical function/module parent rather than the class.
for child in ast.iter_child_nodes(node):
idx.node_scope[child] = scope
if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)):
# A lambda, like a def, opens its own scope: a default-bound param alias
# ((lambda e=exec: e(payload))()) lives in the lambda body's namespace, and
# anything nested inside encloses to the lambda itself.
idx.enclosing[child] = func_enclose
_rec(child, child, child)
elif isinstance(child, (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp)):
# A comprehension opens its own scope in Python 3; its target bindings
# ([e(p) for e in [exec]]) belong to that scope, enclosing to func_enclose.
idx.enclosing[child] = func_enclose
_rec(child, child, func_enclose)
elif isinstance(child, ast.ClassDef):
# A class body executes immediately with its OWN namespace, so it is a
# real alias scope (class C: e = eval; e(...) runs eval), but its names
# are not visible to methods defined inside it -- those enclose to
# func_enclose, skipping the class.
idx.enclosing[child] = func_enclose
_rec(child, child, func_enclose)
else:
_rec(child, scope, func_enclose)
_rec(tree, tree, tree)
# os / subprocess import + from-import aliases are collected tree-wide (imports
# are lexically visible module-wide in practice) and shared across scopes.
os_aliases = {"os"}
subprocess_aliases = {"subprocess"}
from_aliases: dict[str, str] = {}
builtins_aliases = {"builtins", "__builtins__"}
importlib_aliases = {"importlib"}
# `compile` bound by name (bare builtin or `from builtins import compile as comp`).
compile_aliases = {"compile"}
deser_module_aliases: dict[str, str] = {m: m for m in _DESERIALIZE_MODULES}
for n in ast.walk(tree):
if isinstance(n, ast.Import):
for a in n.names:
if a.name == "os":
os_aliases.add(a.asname or "os")
elif a.name in ("posix", "nt"):
# posix / nt are the C backend os wraps (posix.system == os.system), so a
# single-assignment alias s = posix.system resolves to an os shell sink.
os_aliases.add(a.asname or a.name)
elif a.name == "subprocess":
subprocess_aliases.add(a.asname or "subprocess")
elif a.name == "builtins":
builtins_aliases.add(a.asname or "builtins")
elif a.name == "importlib":
importlib_aliases.add(a.asname or "importlib")
if a.name in _DESERIALIZE_MODULES:
deser_module_aliases[a.asname or a.name] = a.name
elif isinstance(n, ast.ImportFrom) and n.module in ("os", "subprocess", "posix", "nt"):
_eff = "subprocess" if n.module == "subprocess" else "os"
for a in n.names:
fq = f"{_eff}.{a.name}"
if fq in _SHELL_SINK_FUNCS:
from_aliases[a.asname or a.name] = fq
elif isinstance(n, ast.ImportFrom) and n.module == "builtins":
for a in n.names:
if a.name == "compile":
compile_aliases.add(a.asname or "compile")
def _rhs_is_compile_call(rhs):
# `compile(...)` reached as the bare builtin, `builtins.compile(...)`, or a
# `from builtins import compile as comp` alias -- the callee forms that produce a
# code object bound to a name (for the types.FunctionType(c) execution gadget).
if not isinstance(rhs, ast.Call):
return False
f = rhs.func
if isinstance(f, ast.Name):
return f.id in compile_aliases
if (
isinstance(f, ast.Attribute)
and f.attr == "compile"
and isinstance(f.value, ast.Name)
and f.value.id in builtins_aliases
):
return True
return False
def _rhs_exec_builtin(rhs):
# bare `exec` / `eval` / `compile`, or `builtins.eval` (attribute form).
if isinstance(rhs, ast.Name) and rhs.id in _EXEC_BUILTINS:
return rhs.id
if (
isinstance(rhs, ast.Attribute)
and rhs.attr in _EXEC_BUILTINS
and isinstance(rhs.value, ast.Name)
and rhs.value.id in builtins_aliases
):
return rhs.attr
return None
def _rhs_import_func(rhs):
# `__import__` / `importlib.import_module` (+ reload) bound to a name.
if isinstance(rhs, ast.Name) and rhs.id in ("__import__", "import_module"):
return True
if (
isinstance(rhs, ast.Attribute)
and rhs.attr in ("import_module", "reload", "__import__")
and isinstance(rhs.value, ast.Name)
and rhs.value.id in importlib_aliases
):
return True
return False
def _rhs_deserializer(rhs):
# `pickle.loads` (+ aliased module) bound to a name.
if isinstance(rhs, ast.Attribute) and isinstance(rhs.value, ast.Name):
canon = deser_module_aliases.get(rhs.value.id)
if canon is not None:
fq = f"{canon}.{rhs.attr}"
if fq in _CODE_DESERIALIZE_SINKS:
return fq
return None
def _unwrap_container_index(rhs):
# `s = [os.system][0]` / `e = {'e': exec}['e']` / `l = (pickle.loads,)[0]`: the
# unwrapped callable is assigned first, then called. Resolve the inline
# literal-container index to the element node so the sink resolvers below see the
# real callable instead of an opaque Subscript.
if not isinstance(rhs, ast.Subscript):
return rhs
container = rhs.value
ci = _const_fold(rhs.slice, const_env)
if isinstance(container, (ast.List, ast.Tuple)) and isinstance(ci, int):
if -len(container.elts) <= ci < len(container.elts):
return container.elts[ci]
if isinstance(container, ast.Dict) and ci is not None:
for k, v in zip(container.keys, container.values):
if k is not None and _const_fold(k, const_env) == ci:
return v
return rhs
scopes = [tree] + [
n
for n in ast.walk(tree)
if isinstance(
n,
(
ast.FunctionDef,
ast.AsyncFunctionDef,
ast.ClassDef,
ast.Lambda,
ast.ListComp,
ast.SetComp,
ast.DictComp,
ast.GeneratorExp,
),
)
]
for scope in scopes:
counts: dict[str, int] = {}
rebound: set[str] = set()
global_names: set[str] = set()
nonlocal_names: set[str] = set()
assigns: list[tuple[str, ast.expr]] = []
allnames: set[str] = set()
# Function parameters bind local names that lexically shadow an outer alias of
# the same name, so count them as local assignments for the shadowing rules.
_sargs = getattr(scope, "args", None)
if _sargs is not None:
for _a in list(_sargs.posonlyargs) + list(_sargs.args) + list(_sargs.kwonlyargs):
allnames.add(_a.arg)
for _extra in (_sargs.vararg, _sargs.kwarg):
if _extra is not None:
allnames.add(_extra.arg)
for n in _walk_scope_local(scope):
if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Store):
counts[n.id] = counts.get(n.id, 0) + 1
allnames.add(n.id)
elif isinstance(n, ast.Global):
rebound.update(n.names)
global_names.update(n.names)
elif isinstance(n, ast.Nonlocal):
rebound.update(n.names)
nonlocal_names.update(n.names)
elif (
isinstance(n, ast.Assign)
and len(n.targets) == 1
and isinstance(n.targets[0], ast.Name)
):
assigns.append((n.targets[0].id, n.value))
elif (
# An annotated single-assignment (e: object = exec) is a real binding whose
# RHS must be recorded like a plain Assign, else e(payload) skips analysis.
isinstance(n, ast.AnnAssign)
and n.value is not None
and isinstance(n.target, ast.Name)
):
assigns.append((n.target.id, n.value))
elif (
# A parallel unpacking assignment binds each name to the matching RHS element
# ((s,) = (os.system,); [e] = [exec]; a, b = os.system, 1), which then reaches
# a sink call the same way a plain alias does. Pair a literal tuple/list target
# with a literal tuple/list RHS of equal length element-wise so those aliases
# are recorded; a starred / mismatched / non-literal RHS is left alone.
isinstance(n, ast.Assign)
and len(n.targets) == 1
and isinstance(n.targets[0], (ast.Tuple, ast.List))
and isinstance(n.value, (ast.Tuple, ast.List))
and len(n.targets[0].elts) == len(n.value.elts)
and not any(isinstance(_t, ast.Starred) for _t in n.targets[0].elts)
):
for _tgt, _val in zip(n.targets[0].elts, n.value.elts):
if isinstance(_tgt, ast.Name):
assigns.append((_tgt.id, _val))
# A comprehension generator binds its target like a single-assignment alias when the
# iterable is a one-element literal: [e(p) for e in [exec]] binds e to exec, so the
# payload passed through e must still get eval/exec recursion.
for _gen in getattr(scope, "generators", []):
if (
isinstance(_gen.target, ast.Name)
and isinstance(_gen.iter, (ast.List, ast.Tuple, ast.Set))
and len(_gen.iter.elts) == 1
):
_tn = _gen.target.id
counts[_tn] = counts.get(_tn, 0) + 1
allnames.add(_tn)
assigns.append((_tn, _gen.iter.elts[0]))
# A `global`/`nonlocal`-declared name is NOT a local binding, so it must not shadow an
# outer alias here (its assignment rebinds the target scope instead).
allnames -= rebound
idx.assigned[scope] = allnames
smap: dict[str, str] = {}
emap: dict[str, str] = {}
cmap: dict[str, tuple] = {}
camap: dict[str, bool] = {}
imap: dict[str, bool] = {}
dmap: dict[str, str] = {}
scmap: dict[str, object] = {}
rnmap: dict[str, ast.expr] = {}
# Process assignments in SOURCE order so a chained single-assignment alias resolves
# against the earlier binding it copies (s = os.system; t = s -> t is os.system). The
# scope walk yields assignments out of order, so sort by the RHS position; Python
# binds top-to-bottom, so the aliased name is always defined on an earlier line.
assigns.sort(key = lambda _p: (getattr(_p[1], "lineno", 0), getattr(_p[1], "col_offset", 0)))
for name, rhs in assigns:
if counts.get(name) != 1 or name in rebound:
continue
# Single-assignment RHS node, used by the read scanner to resolve a pathlib
# expression bound to a name (p = Path('..') / 'etc' / 'passwd'; p.read_text()).
rnmap[name] = rhs
# An inline-container index RHS (s = [os.system][0]) hides the callable; resolve
# it to the element so the sink resolvers below see the real sink.
rhs_eff = _unwrap_container_index(rhs)
fq = _resolve_static_shell_sink(rhs_eff, os_aliases, subprocess_aliases, from_aliases)
# A chained single-assignment alias (s = os.system; t = s; t('rm -rf /')): the RHS
# is a bare Name already resolved to a sink earlier in this scope (assigns are in
# source order), so propagate its sink identity instead of dropping it.
if fq is None and isinstance(rhs_eff, ast.Name) and rhs_eff.id in smap:
fq = smap[rhs_eff.id]
if fq:
smap[name] = fq
eb = _rhs_exec_builtin(rhs_eff)
if eb is None and isinstance(rhs_eff, ast.Name) and rhs_eff.id in emap:
eb = emap[rhs_eff.id] # chained alias e = exec; f = e; f(payload)
if eb is not None:
emap[name] = eb
elif (
_rhs_is_compile_call(rhs_eff)
or (
# A local alias of compile (cfn = compile; co = cfn(src, ...)) is not in
# compile_aliases, so resolve the callee through this scope's exec-builtin
# map (built in source order, cfn precedes co) before giving up.
isinstance(rhs_eff, ast.Call)
and isinstance(rhs_eff.func, ast.Name)
and emap.get(rhs_eff.func.id) == "compile"
)
) and _compile_source_node(rhs_eff) is not None:
# Any `c = compile(...)` (bare / builtins.compile / from-import alias)
# binds a code object, tracked for the types.FunctionType(c) execution
# gadget below (dynamic or foldable payload). The source may be positional
# OR the source= keyword.
camap[name] = True
v = _const_fold(_compile_source_node(rhs_eff), const_env)
if isinstance(v, (str, bytes, bytearray)):
cmap[name] = (
_recovered_source(v),
_compile_mode(rhs_eff, const_env),
isinstance(v, (bytes, bytearray)),
)
if _rhs_import_func(rhs_eff):
imap[name] = True
dfq = _rhs_deserializer(rhs_eff)
if dfq is None and isinstance(rhs_eff, ast.Name) and rhs_eff.id in dmap:
dfq = dmap[rhs_eff.id] # chained alias d = pickle.loads; e = d; e(payload)
if dfq is not None:
dmap[name] = dfq
# Single-assignment string/bytes path constant (p = '/etc/passwd'), used by
# the sensitive-read scanner to fold function-local read paths.
cv = _const_fold(rhs, const_env)
if isinstance(cv, (str, bytes, bytearray)):
scmap[name] = cv
# A parameter DEFAULT that is a dangerous callable acts as an alias inside the body:
# def f(e=exec): e(payload) / def f(s=os.system): s('rm -rf /'). Bind it like a
# single-assignment alias unless the parameter is reassigned in the body.
if _sargs is not None:
_pos = list(_sargs.posonlyargs) + list(_sargs.args)
_paired = list(zip(_pos[len(_pos) - len(_sargs.defaults) :], _sargs.defaults))
_paired += [
(a, d) for a, d in zip(_sargs.kwonlyargs, _sargs.kw_defaults) if d is not None
]
for _p, _d in _paired:
_pn = _p.arg
if counts.get(_pn, 0) != 0 or _pn in rebound:
continue
_de = _unwrap_container_index(_d)
_dfq_sh = _resolve_static_shell_sink(
_de, os_aliases, subprocess_aliases, from_aliases
)
if _dfq_sh and _pn not in smap:
smap[_pn] = _dfq_sh
_deb = _rhs_exec_builtin(_de)
if _deb is not None and _pn not in emap:
emap[_pn] = _deb
_ddfq = _rhs_deserializer(_de)
if _ddfq is not None and _pn not in dmap:
dmap[_pn] = _ddfq
if _rhs_import_func(_de) and _pn not in imap:
imap[_pn] = True
# A `global name = <sink>` (or `nonlocal name = <sink>`) rebinds the name in the
# TARGET scope (module for global, nearest enclosing scope for nonlocal), NOT locally,
# so `global s; s = os.system; s('rm -rf /')` must record the alias in that target
# scope -- otherwise the local pass skips it (name in rebound) and the call resolves to
# nothing. Target scopes are processed before nested scopes, so setdefault preserves
# any alias they already hold.
if global_names or nonlocal_names:
def _chain_lookup(nm, table, local_map):
# Resolve a bare-name RHS alias (global t; t = s) against this scope's just-built
# local map, then the already-indexed enclosing / module scopes -- so a
# `s = os.system` at module scope copied into a `global t; t = s` is propagated,
# matching the chained-alias resolution the local pass does via smap[rhs.id].
_sc = scope
while _sc is not None:
_m = local_map if _sc is scope else table.get(_sc, {})
if nm in _m:
return _m[nm]
_sc = idx.enclosing.get(_sc)
return None
for name, rhs in assigns:
if name in global_names:
_target = tree
elif name in nonlocal_names:
_target = idx.enclosing.get(scope)
else:
continue
if _target is None:
continue
_rhs_eff = _unwrap_container_index(rhs)
_gfq = _resolve_static_shell_sink(
_rhs_eff, os_aliases, subprocess_aliases, from_aliases
)
if _gfq is None and isinstance(_rhs_eff, ast.Name):
_gfq = _chain_lookup(_rhs_eff.id, idx.shell, smap)
if _gfq:
idx.shell.setdefault(_target, {}).setdefault(name, _gfq)
_geb = _rhs_exec_builtin(_rhs_eff)
if _geb is None and isinstance(_rhs_eff, ast.Name):
_geb = _chain_lookup(_rhs_eff.id, idx.execb, emap)
if _geb is not None:
idx.execb.setdefault(_target, {}).setdefault(name, _geb)
_gdfq = _rhs_deserializer(_rhs_eff)
if _gdfq is None and isinstance(_rhs_eff, ast.Name):
_gdfq = _chain_lookup(_rhs_eff.id, idx.deser, dmap)
if _gdfq is not None:
idx.deser.setdefault(_target, {}).setdefault(name, _gdfq)
if smap:
idx.shell[scope] = smap
if emap:
idx.execb[scope] = emap
if cmap:
idx.compiled[scope] = cmap
if camap:
idx.compiledany[scope] = camap
if imap:
idx.impf[scope] = imap
if dmap:
idx.deser[scope] = dmap
if scmap:
idx.strconst[scope] = scmap
if rnmap:
idx.rhsnode[scope] = rnmap
# Class-body aliases are also reachable as ClassName.attr from OUTSIDE the class
# (class C: f = os.system; C.f('rm -rf /')), which lexical scope resolution does not
# cover, so index them by the class name too.
if isinstance(scope, ast.ClassDef):
if smap:
idx.class_shell[scope.name] = dict(smap)
if emap:
idx.class_execb[scope.name] = dict(emap)
if dmap:
idx.class_deser[scope.name] = dict(dmap)
_bases = [b.id for b in scope.bases if isinstance(b, ast.Name)]
if _bases:
idx.class_bases[scope.name] = _bases
# Instance-attribute sink aliases (c.e = exec; c.e(payload) / obj.s = os.system;
# obj.s('rm -rf /')): a simple `Name.attr = <sink>` store binds the attribute to a
# dangerous callable. Tracked tree-wide by (receiver_name, attr) as a fail-closed
# over-approximation, so a later call through the same receiver name gets analyzed.
for n in ast.walk(tree):
if not (
isinstance(n, ast.Assign)
and len(n.targets) == 1
and isinstance(n.targets[0], ast.Attribute)
and isinstance(n.targets[0].value, ast.Name)
):
continue
key = (n.targets[0].value.id, n.targets[0].attr)
rhs_eff = _unwrap_container_index(n.value)
_fq = _resolve_static_shell_sink(rhs_eff, os_aliases, subprocess_aliases, from_aliases)
if _fq:
idx.instance_shell[key] = _fq
_eb = _rhs_exec_builtin(rhs_eff)
if _eb is not None:
idx.instance_execb[key] = _eb
_dfq = _rhs_deserializer(rhs_eff)
if _dfq is not None:
idx.instance_deser[key] = _dfq
return idx
def _payload_has_obfuscation_primitive(node):
"""True when a (non-plain) exec/eval payload is assembled from decode / fetch /
runtime-assembly primitives -- the canonical loader shapes that are essentially
never benign inside a sandbox."""
if node is None or isinstance(node, (ast.Name, ast.Constant)):
return False
for sub in ast.walk(node):
if isinstance(sub, ast.Call):
fn = sub.func
fq = _fq_attr_name(fn)
attr = (
fn.attr
if isinstance(fn, ast.Attribute)
else (fn.id if isinstance(fn, ast.Name) else "")
)
if fq in _CODE_DESERIALIZE_SINKS:
return True
# A dynamic exec payload produced by another eval/exec/compile is a
# nested-dynamic-exec obfuscation (also fails closed on eval(eval(...))).
if isinstance(fn, ast.Name) and fn.id in _EXEC_BUILTINS:
return True
if attr in _DECODE_ATTRS or attr in ("decode", "translate"):
return True
if fq and any(fq.startswith(p) for p in _FETCH_FQ_PREFIXES):
return True
if attr == "join" and sub.args:
a0 = sub.args[0]
if isinstance(a0, (ast.GeneratorExp, ast.ListComp, ast.SetComp)):
return True
if (
isinstance(a0, ast.Call)
and isinstance(a0.func, ast.Name)
and a0.func.id in ("map", "filter")
):
return True
if isinstance(fn, ast.Name) and fn.id in ("bytes", "bytearray") and sub.args:
a0 = sub.args[0]
if isinstance(a0, (ast.GeneratorExp, ast.ListComp, ast.SetComp)):
return True
elif isinstance(sub, ast.Attribute) and sub.attr in ("text", "content"):
if isinstance(sub.value, ast.Call):
return True
elif isinstance(sub, ast.Subscript) and isinstance(sub.slice, ast.Slice):
if sub.slice.step is not None and _const_fold(sub.slice.step) == -1:
return True
elif isinstance(sub, ast.BinOp) and isinstance(sub.op, ast.Mult):
# Large string/bytes repetition assembles an oversized payload (parse
# bomb) that the folder refuses on size; fail closed.
for a, b in ((sub.left, sub.right), (sub.right, sub.left)):
sv = _const_fold(a)
nv = _const_fold(b)
if isinstance(sv, (str, bytes, bytearray)) and isinstance(nv, int) and nv >= 1024:
return True
return False
def _safe_parse_inner(src, mode, depth, budget):
"""DoS-safe gateway to ast.parse on an attacker-influenced payload string.
Returns one of ("PARSED", tree|None), ("SYNTAX_BAD", None), ("BOUND_HIT", None).
A linear bracket-depth pre-scan rejects pathological nesting *before* the C
parser runs (defends the CPython C-stack overflow on deeply-nested input)."""
if depth >= _MAX_UNWRAP_DEPTH:
return ("BOUND_HIT", None)
if len(src) > _MAX_INNER_SRC:
return ("BOUND_HIT", None)
if budget.inner_parses >= _MAX_INNER_PARSES:
return ("BOUND_HIT", None)
if budget.inner_chars + len(src) > _MAX_TOTAL_INNER_CHARS:
return ("BOUND_HIT", None)
if _bracket_depth(src) > _MAX_BRACKET_DEPTH:
return ("BOUND_HIT", None)
budget.inner_parses += 1
budget.inner_chars += len(src)
try:
return ("PARSED", ast.parse(src, mode = mode))
except SyntaxError:
try:
ast.parse(src, mode = ("exec" if mode == "eval" else "eval"))
return ("PARSED", None)
except SyntaxError:
return ("SYNTAX_BAD", None)
except (RecursionError, MemoryError, ValueError):
return ("BOUND_HIT", None)
def _first_unsafe_reason(info):
for key in (
"shell_escapes",
"dynamic_exec",
"network_calls",
"sensitive_file_reads",
"filesystem_violations",
"signal_tampering",
"exception_catching",
):
for item in info.get(key, []) or []:
desc = item.get("description")
if desc:
return desc
return "unsafe operation"
def _recover_exec_payload(node, func_id, const_env, compiled_env):
"""Recover a statically foldable source string for eval/exec/compile.
Returns ("RECOVERED", src, mode, is_bytes) / ("DYNAMIC", None, None, False) /
("NO_PAYLOAD", None, None, False). ``is_bytes`` records that the payload folded to
a bytes/bytearray literal -- ``exec``/``compile`` honor PEP 263 coding cookies on
bytes, so a bytes payload that fails to parse as UTF-8 Python is treated as an
obfuscation vector by the caller rather than a harmless SyntaxError.
"""
if node.args:
arg0 = node.args[0]
elif func_id == "compile":
# compile(source=..., filename=..., mode=...) passes its payload as the source=
# keyword with no positional args. compile() alone does not execute, but its code
# object runs via a gadget (types.FunctionType(c)(); fn.__code__ = c; fn()), so the
# keyword-only source must be analyzed exactly like the positional form rather than
# slipping through as having no payload. (eval / exec take no keyword arguments in
# CPython, so an empty node.args there is genuinely payload-less.)
arg0 = _compile_source_node(node)
if arg0 is None:
return ("NO_PAYLOAD", None, None, False)
else:
return ("NO_PAYLOAD", None, None, False)
base_mode = "eval" if func_id == "eval" else "exec"
# exec(compile("...", ...)) / eval(compile("...", "<s>", "eval")) -- the compile source may be
# positional or the source= keyword.
if (
isinstance(arg0, ast.Call)
and isinstance(arg0.func, ast.Name)
and arg0.func.id == "compile"
and _compile_source_node(arg0) is not None
):
v = _const_fold(_compile_source_node(arg0), const_env)
if isinstance(v, (str, bytes, bytearray)):
return (
"RECOVERED",
_recovered_source(v),
_compile_mode(arg0, const_env),
isinstance(v, (bytes, bytearray)),
)
return ("DYNAMIC", None, None, False)
# c = compile("..."); exec(c)
if isinstance(arg0, ast.Name) and arg0.id in compiled_env:
csrc, cmode, cbytes = compiled_env[arg0.id]
return ("RECOVERED", csrc, cmode, cbytes)
v = _const_fold(arg0, const_env)
if isinstance(v, (str, bytes, bytearray)):
mode = _compile_mode(node, const_env) if func_id == "compile" else base_mode
return ("RECOVERED", _recovered_source(v), mode, isinstance(v, (bytes, bytearray)))
return ("DYNAMIC", None, None, False)
# --------------------------------------------------------------------------
# Stage 3: static sensitive-read scanner.
#
# Filesystem WRITE confinement is enforced at runtime by the realpath backstop
# (Stage 5), which is strictly more robust than static path proving. Reads are not
# confined there, so this small static pass blocks sandboxed code from reading host
# secrets: any call arg that folds to a sensitive host path (covers open()/os.open
# and library loaders like pandas.read_csv('/etc/shadow')), plus `..`/`~` traversal
# on the dedicated open()/read callees. Dynamic paths are left to the backstop.
# --------------------------------------------------------------------------
# Sensitive read targets: exact host-identity / credential files, credential dirs,
# and the classic /proc self-inspection paths. Substring tokens are only consulted
# for absolute or ~-rooted paths with no whitespace (avoids sentence false positives).
_SANDBOX_SENSITIVE_EXACT = frozenset(
{"/etc/passwd", "/etc/shadow", "/etc/sudoers", "/etc/gshadow", "/etc/master.passwd"}
)
_SANDBOX_SENSITIVE_DIR_PARTS = (
"/etc/ssh/",
"/root/",
"/.ssh/",
"/.aws/",
"/.config/gcloud",
"/.kube/",
"/.docker/",
# In-cluster Kubernetes service-account credentials (token / ca.crt / namespace)
# mounted into every pod; reading the token impersonates the pod to the API server.
"/var/run/secrets/kubernetes.io/",
"/run/secrets/kubernetes.io/",
)
_SANDBOX_SENSITIVE_TOKENS = (
"id_rsa",
"id_ed25519",
".pem",
".netrc",
"credentials",
".git-credentials",
"/.huggingface/token",
".kube/config",
)
_SANDBOX_SENSITIVE_RE = re.compile(
r"^/proc/(?:self|\d+)/(?:environ|cmdline|maps|mem|task/\d+/environ)$"
)
def _is_sensitive_abs_path(s):
"""Provably-sensitive absolute (or ~-rooted) path, whitespace-free."""
if not isinstance(s, str) or not s:
return False
norm = s.replace("\\", "/")
if any(ch.isspace() for ch in norm):
return False
if not (norm.startswith("/") or norm.startswith("~")):
return False
if norm in _SANDBOX_SENSITIVE_EXACT:
return True
# The directory markers carry a trailing slash to match descendants (/root/id_rsa);
# append one to the candidate so the directory ITSELF (an unguarded `ls /root` /
# `find /etc/ssh`) matches too, without loosening the component boundary.
if any(part in (norm + "/") for part in _SANDBOX_SENSITIVE_DIR_PARTS):
return True
if _SANDBOX_SENSITIVE_RE.match(norm):
return True
low = norm.lower()
return any(tok in low for tok in _SANDBOX_SENSITIVE_TOKENS)
# Punctuation separators plus the bash compound-statement keywords (if/while/until/then/do/
# else/elif), so a reader in a CONDITION body (`if cat $SECRET; then :; fi`) is scanned at
# command position rather than treated as an argument of the keyword.
_READ_SCAN_SEPARATORS = (
";",
"&&",
"||",
"|",
"&",
"(",
")",
"`",
"{",
"}",
"\n",
) + tuple(_SHELL_KEYWORDS_AS_SEP)
def _join_chdir(base, newdir):
"""Resolve an ``env -C DIR`` / ``--chdir=DIR`` operand against the current child cwd.
A relative DIR chdirs relative to wherever the child already is (an ambient
subprocess ``cwd=`` or a prior ``env -C``), so ``env -C . cat passwd`` under
``cwd=/etc`` still reads ``/etc/passwd``; join it onto the current base rather
than replacing the base with the bare relative fragment. An absolute / ``~``
DIR overrides the base outright."""
if newdir.startswith("/") or newdir.startswith("~"):
return newdir
return os.path.join(base, newdir) if base else newdir
def _extract_command_subs(s):
"""Extract the inner payloads of ``$(...)`` and backtick command substitutions from a
shell string. Substitutions run inside DOUBLE quotes (``echo "$(head /etc/passwd)"``) but
are suppressed entirely inside SINGLE quotes (``echo '$(head /etc/passwd)'`` is a literal),
so single-quoted spans are skipped to avoid over-blocking benign literals. Returns a list of
inner command strings for recursive read scanning. ``$((arith))`` yields a harmless
``(arith)`` payload that scans clean."""
subs = []
i, n = 0, len(s)
in_double = False
while i < n:
c = s[i]
# A single quote OUTSIDE double quotes opens a literal span in which $() / backticks do
# not expand; skip to its close. (Inside double quotes a `'` is an ordinary character.)
if c == "'" and not in_double:
j = s.find("'", i + 1)
if j == -1:
break # unterminated single quote: rest is literal
i = j + 1
elif c == '"':
in_double = not in_double
i += 1
elif c == "`":
j = s.find("`", i + 1)
if j == -1:
break
subs.append(s[i + 1 : j])
i = j + 1
elif c == "$" and i + 1 < n and s[i + 1] == "(":
depth = 1
k = i + 2
while k < n and depth:
if s[k] == "(":
depth += 1
elif s[k] == ")":
depth -= 1
k += 1
subs.append(s[i + 2 : k - 1] if depth == 0 else s[i + 2 : k])
i = k
elif c in "<>" and not in_double and i + 1 < n and s[i + 1] == "(":
# Process substitution <(cmd) / >(cmd): bash runs cmd in a child even when the OUTER
# command is a non-reader (echo <(cat /etc/passwd >&2) leaks the file), so scan the
# inner payload too. It is a word-level construct (not performed inside quotes), so
# single-quoted spans are already skipped and double-quoted text is left literal.
depth = 1
k = i + 2
while k < n and depth:
if s[k] == "(":
depth += 1
elif s[k] == ")":
depth -= 1
k += 1
subs.append(s[i + 2 : k - 1] if depth == 0 else s[i + 2 : k])
i = k
else:
i += 1
return subs
def _resolve_read_chdir(operand, assigns):
"""Resolve an ``env -C DIR`` operand for the read scanner. Returns ``(dir, dynamic)``:
- a ``$VAR`` / ``${VAR}`` that a preceding assignment in the same command bound (``P=/etc;
env -C $P cat passwd``) resolves to that value so the read is combined and caught;
- any other operand carrying ``$`` / backtick is an UNKNOWN expansion that the unguarded
child can point outside the workdir, so ``dynamic=True`` (fail closed for relative reads);
- a plain literal DIR resolves to itself."""
m = re.fullmatch(r"\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?", operand)
if m:
if assigns and m.group(1) in assigns:
return assigns[m.group(1)], False
return None, True
if "$" in operand or "`" in operand:
return None, True
return operand, False
def _argv_env_chdir(str_elts):
"""Extract an ``env -C DIR`` / ``--chdir[=DIR]`` target from a folded argv vector.
A wrapper-prefixed child that chdirs before the reader (``['env', '-C', '/etc', 'cat',
'passwd']``) reads ``/etc/passwd`` in the unguarded child, so the relative reader arg must
be resolved against DIR. Returns the DIR string (or None). Only the ``env`` wrapper honors
``-C``; other flags / wrappers are skipped until the real command word is reached."""
i, n = 0, len(str_elts)
wrapper = None
while i < n:
tok = str_elts[i]
if tok is None:
return None
if _ASSIGNMENT_RE.match(tok):
i += 1
continue
if tok.startswith("-"):
if wrapper == "env":
if tok in ("-C", "--chdir"):
return str_elts[i + 1] if i + 1 < n else None
if tok.startswith("--chdir="):
return tok.split("=", 1)[1]
# A wrapper flag that consumes the NEXT token (env -u NAME, sudo -u user, nice -n 5,
# timeout -k 5): skip both so the operand is not mistaken for the command word.
if wrapper is not None and _wrapper_flag_takes_operand(wrapper, tok):
i += 2
continue
i += 1
continue
base = os.path.basename(tok).lower()
if base in _COMMAND_PREFIXES:
wrapper = base
i += 1
continue
# A wrapper's numeric / duration operand (timeout 1, nice 5): skip it and keep scanning
# for a following env -C, rather than stopping at it as the executed command word. Without
# this, `env -C` hidden behind `timeout 1 env -C /etc ...` was never reached.
if wrapper is not None and _is_wrapper_numeric_arg(tok):
i += 1
continue
return None # reached the executed command word before any env -C
return None
def _argv_command_word_index(str_elts):
"""Index of the EXECUTED command word in a folded argv vector, skipping VAR=value
assignments and command wrappers (env / sudo / nice / timeout / ...) with their flags and
numeric / separated operands. None if a token is unresolved (None) or no command word is
reached. Lets a wrapper-hidden reader (['timeout', '1', 'cat', 'x']) be found at its real
position instead of stopping at argv[0]."""
i, n = 0, len(str_elts)
wrapper = None
while i < n:
tok = str_elts[i]
if tok is None:
return None
if _ASSIGNMENT_RE.match(tok):
i += 1
continue
if tok.startswith("-"):
if wrapper is not None and _wrapper_flag_takes_operand(wrapper, tok):
i += 2
continue
i += 1
continue
base = os.path.basename(tok).lower()
if base in _COMMAND_PREFIXES:
wrapper = base
i += 1
continue
if wrapper is not None and _is_wrapper_numeric_arg(tok):
i += 1
continue
return i
return None
def _scan_command_string_for_reads(
command,
*,
strict_traversal,
cwd = None,
cwd_dynamic = False,
_depth = 0,
):
"""Scan a shell command STRING for an embedded host-secret read; return a short reason or
None. Covers literal sensitive / traversal paths, input redirects, and $ / backtick /
escaping-glob expansions on file-reading commands. Reads from a shell child are not
runtime-confined, so these are refused statically.
strict_traversal=True blocks ANY ``..`` / ``~`` read path (the os.system() shell-string
policy in the Python tool); False blocks only a traversal that resolves onto a sensitive
path, so ordinary terminal relative navigation (``ls ../src``) is not flagged.
The reader / command word is resolved past leading ``VAR=value`` assignments and command
wrappers (env / sudo / nice / timeout / ...), a ``VAR=value`` assignment whose value is a
sensitive path is flagged, and a nested ``bash -c '<payload>'`` shell has its payload
recursively scanned, so a read hidden behind a normal command-prefix form is still caught."""
if not command or _depth > 6:
return None
# Model bash brace expansion so a brace-hidden reader / path (`{cat,/etc/passwd}`) is seen.
cmd = _expand_braces(_expand_ifs(_normalize_ansi_c_quotes(command)))
def _traversal_hits_sensitive(norm):
# A relative path that climbs out of the workdir with '..' can name a host secret
# (../../../../etc/passwd). Resolve the climb and test whether the descent lands on a
# sensitive path; a plain in-tree relative path (../sibling/file.txt) is left alone.
if ".." not in norm.split("/"):
return False
try:
canon = os.path.normpath(norm)
except Exception:
canon = norm
parts = [p for p in canon.split("/") if p not in ("", ".", "..")]
return bool(parts) and _is_sensitive_abs_path("/" + "/".join(parts))
def _flag(s):
norm = s.replace("\\", "/")
try:
canon = os.path.normpath(norm)
except Exception:
canon = norm
if _is_sensitive_abs_path(norm) or _is_sensitive_abs_path(canon):
return f"{s!r} is a sensitive host identity / credential file"
if strict_traversal:
if s[:1] == "~" or ".." in norm.split("/"):
return f"{s!r} escapes the session workdir via path traversal"
elif _traversal_hits_sensitive(norm):
return f"{s!r} reads a sensitive host path via directory traversal"
return None
def _escaping_glob(tok):
if not any(g in tok for g in "*?["):
return False
tn = tok.replace("\\", "/")
return tok[:1] == "~" or tn.startswith("/")
def _check_assignment_rhs(tok):
# FOO=/etc/passwd binds a sensitive path into a variable that a later reader dereferences
# (P=/etc/passwd cat ${P}); flag the assigned value directly.
_rhs = tok.split("=", 1)[1] if "=" in tok else ""
for _pc in re.split(r"[;|&<>()`{}]+", _rhs):
if _pc and not _pc.startswith("-"):
_r = _flag(_pc)
if _r is not None:
return _r
return None
# Re-tokenize keeping redirects / separators for the input-redirect + expansion scan.
try:
_lx = shlex.shlex(cmd, posix = True, punctuation_chars = ";&|()`<>")
_lx.whitespace_split = True
ptoks = list(_lx)
except ValueError:
ptoks = cmd.split()
# Literal-path token scan (absolute-sensitive + traversal), COMMAND-WORD aware: a sensitive
# path that a non-reader merely prints / passes as data (echo /etc/passwd, printf %s
# /etc/passwd) is not a read, so its args are exempt. Every other command word -- readers AND
# unknown commands -- still fails closed. A prefix assignment binding a sensitive path
# (P=/etc/passwd cat ${P}) is flagged at the command position; punctuation glued to a word
# (cat /etc/passwd|wc) is split so the path piece is still checked.
_lit_at_cmd = True
_lit_wrapper = None
_lit_is_reader_ctx = True # unknown command word -> still fail closed
for _lt in ptoks:
if _lt in _READ_SCAN_SEPARATORS:
_lit_at_cmd = True
_lit_wrapper = None
_lit_is_reader_ctx = True
continue
if _lit_at_cmd:
if _ASSIGNMENT_RE.match(_lt):
_r = _check_assignment_rhs(_lt)
if _r is not None:
return _r
continue # assignment prefix; command word still ahead
if _lt.startswith("-"):
continue # wrapper flag before the command word
_ltbase = os.path.basename(_lt).lower()
if _ltbase in _COMMAND_PREFIXES:
_lit_wrapper = _ltbase
continue
if _lit_wrapper is not None and _is_wrapper_numeric_arg(_lt):
continue
# command word resolved: a known non-reader exempts its args from the literal scan.
_lit_is_reader_ctx = _ltbase not in _SHELL_NON_READER_COMMANDS
_lit_at_cmd = False
continue
if not _lit_is_reader_ctx:
continue # echo / printf / ... argument: the path is data, not a read
for _piece in re.split(r"[;|&<>()`{}]+", _lt):
if _piece and not _piece.startswith("-"):
_r = _flag(_piece)
if _r is not None:
return _r
# find <ROOT...> ... -exec READER {} \; loses the search ROOT in the -exec segment, so a reader
# over an ABSOLUTE / sensitive root (find /etc -name passwd -exec cat {} ;) reads a host secret
# the segment scan alone cannot see ({} carries no path). Compute the find roots (the leading
# operands before the first predicate) and note whether any escapes the workdir.
_find_roots = []
for _fi2, _ft2 in enumerate(ptoks):
if os.path.basename(_ft2).lower() == "find":
_rj = _fi2 + 1
while _rj < len(ptoks):
_rt = ptoks[_rj]
if _rt.startswith("-") or _rt in ("(", "!", ",") or _rt in _READ_SCAN_SEPARATORS:
break
_find_roots.append(_rt)
_rj += 1
break
_find_escaping_root = any(
_arg_escapes_workdir(_r) or _is_sensitive_abs_path(_r.replace("\\", "/"))
for _r in _find_roots
)
# find ... -exec CMD ... ; runs CMD directly on each match; CMD may be a nested shell
# (sh -c 'cat /etc/passwd') or a reader, so scan each -exec segment through this scanner
# (mirrors the blocked-command find -exec handling). The main command-word loop below only
# recurses into a shell that IS the command word, so the quoted -c payload would otherwise
# be treated as one inert argument.
for _fi, _ft in enumerate(ptoks):
if _ft in _FIND_EXEC_FLAGS:
_seg = []
_fj = _fi + 1
while _fj < len(ptoks) and ptoks[_fj] not in _FIND_EXEC_TERMINATORS:
_seg.append(ptoks[_fj])
_fj += 1
if _seg:
# A reader -exec that references {} over an escaping find root reads host files.
if _find_escaping_root and "{}" in _seg:
_si = 0
while _si < len(_seg) and (
_seg[_si].startswith("-")
or os.path.basename(_seg[_si]).lower() in _COMMAND_PREFIXES
or _ASSIGNMENT_RE.match(_seg[_si])
):
_si += 1
_segcmd = os.path.basename(_seg[_si]).lower() if _si < len(_seg) else ""
if _segcmd in _SHELL_READ_COMMANDS:
return (
f"find -exec {_segcmd} {{}} over an escaping search root "
f"reads a host file the {{}} placeholder hides"
)
_r = _scan_command_string_for_reads(
shlex.join(_seg),
strict_traversal = strict_traversal,
cwd = cwd,
cwd_dynamic = cwd_dynamic,
_depth = _depth + 1,
)
if _r is not None:
return _r
# trap 'CMD' SIG: the quoted handler runs as shell code on EXIT / a signal; scan it.
# Skip trap options / the -- terminator (trap -- 'CMD' EXIT, trap -p) so the handler
# operand is not mistaken for -- and left unscanned.
if _ft == "trap":
_tj = _fi + 1
while _tj < len(ptoks) and ptoks[_tj].startswith("-") and len(ptoks[_tj]) > 1:
_tj += 1
_th = ptoks[_tj] if _tj < len(ptoks) else None
if _th and _th != "-" and _th not in _READ_SCAN_SEPARATORS:
_r = _scan_command_string_for_reads(
_th,
strict_traversal = strict_traversal,
cwd = cwd,
cwd_dynamic = cwd_dynamic,
_depth = _depth + 1,
)
if _r is not None:
return _r
# A command substitution ($(...) / `...`) runs its payload as a shell command regardless of
# surrounding quotes, so `echo "$(head /etc/passwd)"` reads the file even though the outer
# command is not a reader and the tokenizer keeps the quoted substitution as one argument.
# Scan each substitution payload recursively, independent of the outer command word.
for _cs in _extract_command_subs(cmd):
if _cs.strip():
_r = _scan_command_string_for_reads(
_cs,
strict_traversal = strict_traversal,
cwd = cwd,
cwd_dynamic = cwd_dynamic,
_depth = _depth + 1,
)
if _r is not None:
return _r
# env -S 'cmd' / --split-string='cmd' splits its operand into a fresh command line that runs
# as the child, so a reader-only payload (env --split-string='cat /etc/passwd') is a host-file
# read even though env is a wrapper. Recurse each env split-string payload into the read scan.
for _si, _st in enumerate(ptoks):
_sl = _st.lower()
_spayload = None
if _sl in ("-s", "--split-string") and _si + 1 < len(ptoks):
_spayload = ptoks[_si + 1]
elif _sl.startswith("-s") and _sl != "-s" and not _sl.startswith("--"):
_spayload = _st[2:] # glued short form: -S'cmd' / -Scmd
elif _sl.startswith("--split-string="):
_spayload = _st[len("--split-string=") :]
if not _spayload:
continue
# Confirm the split-string belongs to an `env` wrapper (not a -s flag of another command).
_is_env = False
for _sj in range(_si - 1, -1, -1):
_sp = ptoks[_sj]
if _sp in _READ_SCAN_SEPARATORS:
break
if _sp.startswith("-"):
continue
_is_env = os.path.basename(_sp).lower() == "env"
break
if _is_env:
_r = _scan_command_string_for_reads(
_spayload,
strict_traversal = strict_traversal,
cwd = cwd,
cwd_dynamic = cwd_dynamic,
_depth = _depth + 1,
)
if _r is not None:
return _r
def _risky_read_target(tgt):
if not tgt:
return False
if "$" in tgt or "`" in tgt or _escaping_glob(tgt):
return True
tn = tgt.replace("\\", "/")
if _is_sensitive_abs_path(tgt):
return True
return ".." in tn.split("/") if strict_traversal else _traversal_hits_sensitive(tn)
_at_cmd = True
_cur_reader = False
_wrapper = None
_skip_operand = False
# The child's cwd for a relative read: seeded from an ambient cwd (a subprocess cwd=), and
# overridable per-command by env -C DIR. Resets to the ambient cwd at each separator.
_chdir = cwd
_pending_chdir = False
_pending_argfile = False
# env -C DIR whose DIR is an unknown expansion ($UNRESOLVED / backtick): the child's cwd is
# unprovable, so a later relative reader arg fails closed. Reset per command, like _chdir.
_chdir_dynamic = False
# Shell VAR=value bindings seen so far (P=/etc; env -C $P ...), so an env -C $P operand
# resolves to /etc and the read is combined + caught. Persists across separators.
_local_assigns = {}
# jq reads files through explicit options, but its positional FILTER legitimately contains
# `$` (jq variables: jq -n --rawfile x f '$x'), so jq is NOT a generic reader -- that would
# misfire on every filter. Scan only jq's file-valued options: --rawfile NAME FILE /
# --slurpfile NAME FILE read FILE into a variable, and -f / --from-file FILE read the program
# file. A sensitive / expanded / escaping FILE operand exfiltrates a host secret.
for _ji, _jt in enumerate(ptoks):
if os.path.basename(_jt).lower() != "jq":
continue
_jk = _ji + 1
while _jk < len(ptoks) and ptoks[_jk] not in _READ_SCAN_SEPARATORS:
_jw = ptoks[_jk]
if _jw in ("--rawfile", "--slurpfile") and _jk + 2 < len(ptoks):
if _risky_read_target(ptoks[_jk + 2]):
return f"jq reads a sensitive file {ptoks[_jk + 2]!r}"
_jk += 3
continue
if _jw in ("-f", "--from-file") and _jk + 1 < len(ptoks):
if _risky_read_target(ptoks[_jk + 1]):
return f"jq reads a program file {ptoks[_jk + 1]!r}"
_jk += 2
continue
if _jw.startswith("--from-file="):
if _risky_read_target(_jw.split("=", 1)[1]):
return f"jq reads a program file {_jw.split('=', 1)[1]!r}"
_jk += 1
for _pi, _pt in enumerate(ptoks):
if _pt in _READ_SCAN_SEPARATORS:
# env -C `...` / env -C $(...): the substitution operand STARTS with a punctuation
# token ( ` or ( ) that the tokenizer emits as a separator, so the pending env -C
# never captured it. Mark the cwd dynamic here so the trailing reader fails closed.
if _pending_chdir and _pt in ("(", "`"):
_chdir_dynamic = True
_at_cmd = True
_cur_reader = False
_wrapper = None
_skip_operand = False
_pending_argfile = False
_chdir = cwd
# A command-substitution punctuation token ( ( ) ` ) does NOT end the current
# command, so it must not drop a pending env -C dynamic-cwd flag: env -C $(printf
# /etc) cat passwd tokenizes the operand into $ ( printf /etc ), and the ( / )
# would otherwise reset _chdir_dynamic before the trailing reader is scanned. Only a
# real command separator ( ; | & newline / keyword ) ends the env -C scope.
if _pt not in ("(", ")", "`"):
_chdir_dynamic = False
_pending_chdir = False
continue
if _pt.startswith("<"):
_rt = _pt.lstrip("<") or (ptoks[_pi + 1] if _pi + 1 < len(ptoks) else "")
if _risky_read_target(_rt):
return f"shell input redirect from a non-literal / sensitive path {_rt!r}"
continue
if _pt.startswith(">"):
continue # output redirects are handled by _find_blocked_commands
if _at_cmd:
if _skip_operand: # a wrapper flag's separated operand (env -u NAME)
if _pending_chdir: # ...but env -C DIR's operand is the child cwd
_rdir, _rdyn = _resolve_read_chdir(_pt, _local_assigns)
if _rdyn:
_chdir_dynamic = True
else:
_chdir = _join_chdir(_chdir, _rdir)
_pending_chdir = False
elif _pending_argfile: # ...and xargs -a FILE reads FILE
_pending_argfile = False
if _risky_read_target(_pt):
return f"xargs reads arguments from a sensitive path {_pt!r}"
_skip_operand = False
continue
if _ASSIGNMENT_RE.match(_pt):
_r = _check_assignment_rhs(_pt)
if _r is not None:
return _r
_an, _, _av = _pt.partition("=")
_local_assigns[_an.rstrip("+")] = _av
continue # assignment prefix; the command word is still ahead
if _pt.startswith("-"):
# env -C DIR / --chdir DIR changes the child's cwd before the command runs, so
# a later relative reader arg (env -C /etc cat passwd -> /etc/passwd) resolves
# against DIR, not the workdir. Capture DIR instead of just skipping it.
if _wrapper == "env" and _pt in ("-C", "--chdir"):
_pending_chdir = True
_skip_operand = True
elif _wrapper == "env" and _pt.startswith("--chdir="):
_rdir, _rdyn = _resolve_read_chdir(_pt.split("=", 1)[1], _local_assigns)
if _rdyn:
_chdir_dynamic = True
else:
_chdir = _join_chdir(_chdir, _rdir)
# xargs -a FILE / --arg-file[=]FILE reads its argument list FROM that file, so a
# sensitive / expanded target is a host-file read even though xargs is a wrapper.
elif _wrapper == "xargs" and _pt in ("-a", "--arg-file"):
_pending_argfile = True
_skip_operand = True
elif _wrapper == "xargs" and _pt.startswith("--arg-file="):
if _risky_read_target(_pt.split("=", 1)[1]):
return (
f"xargs reads arguments from a sensitive path {_pt.split('=', 1)[1]!r}"
)
elif _wrapper and _wrapper_flag_takes_operand(_wrapper, _pt):
_skip_operand = True
continue # wrapper flag; still before the command word
# A wrapper's numeric operand (`timeout 1 bash -c ...`, `nice 5 cat ...`) is not
# the command word; skip it so the real command (bash / cat) after it is scanned.
if _wrapper and _is_wrapper_numeric_arg(_pt):
continue
_base = os.path.basename(_pt).lower()
if _base in _COMMAND_PREFIXES:
_wrapper = _base
continue # env / sudo / nice / timeout ...; command word is still ahead
if _base in _SHELL_BINARIES:
# A nested shell runs its -c payload as fresh shell code; scan it recursively.
for _k in range(_pi + 1, len(ptoks)):
_ft = ptoks[_k]
if _ft in _READ_SCAN_SEPARATORS:
break
_fl = _ft.lower()
if _fl == "-c" or (
_fl.startswith("-") and not _fl.startswith("--") and _fl.endswith("c")
):
if _k + 1 < len(ptoks):
# Propagate the CURRENT env -C cwd AND its dynamic flag: env -C
# ${X:-/etc} bash -c 'cat passwd' chdirs to an unprovable dir, so the
# nested payload's relative reads must fail closed too (not just the
# original ambient cwd_dynamic).
_r = _scan_command_string_for_reads(
ptoks[_k + 1],
strict_traversal = strict_traversal,
cwd = _chdir,
cwd_dynamic = cwd_dynamic or _chdir_dynamic,
_depth = _depth + 1,
)
if _r is not None:
return _r
break
_at_cmd = False
_cur_reader = False
_wrapper = None
continue
_cur_reader = _base in _SHELL_READ_COMMANDS
_at_cmd = False
_wrapper = None
continue
if _cur_reader and not _pt.startswith("-"):
if "$" in _pt or "`" in _pt or _escaping_glob(_pt):
return f"shell read command reads an expanded path {_pt!r}"
_rel = not _pt.startswith("/") and not _pt.startswith("~")
# A relative reader arg under an env -C whose DIR was an UNKNOWN expansion (env -C
# $UNRESOLVED cat passwd) cannot be proven sandbox-local, so fail closed.
if _chdir_dynamic and _rel:
return f"shell read command reads {_pt!r} under an expanded chdir"
# Under a known chdir (env -C DIR or an ambient subprocess cwd=), a relative reader
# arg resolves against DIR (cat passwd + cwd=/etc -> /etc/passwd).
if _chdir and _rel:
_r = _flag(os.path.join(_chdir, _pt))
if _r is not None:
return _r
# A relative reader arg under a NON-literal cwd cannot be proven sandbox-local, so
# fail closed (subprocess.run('cat passwd', shell=True, cwd=P)).
if cwd_dynamic and _chdir is None and _rel:
return f"shell read command reads {_pt!r} under a non-literal cwd"
return None
def _command_reads_sensitive(command: str) -> str | None:
"""Scan a raw terminal-tool shell command STRING for an embedded host-secret read; return a
short reason or None. The terminal tool runs the command in an unguarded shell child (the
runtime open() backstop only wraps the Python tool), so a read of an identity / credential
file, a sensitive-target ``..`` traversal, or an escaping glob / ``$()`` / backtick
expansion on a file-reading command must be refused statically. Benign in-tree relative
navigation is left alone (strict_traversal disabled)."""
return _scan_command_string_for_reads(command, strict_traversal = False)
# Stage 4: pragmatic aliasing (single-assignment alias + inline literal container).
# Catches `s = os.system; s('rm -rf /')` and `[os.system][0](...)` feeding the
# existing shell-command denylist. Deliberately low-FP: only unambiguous single
# assignments (a name stored exactly once) and inline literal containers, never a
# flow-insensitive union (so `s = os.system; s = print; s('hi')` is NOT aliased).
# --------------------------------------------------------------------------
_SHELL_SINK_FUNCS = frozenset(
{
"os.system",
"os.popen",
"os.popen2",
"os.popen3",
"os.popen4",
"os.execl",
"os.execle",
"os.execlp",
"os.execlpe",
"os.execv",
"os.execve",
"os.execvp",
"os.execvpe",
"os.spawnl",
"os.spawnle",
"os.spawnlp",
"os.spawnlpe",
"os.spawnv",
"os.spawnve",
"os.spawnvp",
"os.spawnvpe",
"os.posix_spawn",
"os.posix_spawnp",
"subprocess.run",
"subprocess.call",
"subprocess.check_call",
"subprocess.check_output",
"subprocess.Popen",
"subprocess.getoutput",
"subprocess.getstatusoutput",
}
)
def _resolve_static_shell_sink(node, os_aliases, subprocess_aliases, from_aliases):
"""Resolve an expression to a shell-sink fully-qualified name, else None."""
if isinstance(node, ast.Attribute):
v = node.value
# os / posix / nt receiver as a simple name (os.system, posix.system) or a re-exported
# module reached as an attribute (pathlib.os.system, tempfile.os.system).
is_os = (isinstance(v, ast.Name) and v.id in os_aliases) or (
isinstance(v, ast.Attribute) and v.attr in ("os", "posix", "nt")
)
is_sp = (isinstance(v, ast.Name) and v.id in subprocess_aliases) or (
isinstance(v, ast.Attribute) and v.attr == "subprocess"
)
if is_os:
fq = f"os.{node.attr}"
if fq in _SHELL_SINK_FUNCS:
return fq
if is_sp:
fq = f"subprocess.{node.attr}"
if fq in _SHELL_SINK_FUNCS:
return fq
if isinstance(node, ast.Name):
return from_aliases.get(node.id)
return None
def _check_signal_escape_patterns(
code: str,
_depth: int = 0,
_budget = None,
):
"""Check for patterns that could escape signal-based timeouts. Returns
(safe: bool, details: dict). Vendored from unsloth_zoo.rl_environments to
avoid importing unsloth_zoo (needs GPU drivers; fails on Apple Silicon)."""
try:
tree = ast.parse(code)
except SyntaxError as e:
return False, {
"error": f"SyntaxError: {e}",
"signal_tampering": [],
"exception_catching": [],
"warnings": [],
}
# Charge this tree's node count against the shared analyzer budget BEFORE the several
# unbounded ast.walk / visitor passes below. A syntactically valid file (or a huge
# recovered exec/eval payload, since the budget is shared across the recursion) with
# hundreds of thousands of nodes would otherwise tie up the Studio parent process
# before the child rlimits apply. Fail closed (block) when the budget is exceeded.
if _budget is None:
_budget = _AnalyzerBudget()
try:
_budget.nodes += sum(1 for _ in ast.walk(tree))
except Exception: # pragma: no cover - defensive
pass
if _budget.nodes > _MAX_ANALYZER_NODES:
return False, {
"error": None,
"signal_tampering": [],
"exception_catching": [],
"shell_escapes": [],
"dynamic_exec": [
{
"type": "analyzer_budget",
"line": -1,
"description": (
"code exceeds the static-analysis node budget (too large to verify safely)"
),
}
],
"network_calls": [],
"sensitive_file_reads": [],
"filesystem_violations": [],
"warnings": [],
}
signal_tampering = []
exception_catching = []
shell_escapes = []
dynamic_exec = []
filesystem_violations = []
warnings = []
# Feature flag + shared budget for the recursive sink analyzer (Stages 2-4).
# Default on; UNSLOTH_STUDIO_SINK_ANALYZER=0 reverts to the legacy blanket
# eval/exec ban and disables filesystem-confinement + aliasing analysis.
_analyzer_on = os.environ.get("UNSLOTH_STUDIO_SINK_ANALYZER", "1") != "0"
if _analyzer_on:
try:
_const_env = _build_const_prop_env(tree)
_scope_idx = _build_scope_alias_index(tree, _const_env)
except Exception: # pragma: no cover - defensive: never crashier than legacy
logger.warning("sandbox analyzer context build failed; legacy fallback", exc_info = True)
_analyzer_on = False
_const_env = {}
_scope_idx = _ScopeAliasIndex(tree)
else:
_const_env = {}
_scope_idx = _ScopeAliasIndex(tree)
def _payload_free_name_hits_caller_alias(src, mode, node):
"""A FREE name in an exec/eval payload that resolves, in the CALLER's scope at ``node``, to
a shell / exec-builtin / deserialize / import alias -- exec/eval run in the caller
namespace, so ``f`` in ``exec('f(...)')`` is the caller's ``f = os.system``. Returns the
offending name or None. Builtins / undefined names never resolve, so ``exec('print(1)')``
and ``exec('x = 1')`` stay allowed."""
try:
inner = ast.parse(src, mode = "eval" if mode == "eval" else "exec")
except Exception:
return None
bound: set[str] = set()
loaded: set[str] = set()
for n in ast.walk(inner):
if isinstance(n, ast.Name):
if isinstance(n.ctx, (ast.Store, ast.Del)):
bound.add(n.id)
elif isinstance(n.ctx, ast.Load):
loaded.add(n.id)
elif isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
bound.add(n.name)
elif isinstance(n, (ast.Import, ast.ImportFrom)):
for _al in n.names:
bound.add((_al.asname or _al.name).split(".")[0])
for nm in loaded - bound:
for _kind in ("shell", "execb", "deser", "impf"):
if _scope_idx.resolve(nm, node, _kind):
return nm
return None
def _analyze_exec_call(node, func_id):
"""Stage 2 driver: recover + recurse a foldable payload, else dynamic policy."""
try:
# Resolve compiled-code aliases (c = compile(...)) in the CALL's scope so a
# safe alias in one function cannot shadow a dynamic exec(c) in another.
_compiled_here = _scope_idx.effective(node, "compiled")
kind, src, mode, is_bytes = _recover_exec_payload(
node, func_id, _const_env, _compiled_here
)
if kind == "NO_PAYLOAD":
return
if kind == "RECOVERED":
parsed_kind, _ = _safe_parse_inner(src, mode, _depth, _budget)
if parsed_kind == "PARSED":
inner_safe, inner_info = _check_signal_escape_patterns(src, _depth + 1, _budget)
if not inner_safe and not inner_info.get("error"):
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": (
f"{func_id}() payload reaches unsafe operation: "
f"{_first_unsafe_reason(inner_info)}"
),
}
)
return
# exec()/eval() run in the CALLER namespace, so the payload -- scanned above as a
# fresh module -- can reference a caller-scope alias the inner pass cannot see
# (import os; f = os.system; exec("f('rm -rf /')")). Fail closed when a payload
# FREE name resolves to a shell / exec / deserialize / import alias at this call.
_alias = _payload_free_name_hits_caller_alias(src, mode, node)
if _alias is not None:
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": (
f"{func_id}() payload references caller alias {_alias!r} "
"bound to a shell / exec / deserialize sink"
),
}
)
return
if parsed_kind == "BOUND_HIT":
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": (
f"{func_id}() payload exceeds static-analysis bounds "
"(oversized / too-deeply-nested / too-many-layers)"
),
}
)
return
# SYNTAX_BAD on a fully RECOVERED literal: we hold the exact
# source and it simply is not valid Python for this sink's mode,
# so it raises SyntaxError at runtime (same mode as the static
# parse) -- harmless, not an ACE vector. Allow it; only truly
# opaque (non-recoverable) payloads fall to the dynamic policy.
#
# Exception: a *bytes* payload for an executing sink. exec()/eval()/
# compile() honor PEP 263 coding cookies (e.g. "# coding: utf-7") on
# bytes, decoding them through a codec this static pass does not
# replicate -- the UTF-8 view we parsed is SYNTAX_BAD precisely
# because the real (cookie-decoded) source is hidden. A legitimate
# exec(b"...") uses plain ASCII/UTF-8 that parses cleanly, so blocking
# the unparseable-bytes case closes the codec-smuggling vector with
# negligible false positives.
if is_bytes and func_id != "compile":
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": (
f"{func_id}() of a bytes payload that is not valid UTF-8 "
"Python (may smuggle code via a PEP 263 coding cookie)"
),
}
)
return
payload = node.args[0] if node.args else None
if _payload_has_obfuscation_primitive(payload):
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": (
f"{func_id}() of a runtime-decoded / fetched / assembled payload"
),
}
)
else:
# An opaque, non-recoverable payload for an executing sink (eval/exec/
# runpy) is a universal ACE bypass: it can synthesize any shell/network/
# filesystem escape at runtime, invisibly to every static check. compile()
# does not itself run, but its CODE OBJECT can be executed without exec/eval
# (fn.__code__ = compile(src, '<p>', 'exec'); fn()), so an opaque compile
# source is equally unverifiable and is blocked too. A literal source is
# analyzed recursively above; ast.literal_eval / json.loads cover data.
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": (
f"{func_id}() of a non-literal payload cannot be statically "
"verified (use ast.literal_eval / json.loads for data)"
),
}
)
except Exception: # pragma: no cover - fail closed, never crashier than legacy
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": f"dynamic code execution via {func_id}()",
}
)
def _ast_name_matches(node, names):
if isinstance(node, ast.Name):
return node.id in names
elif isinstance(node, ast.Attribute):
full_name = []
current = node
while isinstance(current, ast.Attribute):
full_name.append(current.attr)
current = current.value
if isinstance(current, ast.Name):
full_name.append(current.id)
full_name = ".".join(reversed(full_name))
return full_name in names
return False
# Dangerous os/subprocess functions that can execute shell commands
# (defined at module scope as _SHELL_SINK_FUNCS so Stage 4 alias resolution
# can share it).
_SHELL_EXEC_FUNCS = _SHELL_SINK_FUNCS
# Dynamic-execution / obfuscation primitives that defeat the static (name-based) checks
# above: they build or reach a dangerous callable at runtime, so a bare name match cannot
# see the payload. eval/exec/compile are direct code-execution builtins; __import__ /
# importlib load a module by (possibly computed) name; getattr/setattr on a sensitive
# module implement `getattr(os, 'sys'+'tem')(...)`.
_DYNAMIC_EXEC_BUILTINS = frozenset({"eval", "exec", "compile"})
_DYNAMIC_IMPORT_FUNCS = frozenset(
{"importlib.import_module", "importlib.reload", "importlib.__import__"}
)
# Dynamic import is a real workflow (e.g. importing huggingface_hub), so it is flagged only
# when the target is computed (non-literal name = obfuscation) or names a module that can
# reach code execution / shell / builtins. A benign literal (json, numpy, huggingface_hub)
# passes; the HF upload gate below still validates its call args separately.
_DANGEROUS_IMPORT_NAMES = frozenset(
{
"os",
"posix", # the C module os wraps; __import__('posix').system(...) == os.system
"nt", # Windows analogue of posix
"subprocess",
"sys",
"builtins",
"importlib",
"ctypes",
"pty",
"socket",
"signal",
"resource",
"shutil",
"multiprocessing",
"runpy",
"code",
"codeop",
"pdb",
"mmap",
"fcntl",
}
)
# Attribute-name obfuscation via getattr/setattr is only flagged when aimed at a module
# that can execute code or reach builtins (keeps ordinary getattr(obj, "field") benign).
_DYNAMIC_ATTR_TARGETS = frozenset({"os", "subprocess", "sys", "builtins", "importlib"})
# Introspection "gadget" dunders used to walk from a harmless object to os/builtins
# (``().__class__.__bases__[0].__subclasses__()``). ``__class__`` / ``__dict__`` are
# intentionally excluded (too common); the chain still trips on the others.
# __mro__ and __code__ are deliberately EXCLUDED here: on their own they do not reach
# an execution primitive, and they are read by ordinary ML/debugging code
# (trainer_class.__mro__, fn.__code__), so flagging them over-blocks legitimate
# snippets. The terminal escape primitives below still trip on the real gadget
# chains (().__class__.__bases__[0].__subclasses__(), f.__globals__['os']). A
# SUBSCRIPTED __mro__ (cls.__mro__[1], the base-class extraction shape) is flagged
# separately in visit_Subscript so plain iteration stays allowed.
# cell_contents is the ONLY way to read a closure cell's value, so it is the terminal
# step of recovering a guarded wrapper's original callable via __closure__; flagging
# it closes that recovery even when the __closure__ name was built dynamically.
_GADGET_DUNDERS = frozenset(
{
"__subclasses__",
"__bases__",
"__base__",
"__globals__",
"__builtins__",
"__closure__",
"cell_contents",
# Frame / traceback introspection recovers a runtime-guard wrapper's ORIGINAL
# unguarded callable: it is a free variable (`real`) in the wrapper's frame, so a
# snippet that triggers a denied open() can read it back via a trace hook or the
# caught exception's traceback (frame.f_locals['real'], tb.tb_frame.f_locals) and
# call it directly, escaping the filesystem boundary. __closure__ / cell_contents
# are already blocked, so the frame path is the remaining channel; close it by
# flagging frame acquisition (settrace / _getframe / currentframe / tb_frame) and
# the f_locals / f_globals value read for ANY receiver.
"f_locals",
"f_globals",
"f_back",
"f_builtins",
"tb_frame",
"tb_next",
"gi_frame",
"cr_frame",
"ag_frame",
"settrace",
"setprofile",
"_getframe",
"_current_frames",
"currentframe",
}
)
def _extract_string_from_node(node):
"""Extract a plain string value from an AST node, if it is a constant."""
if isinstance(node, ast.Constant) and isinstance(node.value, str):
return node.value
return None
def _extract_env_scalar(node):
"""A str constant, or a bytes constant decoded to str (os.environb byte keys / values are
the same inherited environment as os.environ), else None."""
if isinstance(node, ast.Constant):
if isinstance(node.value, str):
return node.value
if isinstance(node.value, (bytes, bytearray)):
try:
return bytes(node.value).decode("utf-8", "surrogateescape")
except Exception:
return None
return None
def _extract_strings_from_list(node):
"""Extract string elements from an AST List or Tuple node."""
if isinstance(node, (ast.List, ast.Tuple)):
parts = []
for elt in node.elts:
s = _extract_string_from_node(elt)
if s is not None:
parts.append(s)
return parts
return []
# Kwarg names that carry command content (not control flags like
# check=True, text=True, capture_output=True).
_CMD_KWARGS = frozenset({"args", "command", "executable", "path", "file"})
def _check_shell_argv(elts):
"""Analyze a subprocess argv VECTOR (['bash', '-c', '...'], ['sh', 's.sh']) as a
whole. A shell argv is only safe when it carries an inline -c payload that scans
clean; a script-file / -s / bare-shell / dynamic-payload form runs unscanned code
and is denied. Returns a set of blocked markers (empty if not a shell argv or the
scanned -c payload is benign)."""
if not elts:
return set()
first = _extract_string_from_node(elts[0])
if first is None or os.path.basename(first).lower() not in _SHELL_BINARIES:
return set()
found = set()
i = 1
while i < len(elts):
f = _extract_string_from_node(elts[i])
# -i / -ic (combined short flag) makes the shell INTERACTIVE, sourcing the user's rc
# files (.bashrc, with HOME = the workdir) before any -c payload runs -- unscanned
# startup code in the unguarded child. Mirror the shell-string interactive-rc block.
if f is not None and f.startswith("-") and not f.startswith("--") and "i" in f[1:]:
found.add("shell-interactive-rc:" + os.path.basename(first).lower())
if f is not None and (
f == "-c" or (f.startswith("-") and not f.startswith("--") and f.endswith("c"))
):
if i + 1 < len(elts):
payload = _extract_string_from_node(elts[i + 1])
if payload is None:
found.add("shell-dynamic-c") # unanalyzable inline payload
else:
found |= _find_blocked_commands(payload)
else:
found.add("shell-script:" + first) # -c with no payload
return found
i += 1
# No -c: a script file, -s (stdin), or a bare shell that reads stdin.
found.add("shell-script:" + first)
return found
def _check_args_for_blocked(
args_nodes,
shell_maybe_true = False,
cwd_prefix = "",
):
"""Check if any call arguments contain blocked commands. ``cwd_prefix`` is a synthetic
``env -C <dir> `` wrapper string prepended to a reconstructed argv command when the call
has a literal escaping ``cwd=`` (subprocess.run(['git','init','repo'], cwd='/tmp')), so the
git cwd backscan resolves the child's real working directory."""
found = set()
for arg in args_nodes:
s = _extract_string_from_node(arg)
if s is not None:
found |= _find_blocked_commands(s)
continue
if isinstance(arg, (ast.List, ast.Tuple)):
str_elts = [_extract_string_from_node(e) for e in arg.elts]
first = str_elts[0] if str_elts else None
# With shell=True, POSIX subprocess passes the FIRST sequence element to
# /bin/sh -c as the command string (the rest become $0, $1, ...); so
# subprocess.run(['echo x > /tmp/p'], shell=True) runs a full shell command,
# not an argv vector. Scan elts[0] with the shell parser in that case.
if shell_maybe_true and first is not None:
found |= _find_blocked_commands(first)
# A shell argv vector is analyzed as a whole so `['bash', '-c', 'echo hi']`
# scans the payload instead of tripping the bare-shell block on the 'bash'
# element. A non-shell argv only executes its command word (argv[0] plus any
# wrapper prefix), so scan just that -- scanning every element would misread a
# benign argument (subprocess.run(['echo', 'python'])) as a blocked command.
elif first is not None and os.path.basename(first).lower() in _SHELL_BINARIES:
found |= _check_shell_argv(arg.elts)
else:
_argv_blocked, _cmd_idx = _blocked_in_argv(str_elts)
found |= _argv_blocked
if _cmd_idx is not None and str_elts[_cmd_idx] is not None:
_cmd_base = os.path.basename(str_elts[_cmd_idx]).lower()
# A wrapper-hidden shell binary (env bash s.sh, nice sh -c '...')
# resolves to a shell as its command word; analyze the shell + its args
# (script file or -c payload) so the bare-shell / unscanned-script forms
# are caught.
if _cmd_base in _SHELL_BINARIES:
found |= _check_shell_argv(arg.elts[_cmd_idx:])
# find -exec/-delete, sed -i, sort -o interpret LATER argv elements as
# actions / write flags, and git -c alias.X=!CMD hides a shell dispatch
# in a config operand, so reconstruct a command line and reuse the full
# scanner (which handles those forms). Reconstruct from the FULL argv (not
# just the command word onward) so a preceding wrapper -- e.g. an escaping
# env -C /tmp before git -- is still seen by the git cwd backscan.
elif _cmd_base in _ARGV_TAIL_SCAN_COMMANDS:
found |= _find_blocked_commands(
cwd_prefix
+ " ".join(shlex.quote(s) for s in str_elts if s is not None)
)
# An env WRAPPER in the argv applies NAME=value assignments before the command
# (env PATH=. evil, env BASH_ENV=env.sh bash -c ..., env GIT_DIR=/tmp git init);
# the command-word resolution skips those assignment operands, so reconstruct
# the full argv and reuse the unsafe-PATH / startup-env / git-env checks.
if any(
s is not None and os.path.basename(s).lower() == "env" for s in str_elts
) and any(s is not None and _ASSIGNMENT_RE.match(s) for s in str_elts):
found |= _find_blocked_commands(
" ".join(shlex.quote(s) for s in str_elts if s is not None)
)
continue
for s in _extract_strings_from_list(arg):
found |= _find_blocked_commands(s)
return found
class SignalEscapeVisitor(ast.NodeVisitor):
def __init__(self):
self.imports_signal = False
self.signal_aliases = {"signal"}
self.os_aliases = {"os"}
self.subprocess_aliases = {"subprocess"}
self.importlib_aliases = {"importlib"}
self.sys_aliases = {"sys"}
# __builtins__ is the builtins *module* in __main__ (how the sandbox runs
# user code as `python <file>.py`), so builtins.eval / __builtins__.eval work.
self.builtins_aliases = {"builtins", "__builtins__"}
# Bare name -> fully-qualified form for from-import tracking
# (e.g. "system" -> "os.system").
self.shell_exec_aliases: dict[str, str] = {}
# from importlib import import_module as im -> {"im"}
self.import_func_aliases: set[str] = set()
# from builtins import exec as e -> {"e": "exec"}
self.exec_from_aliases: dict[str, str] = {}
# import pickle as p -> {"p": "pickle"}; from pickle import loads as l -> {"l": "pickle.loads"}
self.deserialize_module_aliases: dict[str, str] = {}
self.deserialize_aliases: dict[str, str] = {}
# `from yaml import load [as X]` / load_all: yaml.load is conditional (safe only
# with a SafeLoader) so it is not a static sink; track the bare-name alias so the
# safe-loader check can be applied to the direct-call form.
self.yaml_load_aliases: dict[str, str] = {}
# `from pickle import Unpickler [as X]`: Unpickler(f).load() reaches the same reduce
# path as pickle.load; track the ctor alias so the .load() method call is flagged.
self.unpickler_aliases: set[str] = set()
# import torch / numpy as np / joblib -> {alias: module}; from joblib import load ->
# {load: "joblib.load"}. Conditional pickle-backed loaders (see _pickle_loader_is_unsafe).
self.pickle_loader_module_aliases: dict[str, str] = {}
self.pickle_loader_func_aliases: dict[str, str] = {}
# import types as t -> {"types", "t"}; from types import FunctionType as F -> {"F"}.
# FunctionType(code, globals)() runs a code object WITHOUT eval/exec, so a
# dynamic compile() result reaches execution through it (see visit_Call).
self.types_aliases = {"types"}
self.functiontype_aliases: set[str] = set()
# import runpy as r -> {"runpy", "r"}. runpy.run_path/run_module execute a
# file/module in the guarded interpreter without the recursive source
# analysis exec/eval receive, so treat those calls as execution sinks.
self.runpy_aliases = {"runpy"}
# from runpy import run_path as X / run_module as Y -> {"X", "Y"}.
self.runpy_func_aliases: set[str] = set()
# import inspect as i -> {"inspect", "i"}. inspect.getclosurevars(fn) hands back
# the cells a guard wrapper closes over (the original unguarded callable), so
# treat it as a closure-recovery gadget like __closure__ / cell_contents.
self.inspect_aliases = {"inspect"}
# from inspect import getclosurevars as g -> {"g"}.
self.getclosurevars_aliases: set[str] = set()
# import operator as op -> {"operator", "op"}. operator.attrgetter('name')(obj)
# is the same attribute-fetch obfuscation as getattr(obj, 'name').
self.operator_aliases = {"operator"}
# from operator import attrgetter as ag -> {"ag"}.
self.attrgetter_aliases: set[str] = set()
# from operator import methodcaller as mc -> {"mc"}. methodcaller('__getattribute__',
# 'system')(os) fetches os.system, the same obfuscation as attrgetter.
self.methodcaller_aliases: set[str] = set()
# import gc as g -> {"gc", "g"}. gc.get_referents / get_referrers / get_objects
# walk the object graph to a guard wrapper's closure cell (the original unguarded
# callable) without spelling __closure__, so treat them as recovery gadgets.
self.gc_aliases = {"gc"}
# from gc import get_referents as gr -> {"gr"}.
self.gc_walk_aliases: set[str] = set()
# import pty as p -> {"pty", "p"}. pty.spawn([...]) / pty.fork() run an unguarded
# child process (a shell) outside the sandbox.
self.pty_aliases: set[str] = set()
# from pty import spawn as s -> {"s"}: bare-name aliases of the pty child sinks.
self.pty_func_aliases: set[str] = set()
self.loop_depth = 0
def visit_Import(self, node):
for alias in node.names:
if alias.name == "signal":
self.imports_signal = True
if alias.asname:
self.signal_aliases.add(alias.asname)
elif alias.name == "os":
self.os_aliases.add(alias.asname or "os")
elif alias.name in ("posix", "nt"):
# posix / nt are the C backend os wraps: posix.system(...) == os.system,
# and posix.exec*/spawn*/popen mirror os. Model them as os aliases so a
# direct `import posix; posix.system('...')` resolves to an os shell sink.
self.os_aliases.add(alias.asname or alias.name)
elif alias.name == "subprocess":
self.subprocess_aliases.add(alias.asname or "subprocess")
elif alias.name == "pty":
# pty.spawn([...]) / pty.fork() run an unguarded child process.
self.pty_aliases.add(alias.asname or "pty")
elif alias.name == "importlib":
self.importlib_aliases.add(alias.asname or "importlib")
elif alias.name == "sys":
self.sys_aliases.add(alias.asname or "sys")
elif alias.name == "builtins":
self.builtins_aliases.add(alias.asname or "builtins")
elif alias.name == "types":
self.types_aliases.add(alias.asname or "types")
elif alias.name == "runpy":
self.runpy_aliases.add(alias.asname or "runpy")
elif alias.name == "inspect":
self.inspect_aliases.add(alias.asname or "inspect")
elif alias.name == "operator":
self.operator_aliases.add(alias.asname or "operator")
elif alias.name == "gc":
self.gc_aliases.add(alias.asname or "gc")
if alias.name in _DESERIALIZE_MODULES:
self.deserialize_module_aliases[alias.asname or alias.name] = alias.name
# import torch / import numpy as np / import joblib: the top-level name (or its
# alias) is the receiver for torch.load / np.load / joblib.load.
_pl_top = alias.name.split(".")[0]
if _pl_top in _PICKLE_LOADER_MODULES and "." not in (alias.asname or alias.name):
self.pickle_loader_module_aliases[alias.asname or alias.name] = _pl_top
self.generic_visit(node)
def visit_ImportFrom(self, node):
if node.module == "signal":
self.imports_signal = True
for alias in node.names:
if alias.name in (
"signal",
"SIGALRM",
"SIG_IGN",
"setitimer",
"ITIMER_REAL",
"pthread_sigmask",
"SIG_BLOCK",
"alarm",
):
self.signal_aliases.add(alias.asname or alias.name)
elif node.module in ("os", "subprocess", "posix", "nt"):
if node.module == "subprocess":
self.subprocess_aliases.add("subprocess")
_eff_mod = "subprocess"
else:
# posix / nt are the C backend os wraps: `from posix import system` is
# os.system, so model the sink under os so it is caught the same way.
self.os_aliases.add("os")
_eff_mod = "os"
# Track from-imports of dangerous functions.
for alias in node.names:
fq = f"{_eff_mod}.{alias.name}"
if fq in _SHELL_EXEC_FUNCS:
self.shell_exec_aliases[alias.asname or alias.name] = fq
elif node.module == "importlib":
for alias in node.names:
if alias.name in ("import_module", "reload", "__import__"):
self.import_func_aliases.add(alias.asname or alias.name)
elif node.module == "builtins":
for alias in node.names:
if alias.name in _DYNAMIC_EXEC_BUILTINS:
self.exec_from_aliases[alias.asname or alias.name] = alias.name
elif alias.name == "__import__":
# `from builtins import __import__ as imp; imp('os').system(...)`
# is a dynamic import exactly like a bare __import__ call.
self.import_func_aliases.add(alias.asname or alias.name)
elif node.module in _DESERIALIZE_MODULES:
for alias in node.names:
fq = f"{node.module}.{alias.name}"
if fq in _CODE_DESERIALIZE_SINKS:
self.deserialize_aliases[alias.asname or alias.name] = fq
elif node.module == "yaml" and alias.name in _YAML_LOAD_METHODS:
self.yaml_load_aliases[alias.asname or alias.name] = alias.name
elif alias.name == "Unpickler" and node.module in _UNPICKLER_MODULES:
self.unpickler_aliases.add(alias.asname or alias.name)
elif node.module in _PICKLE_LOADER_MODULES:
# from joblib import load / from torch import load: the bare-name form of the
# conditional pickle-backed loader; the safe-flag gate is applied at the call.
for alias in node.names:
if alias.name == "load":
self.pickle_loader_func_aliases[alias.asname or alias.name] = (
f"{node.module}.load"
)
elif node.module == "types":
for alias in node.names:
if alias.name == "FunctionType":
self.functiontype_aliases.add(alias.asname or alias.name)
elif node.module == "runpy":
for alias in node.names:
if alias.name in ("run_path", "run_module"):
self.runpy_func_aliases.add(alias.asname or alias.name)
elif node.module == "pty":
# from pty import spawn / fork: bare-name aliases of the pty child sinks.
for alias in node.names:
if alias.name in ("spawn", "fork"):
self.pty_func_aliases.add(alias.asname or alias.name)
elif node.module == "inspect":
for alias in node.names:
if alias.name == "getclosurevars":
self.getclosurevars_aliases.add(alias.asname or alias.name)
elif node.module == "operator":
for alias in node.names:
if alias.name == "attrgetter":
self.attrgetter_aliases.add(alias.asname or alias.name)
elif alias.name == "methodcaller":
self.methodcaller_aliases.add(alias.asname or alias.name)
elif node.module == "gc":
for alias in node.names:
if alias.name in ("get_referents", "get_referrers", "get_objects"):
self.gc_walk_aliases.add(alias.asname or alias.name)
self.generic_visit(node)
def visit_While(self, node):
self.loop_depth += 1
self.generic_visit(node)
self.loop_depth -= 1
def visit_For(self, node):
self.loop_depth += 1
self.generic_visit(node)
self.loop_depth -= 1
def _resolve_container_sink(self, sub):
"""Resolve an inline literal-container index callee to a shell sink fq.
Covers ``[os.system][0]``, ``(os.system,)[0]`` and ``{'k': os.system}['k']``.
"""
def _elt(elt):
fq = _resolve_static_shell_sink(
elt, self.os_aliases, self.subprocess_aliases, self.shell_exec_aliases
)
if fq:
return fq
if isinstance(elt, ast.Name):
return _scope_idx.resolve(elt.id, elt, "shell")
return None
container = sub.value
# Resolve a single-assignment NAME container (d = [os.system]; d[0]('rm -rf /')) to its
# literal, mirroring the exec-container resolver -- a shell sink hidden in an assigned
# container was otherwise missed because the callee is an indexed Name.
if isinstance(container, ast.Name) and container.id in _const_env:
container = _const_env[container.id]
ci = _const_fold(sub.slice, _const_env)
if isinstance(container, (ast.List, ast.Tuple)) and isinstance(ci, int):
if -len(container.elts) <= ci < len(container.elts):
return _elt(container.elts[ci])
if isinstance(container, ast.Dict) and ci is not None:
for k, v in zip(container.keys, container.values):
if k is not None and _const_fold(k, _const_env) == ci:
return _elt(v)
return None
def _resolve_container_exec(self, sub):
"""Resolve an inline literal-container index callee to a dynamic-exec builtin.
Covers ({'e': exec}['e'])(...), [exec][0](...), (eval,)[0](...): an inline
container hiding an eval/exec/compile sink from the bare-name recursion."""
def _elt(elt):
if isinstance(elt, ast.Name):
if elt.id in _DYNAMIC_EXEC_BUILTINS:
return elt.id
if elt.id in self.exec_from_aliases:
return self.exec_from_aliases[elt.id]
if _analyzer_on:
return _scope_idx.resolve(elt.id, elt, "execb")
if (
isinstance(elt, ast.Attribute)
and elt.attr in _DYNAMIC_EXEC_BUILTINS
and _ast_name_matches(elt.value, self.builtins_aliases)
):
return elt.attr
return None
container = sub.value
# Resolve a subscript into a container bound to a single-assignment NAME
# (d = {'e': exec}; d['e'](...), xs = [eval]; xs[0](...)) to the literal container,
# so the exec/eval sink hidden inside it is not missed just because the callee is an
# indexed Name rather than an inline literal.
if isinstance(container, ast.Name) and container.id in _const_env:
container = _const_env[container.id]
ci = _const_fold(sub.slice, _const_env)
if isinstance(container, (ast.List, ast.Tuple)) and isinstance(ci, int):
if -len(container.elts) <= ci < len(container.elts):
return _elt(container.elts[ci])
if isinstance(container, ast.Dict) and ci is not None:
for k, v in zip(container.keys, container.values):
if k is not None and _const_fold(k, _const_env) == ci:
return _elt(v)
return None
def _resolve_container_deser(self, sub):
"""Resolve an inline literal-container index callee to a deserializer sink fq.
Covers ([pickle.loads][0])(payload), (pickle.loads,)[0](...) and
{'k': pickle.loads}['k'](...): an inline container hiding a pickle/marshal
reduce sink from the attribute/name deserializer checks."""
def _elt(elt):
if isinstance(elt, ast.Attribute):
if isinstance(elt.value, ast.Name):
canon = self.deserialize_module_aliases.get(elt.value.id)
if canon is not None:
cand = f"{canon}.{elt.attr}"
if cand in _CODE_DESERIALIZE_SINKS:
return cand
fq = _fq_attr_name(elt)
if fq in _CODE_DESERIALIZE_SINKS:
return fq
elif isinstance(elt, ast.Name):
fq = self.deserialize_aliases.get(elt.id)
if fq is not None:
return fq
if _analyzer_on:
return _scope_idx.resolve(elt.id, elt, "deser")
return None
container = sub.value
# Resolve a single-assignment NAME container (d = {'k': pickle.loads}; d['k'](payload))
# to its literal, mirroring the exec-container resolver above.
if isinstance(container, ast.Name) and container.id in _const_env:
container = _const_env[container.id]
ci = _const_fold(sub.slice, _const_env)
if isinstance(container, (ast.List, ast.Tuple)) and isinstance(ci, int):
if -len(container.elts) <= ci < len(container.elts):
return _elt(container.elts[ci])
if isinstance(container, ast.Dict) and ci is not None:
for k, v in zip(container.keys, container.values):
if k is not None and _const_fold(k, _const_env) == ci:
return _elt(v)
return None
def _is_unpickler_ctor(self, recv):
"""True when ``recv`` constructs a pickle/dill Unpickler instance.
Covers pickle.Unpickler(f) (incl. an aliased module: import pickle as p;
p.Unpickler(f)) and a from-imported ctor (from pickle import Unpickler;
Unpickler(f)); its .load() runs the same reduce payload as pickle.load."""
if not isinstance(recv, ast.Call):
return False
cf = recv.func
if isinstance(cf, ast.Attribute) and cf.attr == "Unpickler":
if isinstance(cf.value, ast.Name):
return self.deserialize_module_aliases.get(cf.value.id) in _UNPICKLER_MODULES
return _fq_attr_name(cf) in {m + ".Unpickler" for m in _UNPICKLER_MODULES}
if isinstance(cf, ast.Name):
return cf.id in self.unpickler_aliases
return False
def _attrgetter_name(self, n):
"""Return the single attribute name for an ``operator.attrgetter('name')``
call (or a ``from operator import attrgetter`` alias), else None. A dotted or
multi-attr getter (attrgetter('a.b'), attrgetter('a', 'b')) returns None."""
if not isinstance(n, ast.Call) or len(n.args) != 1 or n.keywords:
return None
af = n.func
is_attrgetter = (
isinstance(af, ast.Attribute)
and af.attr == "attrgetter"
and _ast_name_matches(af.value, self.operator_aliases)
) or (isinstance(af, ast.Name) and af.id in self.attrgetter_aliases)
if not is_attrgetter:
return None
name = _const_fold(n.args[0], _const_env)
if isinstance(name, str) and "." not in name:
return name
return None
def _methodcaller_getattr_name(self, n):
"""Return the attribute name for an ``operator.methodcaller('__getattribute__',
'name')`` / ``__getattr__`` call (or a from-import alias), else None. This form
fetches ``obj.name`` exactly like attrgetter, so it needs the same normalization."""
if not isinstance(n, ast.Call) or len(n.args) != 2 or n.keywords:
return None
af = n.func
is_mc = (
isinstance(af, ast.Attribute)
and af.attr == "methodcaller"
and _ast_name_matches(af.value, self.operator_aliases)
) or (isinstance(af, ast.Name) and af.id in self.methodcaller_aliases)
if not is_mc:
return None
meth = _const_fold(n.args[0], _const_env)
if meth in ("__getattribute__", "__getattr__"):
name = _const_fold(n.args[1], _const_env)
if isinstance(name, str) and "." not in name:
return name
return None
def _methodcaller_module_call(self, node):
"""Rewrite ``operator.methodcaller('meth', *args)(receiver)`` into the equivalent
``receiver.meth(*args)`` Call when the receiver is an os/subprocess module
reference, so a methodcaller-hidden sink -- methodcaller('system', 'rm -rf /')(os)
-- is analyzed exactly like the direct os.system('rm -rf /') call. Returns the
synthetic Call node (with the original location) or None when the pattern does not
apply. Only os/subprocess receivers are rewritten; a methodcaller aimed at some
other object is left untouched so benign method calls are not misread."""
if len(node.args) != 1 or node.keywords:
return None
mc = node.func
while isinstance(mc, ast.Attribute) and mc.attr == "__call__":
mc = mc.value
if not isinstance(mc, ast.Call) or not mc.args:
return None
mf = mc.func
is_mc = (
isinstance(mf, ast.Attribute)
and mf.attr == "methodcaller"
and _ast_name_matches(mf.value, self.operator_aliases)
) or (isinstance(mf, ast.Name) and mf.id in self.methodcaller_aliases)
if not is_mc:
return None
meth = _const_fold(mc.args[0], _const_env)
if not isinstance(meth, str) or not meth.isidentifier():
return None
receiver = node.args[0]
if not (
isinstance(receiver, ast.Name)
and (receiver.id in self.os_aliases or receiver.id in self.subprocess_aliases)
):
return None
synth = ast.Call(
func = ast.Attribute(value = receiver, attr = meth, ctx = ast.Load()),
args = list(mc.args[1:]),
keywords = list(mc.keywords),
)
ast.copy_location(synth, node)
ast.fix_missing_locations(synth)
return synth
def _sink_ref_desc(self, n):
"""Describe ``n`` when it is a bare reference to a dangerous callable used as a
first-class VALUE (map/reduce/partial argument): a dynamic-exec builtin, a shell
sink (os.system / subprocess.*), a dynamic-import function, or a code
deserializer. Returns a short description or None. The payloads such a sink runs
never reach the recursive analyzer, so passing one by reference is unsafe."""
if isinstance(n, ast.Subscript):
# An inline literal-container index hides the sink from the name/attribute
# checks: map([eval][0], [...]) / partial({'e': exec}['e'], ...). Resolve the
# element node and describe it, the same unwrap direct calls already apply.
container = n.value
ci = _const_fold(n.slice, _const_env)
elt = None
if isinstance(container, (ast.List, ast.Tuple)) and isinstance(ci, int):
if -len(container.elts) <= ci < len(container.elts):
elt = container.elts[ci]
elif isinstance(container, ast.Dict) and ci is not None:
for _k, _v in zip(container.keys, container.values):
if _k is not None and _const_fold(_k, _const_env) == ci:
elt = _v
break
return self._sink_ref_desc(elt) if elt is not None else None
if isinstance(n, ast.Name):
if n.id in _DYNAMIC_EXEC_BUILTINS:
return f"{n.id} (dynamic exec)"
if n.id in self.exec_from_aliases:
return f"{self.exec_from_aliases[n.id]} (dynamic exec)"
if n.id in ("__import__", "import_module") or n.id in self.import_func_aliases:
return "dynamic import"
_sh = self.shell_exec_aliases.get(n.id)
if _sh in _SHELL_EXEC_FUNCS:
return f"{_sh} (shell)"
_ds = self.deserialize_aliases.get(n.id)
if _ds is not None:
return f"{_ds} (deserialize)"
if _analyzer_on:
_r = _scope_idx.resolve(n.id, n, "shell")
if _r in _SHELL_EXEC_FUNCS:
return f"{_r} (shell)"
if _scope_idx.resolve(n.id, n, "execb") in _DYNAMIC_EXEC_BUILTINS:
return f"{_scope_idx.resolve(n.id, n, 'execb')} (dynamic exec)"
_rd = _scope_idx.resolve(n.id, n, "deser")
if _rd:
return f"{_rd} (deserialize)"
return None
if isinstance(n, ast.Attribute):
if n.attr in _DYNAMIC_EXEC_BUILTINS and _ast_name_matches(
n.value, self.builtins_aliases
):
return f"{n.attr} (dynamic exec)"
_sh = _resolve_static_shell_sink(
n, self.os_aliases, self.subprocess_aliases, self.shell_exec_aliases
)
if _sh in _SHELL_EXEC_FUNCS:
return f"{_sh} (shell)"
if isinstance(n.value, ast.Name):
_c = self.deserialize_module_aliases.get(n.value.id)
if _c is not None and f"{_c}.{n.attr}" in _CODE_DESERIALIZE_SINKS:
return f"{_c}.{n.attr} (deserialize)"
_fq = _fq_attr_name(n)
if _fq in _CODE_DESERIALIZE_SINKS:
return f"{_fq} (deserialize)"
if _fq in _SHELL_EXEC_FUNCS:
return f"{_fq} (shell)"
return None
return None
def _rhs_module_attr(self, func, attrs, mod_aliases):
"""A single-assignment alias (x = mod.attr; x(...)) resolving to an attribute in
``attrs`` on a module in ``mod_aliases``. Returns the attribute name or None, so a
re-bound execution sink (r = runpy.run_path, s = pty.spawn) is caught the same as
the direct call."""
if not (_analyzer_on and isinstance(func, ast.Name)):
return None
rhs = _scope_idx.resolve(func.id, func, "rhsnode")
if (
isinstance(rhs, ast.Attribute)
and rhs.attr in attrs
and _ast_name_matches(rhs.value, mod_aliases)
):
return rhs.attr
return None
def _is_sys_modules_expr(self, n):
# `sys.modules` (attribute form) or getattr(sys, 'modules') (the getattr-
# obfuscated form) -- both denote the loader table itself.
if (
isinstance(n, ast.Attribute)
and n.attr == "modules"
and _ast_name_matches(n.value, self.sys_aliases)
):
return True
if (
isinstance(n, ast.Call)
and isinstance(n.func, ast.Name)
and n.func.id == "getattr"
and len(n.args) >= 2
and _ast_name_matches(n.args[0], self.sys_aliases)
and _extract_string_from_node(n.args[1]) == "modules"
):
return True
# object.__getattribute__(sys, 'modules') / type(sys).__getattribute__(sys,
# 'modules'): the unbound dunder accessor reaches the loader table exactly like
# getattr, so treat it the same.
if (
isinstance(n, ast.Call)
and isinstance(n.func, ast.Attribute)
and n.func.attr in ("__getattribute__", "__getattr__")
and len(n.args) >= 2
and _ast_name_matches(n.args[0], self.sys_aliases)
and _extract_string_from_node(n.args[1]) == "modules"
):
return True
return False
def _is_sys_modules(self, n):
# `sys.modules` / getattr(sys, 'modules'), or a single-assignment alias of
# either (m = sys.modules; m.pop('_io')). Used by the loader-table mutation
# checks (sys.modules.pop('posix'); import posix drops the guard-patched module).
if self._is_sys_modules_expr(n):
return True
if _analyzer_on and isinstance(n, ast.Name):
rhs = _scope_idx.resolve(n.id, n, "rhsnode")
if rhs is not None and self._is_sys_modules_expr(rhs):
return True
return False
def _is_sys_meta_path_expr(self, n):
# `sys.meta_path` (attribute form) or getattr(sys, 'meta_path') -- the import
# finder chain into which the sandbox installs its workdir-module vetter.
if (
isinstance(n, ast.Attribute)
and n.attr == "meta_path"
and _ast_name_matches(n.value, self.sys_aliases)
):
return True
if (
isinstance(n, ast.Call)
and isinstance(n.func, ast.Name)
and n.func.id == "getattr"
and len(n.args) >= 2
and _ast_name_matches(n.args[0], self.sys_aliases)
and _extract_string_from_node(n.args[1]) == "meta_path"
):
return True
if (
isinstance(n, ast.Call)
and isinstance(n.func, ast.Attribute)
and n.func.attr in ("__getattribute__", "__getattr__")
and len(n.args) >= 2
and _ast_name_matches(n.args[0], self.sys_aliases)
and _extract_string_from_node(n.args[1]) == "meta_path"
):
return True
return False
def _is_sys_meta_path(self, n):
# `sys.meta_path` (or getattr form), or a single-assignment alias of either
# (mp = sys.meta_path; mp.pop(0)). Used by the import-hook mutation checks: removing
# / reordering the vetter lets a planted workdir module import without source review.
if self._is_sys_meta_path_expr(n):
return True
if _analyzer_on and isinstance(n, ast.Name):
rhs = _scope_idx.resolve(n.id, n, "rhsnode")
if rhs is not None and self._is_sys_meta_path_expr(rhs):
return True
return False
def _is_namespace_dict_expr(self, n):
# globals() / locals() / vars() with no args, or a single-assignment alias of one
# (g = globals(); g['__builtins__']). Used by the namespace-dict subscript check.
def _direct(x):
return (
isinstance(x, ast.Call)
and isinstance(x.func, ast.Name)
and x.func.id in ("globals", "locals", "vars")
and not x.args
)
if _direct(n):
return True
if _analyzer_on and isinstance(n, ast.Name):
rhs = _scope_idx.resolve(n.id, n, "rhsnode")
if rhs is not None and _direct(rhs):
return True
return False
def _is_builtins_ref(self, n):
# The builtins module (builtins / __builtins__ / an import alias), or a
# single-assignment alias of one (b = __builtins__; b.__import__('os')).
if _ast_name_matches(n, self.builtins_aliases):
return True
if _analyzer_on and isinstance(n, ast.Name):
rhs = _scope_idx.resolve(n.id, n, "rhsnode")
if rhs is not None and _ast_name_matches(rhs, self.builtins_aliases):
return True
return False
def _functiontype_arg_is_vetted(self, arg):
"""True only when a ``types.FunctionType(code, ...)`` first arg is statically KNOWN to
be an ordinary in-source function's code object -- ``fn.__code__`` / ``meth.__func__``
-- whose body the analyzer already walked. Fails CLOSED for everything else: a
``compile(...)`` result, a loader's ``get_code()``, ``codeop.compile_command()``,
``marshal.loads()``, a bare name / alias, or a container unwrap all yield a code object
running source the recursive eval/exec analysis never saw, so FunctionType is the
execution gadget and must be blocked. (Denylisting only ``compile`` left other producers
open; an allowlist of the one benign shape is robust against new producers.)"""
return isinstance(arg, ast.Attribute) and arg.attr in ("__code__", "__func__")
def _attr_obfuscation_targets(self):
# Modules whose DYNAMIC attribute / dict access (getattr, vars, __dict__) is
# obfuscation that reaches code execution: the exec / import / shell modules
# PLUS the deserializer modules -- getattr(pickle, 'loads')(x) and
# vars(pickle)['loads'](x) are just pickle.loads(x) with the name hidden.
return (
_DYNAMIC_ATTR_TARGETS
| self.os_aliases
| self.subprocess_aliases
| self.importlib_aliases
| self.sys_aliases
| self.builtins_aliases
| set(self.deserialize_module_aliases)
)
def _stores_hidden_sink(self, value):
# A dynamic-exec builtin (exec / eval / compile / __import__), bare / builtins-attr /
# scope alias, being stashed for later obfuscated invocation.
if isinstance(value, ast.Name):
if value.id in _DYNAMIC_EXEC_BUILTINS or value.id == "__import__":
return True
if value.id in self.exec_from_aliases:
return True
if _analyzer_on and _scope_idx.resolve(value.id, value, "execb"):
return True
if (
isinstance(value, ast.Attribute)
and value.attr in (_DYNAMIC_EXEC_BUILTINS | {"__import__"})
and _ast_name_matches(value.value, self.builtins_aliases)
):
return True
return False
def _code_store_rhs_vetted(self, rhs):
# A value assigned to fn.__code__ that we can prove is safe to execute. An in-source
# function's code (g.__code__ / meth.__func__) is analyzed normally, and a compile()
# result -- direct call or a c = compile(...) alias -- has its SOURCE analyzed at the
# compile site (a malicious / opaque source is flagged there). Everything else (a
# producer code object from codeop / a loader's get_code() / marshal, or an opaque
# name) is unvetted and fails closed.
if isinstance(rhs, ast.Attribute) and rhs.attr in ("__code__", "__func__"):
return True
if isinstance(rhs, ast.Call):
rf = rhs.func
if isinstance(rf, ast.Name) and (
rf.id == "compile" or self.exec_from_aliases.get(rf.id) == "compile"
):
return True
if (
isinstance(rf, ast.Attribute)
and rf.attr == "compile"
and _ast_name_matches(rf.value, self.builtins_aliases)
):
return True
if _analyzer_on and isinstance(rhs, ast.Name):
if _scope_idx.resolve(rhs.id, rhs, "compiledany"):
return True
if _scope_idx.resolve(rhs.id, rhs, "execb") == "compile":
return True
return False
def _is_environ_receiver(self, _v):
# os.environ / os.environb (or a bare `environ` / `environb` from `from os import ...`).
# environb is the SAME inherited process environment via byte keys/values.
return (
isinstance(_v, ast.Attribute)
and _v.attr in ("environ", "environb")
and isinstance(_v.value, ast.Name)
and _v.value.id in self.os_aliases
) or (isinstance(_v, ast.Name) and _v.id in ("environ", "environb"))
def _environ_subscript_key(self, target):
# The literal key of an os.environ[...] / os.environb[...] (or a bare environ[...] /
# environb[...]) subscript assignment target; None otherwise. A bytes key is decoded.
if not isinstance(target, ast.Subscript):
return None
if not self._is_environ_receiver(target.value):
return None
return _extract_env_scalar(target.slice)
def _env_mutation_escape(self, key, value_node):
# A short reason when setting env var ``key`` to ``value_node`` is a child-escape
# prelude (mirrors the subprocess env={...} mapping analysis), else None. The mutated
# process environment is inherited by a later unguarded child.
_vs = _extract_env_scalar(value_node)
if key == "PATH":
if isinstance(_vs, str):
if _path_value_is_unsafe(_vs):
return "PATH set to a relative / cwd entry (a bare argv resolves to a workdir exec)"
return None
# A non-literal PATH value that provably prepends / embeds a relative / cwd entry
# ('.:' + os.environ['PATH'], f'.:{x}'); a dynamic ABSOLUTE extension stays allowed.
if _dynamic_path_value_unsafe(value_node, _const_env):
return "PATH prepends a relative / cwd entry (dynamic value)"
return None
if key in ("BASH_ENV", "ENV"):
return None if _vs == "" else "a shell startup file a child shell sources"
if isinstance(key, str) and key.startswith("GIT_CONFIG"):
return "overrides git config / drops the sandbox hook suppression"
if key in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE"):
if isinstance(_vs, str) and _arg_escapes_workdir(_vs):
return "points git's repo / tree outside the workdir"
return None
return None
def visit_Assign(self, node):
# os.environ['PATH'] = '.' (or BASH_ENV / ENV / GIT_CONFIG* / GIT_DIR) mutates the
# INHERITED environment a later unguarded subprocess child reads, the same escape as
# passing env={...} to the child: a bare-argv workdir exec via PATH='.', a sourced
# BASH_ENV script, or a dropped GIT_CONFIG hook suppression. Flag the mutation itself.
for _t in node.targets:
_envkey = self._environ_subscript_key(_t)
if _envkey is not None:
_reason = self._env_mutation_escape(_envkey, node.value)
if _reason is not None:
shell_escapes.append(
{
"type": "shell_escape",
"line": getattr(node, "lineno", -1),
"description": f"os.environ[{_envkey!r}] mutation: {_reason}",
}
)
# d['e'] = exec / lst[0] = eval -- storing a dynamic-exec builtin into a CONTAINER
# element (not a plain name, which the alias tracker already follows) hides the sink
# from the name / attribute call checks, and the later d['e'](payload) then runs
# unreviewed. There is no benign reason to stash exec / eval / compile / __import__ in
# a container slot, so flag the store itself.
if any(isinstance(t, ast.Subscript) for t in node.targets) and self._stores_hidden_sink(
node.value
):
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": (
"a dynamic-exec builtin stored into a container element "
"(obfuscated exec alias)"
),
}
)
# fn.__code__ = <code object> rebinds a function's body, so fn() then runs that code
# WITHOUT eval / exec. A code object from an unvetted producer (codeop.compile_command,
# a loader's get_code(), marshal) runs source the recursive analysis never saw, the
# __code__ twin of the FunctionType gadget. Flag a __code__ store whose RHS is not a
# vetted in-source / compile()-analyzed code object.
if any(
isinstance(t, ast.Attribute) and t.attr == "__code__" for t in node.targets
) and not self._code_store_rhs_vetted(node.value):
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": (
"an unvetted code object assigned to __code__ "
"(executes via the function without eval/exec)"
),
}
)
self.generic_visit(node)
def visit_Call(self, node):
# operator.methodcaller('system', 'rm -rf /')(os) applies a deferred method to a
# module receiver; rewrite it to the direct os.system('rm -rf /') call and analyze
# that instead so the hidden shell/exec sink is not missed.
_mc_rewrite = self._methodcaller_module_call(node)
if _mc_rewrite is not None:
self.visit_Call(_mc_rewrite)
return
# os.environ.update({'PATH': '.:...'}) / .update(PATH='...') / .setdefault('PATH', ...)
# (and the os.environb byte forms) mutate the inherited environment WITHOUT a subscript
# assignment, the same child escape as os.environ['PATH'] = ...; run each (key, value)
# pair through the mutation policy.
_mf = node.func
if isinstance(_mf, ast.Attribute) and _mf.attr in ("update", "setdefault"):
if self._is_environ_receiver(_mf.value):
_pairs = []
if _mf.attr == "setdefault" and len(node.args) >= 2:
_sk = _extract_env_scalar(node.args[0])
if _sk is not None:
_pairs.append((_sk, node.args[1]))
elif _mf.attr == "update":
if node.args and isinstance(node.args[0], ast.Dict):
for _kn, _vn in zip(node.args[0].keys, node.args[0].values):
if _kn is not None:
_dk = _extract_env_scalar(_kn)
if _dk is not None:
_pairs.append((_dk, _vn))
for _kw in node.keywords:
if _kw.arg is not None:
_pairs.append((_kw.arg, _kw.value))
for _pk, _pvn in _pairs:
_preason = self._env_mutation_escape(_pk, _pvn)
if _preason is not None:
shell_escapes.append(
{
"type": "shell_escape",
"line": getattr(node, "lineno", -1),
"description": (
f"os.environ.{_mf.attr}({_pk!r}) mutation: {_preason}"
),
}
)
if self._is_unbound_mro_gadget(node):
# type.mro(io.FileIO) / type.__getattribute__(io.FileIO, '__mro__') /
# getattr(io.FileIO, 'mro'): reaches the unguarded MRO without a .mro / .__mro__
# attribute for visit_Attribute to see.
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": "unbound MRO access on a file class recovers an unguarded base (gadget)",
}
)
func = node.func
# A trailing `.__call__` invokes the underlying callable through its bound
# method: os.system.__call__(cmd), __import__.__call__('os'),
# pickle.loads.__call__(blob). Strip it so the shell / import / deserializer /
# attribute resolvers below see the real sink instead of a plain attribute.
_ecf = func
while isinstance(_ecf, ast.Attribute) and _ecf.attr == "__call__":
_ecf = _ecf.value
func_name = None
if isinstance(func, ast.Attribute):
if isinstance(func.value, ast.Name):
if func.value.id in self.signal_aliases:
func_name = f"signal.{func.attr}"
elif isinstance(func, ast.Name):
if func.id in ("signal", "setitimer", "alarm", "pthread_sigmask"):
func_name = func.id
if func_name:
if func_name in ("signal.signal", "signal"):
if len(node.args) >= 1:
if _ast_name_matches(node.args[0], ("SIGALRM", "signal.SIGALRM")):
signal_tampering.append(
{
"type": "signal_handler_override",
"line": node.lineno,
"description": "Overrides SIGALRM handler",
}
)
elif func_name in ("signal.setitimer", "setitimer"):
if len(node.args) >= 1:
if _ast_name_matches(node.args[0], ("ITIMER_REAL", "signal.ITIMER_REAL")):
signal_tampering.append(
{
"type": "timer_manipulation",
"line": node.lineno,
"description": "Manipulates ITIMER_REAL timer",
}
)
elif func_name in ("signal.alarm", "alarm"):
signal_tampering.append(
{
"type": "alarm_manipulation",
"line": node.lineno,
"description": "Manipulates alarm timer",
}
)
elif func_name in ("signal.pthread_sigmask", "pthread_sigmask"):
signal_tampering.append(
{
"type": "signal_mask",
"line": node.lineno,
"description": "Modifies signal mask (may block SIGALRM)",
}
)
# --- Shell escape detection ---
# Resolve the FQ function name for os.*/subprocess.* (via the __call__-stripped
# effective callee so os.system.__call__(cmd) resolves to os.system).
shell_func = None
if isinstance(_ecf, ast.Attribute):
if isinstance(_ecf.value, ast.Name):
if _ecf.value.id in self.os_aliases:
shell_func = f"os.{_ecf.attr}"
elif _ecf.value.id in self.subprocess_aliases:
shell_func = f"subprocess.{_ecf.attr}"
# class-body alias reached as ClassName.attr (class C: f = os.system;
# C.f('rm -rf /')), or an instance-attribute alias (obj.s = os.system;
# obj.s('rm -rf /')).
elif _analyzer_on:
shell_func = _scope_idx.resolve_class_attr(
_ecf.value.id, _ecf.attr, "shell"
) or _scope_idx.resolve_instance_attr(_ecf.value.id, _ecf.attr, "shell")
elif (
_analyzer_on
and isinstance(_ecf.value, ast.Call)
and isinstance(_ecf.value.func, ast.Name)
):
# An instance built inline, ClassName().attr: instance lookup still returns
# the class-body sink alias. class C: s = os.system; C().s('rm -rf /').
shell_func = _scope_idx.resolve_class_attr(
_ecf.value.func.id, _ecf.attr, "shell"
)
elif isinstance(_ecf.value, ast.Attribute) and _ecf.value.attr in ("os", "posix"):
# A stdlib module that re-exports os as an attribute (pathlib.os.system,
# tempfile.os.system, subprocess.os.system): the `.os` attribute IS the os
# module, so treat the chain as an os.* sink.
shell_func = f"os.{_ecf.attr}"
elif isinstance(_ecf.value, ast.Attribute) and _ecf.value.attr == "subprocess":
# ...and *.subprocess.run (a module re-exporting subprocess).
shell_func = f"subprocess.{_ecf.attr}"
if shell_func is None:
# getattr(<mod>, 'os').system(...) / vars(<mod>)['os'].system(...) /
# <mod>.__dict__['subprocess'].run(...): a re-export fetched by NAME -- the
# call / subscript twin of the <mod>.os.system attribute form, and reachable
# off a call-returned module (getattr(__import__('pathlib'), 'os')). Map the
# fetched os / posix / subprocess to the sink module so the chain is a sink.
_rx = _reexport_dangerous_module_name(_ecf.value)
if _rx:
shell_func = f"{_rx}.{_ecf.attr}"
elif isinstance(_ecf, ast.Name):
# from-import aliases: from os import system; system(...)
shell_func = self.shell_exec_aliases.get(_ecf.id)
# Stage 4: single-assignment alias `s = os.system; s('rm -rf /')`,
# resolved in the call's own scope (per-function).
if shell_func is None and _analyzer_on:
shell_func = _scope_idx.resolve(_ecf.id, _ecf, "shell")
elif _analyzer_on and isinstance(_ecf, ast.Subscript):
# Stage 4: inline literal container index `[os.system][0](...)`.
shell_func = self._resolve_container_sink(_ecf)
if shell_func and shell_func in _SHELL_EXEC_FUNCS:
# Expand **kwargs dicts to inspect their keys.
expanded_kwargs: dict[str, ast.AST] = {}
has_opaque_kwargs = False
for kw in node.keywords:
if kw.arg is not None:
expanded_kwargs[kw.arg] = kw.value
elif isinstance(kw.value, ast.Dict):
for k, v in zip(kw.value.keys, kw.value.values):
key = _extract_string_from_node(k) if k else None
if key is not None:
expanded_kwargs[key] = v
else:
has_opaque_kwargs = True
cmd_kw_values = [v for k, v in expanded_kwargs.items() if k in _CMD_KWARGS]
all_call_args = list(node.args) + cmd_kw_values
# A non-literal-False shell= is treated as potentially True (conservative), so a
# sequence's first element is the shell -c command string, not an argv vector.
_shell_node = expanded_kwargs.get("shell")
_shell_maybe_true = not (
_shell_node is None
or (isinstance(_shell_node, ast.Constant) and _shell_node.value is False)
)
# A literal cwd= that escapes the workdir sets the child's real working directory,
# so a relative write operand (subprocess.run(['git','init','repo'], cwd='/tmp'))
# lands OUTSIDE the session. Model it as a synthetic `env -C <cwd>` wrapper so the
# git cwd backscan resolves the escape; a workdir-relative / in-tree cwd adds no
# prefix and stays allowed.
_cwd_node = expanded_kwargs.get("cwd")
_cwd_str = _extract_string_from_node(_cwd_node) if _cwd_node is not None else None
_cwd_prefix = ""
if _cwd_str is not None and _arg_escapes_workdir(_cwd_str):
_cwd_prefix = "env -C " + shlex.quote(_cwd_str) + " "
blocked_in_args = _check_args_for_blocked(
all_call_args, _shell_maybe_true, _cwd_prefix
)
# The argv sequence can be given positionally (run(['bash', ...])) or through the
# public args= keyword (run(args=['bash', ...])), which this analyzer already
# collects in _CMD_KWARGS. Resolve either form so the executable= and shell-child
# checks below are not bypassed by moving the command into args=.
_argv0_node = node.args[0] if node.args else expanded_kwargs.get("args")
# subprocess(..., executable=PROG) makes PROG the real program while the argv
# TAIL still supplies its flags/args, so scanning executable and argv separately
# misses run(['x', '-i', 's/a/b/', '/f'], executable='/usr/bin/sed') (child runs
# sed -i). Reconstruct PROG + argv[1:] and scan the effective command line.
_exe_node = expanded_kwargs.get("executable")
_exe = _extract_string_from_node(_exe_node) if _exe_node is not None else None
if (
_exe is not None
and not _shell_maybe_true
and isinstance(_argv0_node, (ast.List, ast.Tuple))
):
_tail = [_extract_string_from_node(e) for e in _argv0_node.elts[1:]]
_combined = [_exe] + _tail
if all(_c is not None for _c in _combined):
blocked_in_args = blocked_in_args | _find_blocked_commands(
" ".join(shlex.quote(_c) for _c in _combined)
)
# A shell startup variable (BASH_ENV / ENV) in the env= mapping names a script
# bash / sh SOURCES before the -c payload runs, executing unscanned code
# (subprocess.run(['bash','-c','echo OK'], env={'BASH_ENV':'env.sh'})). Flag a
# non-empty value; an empty string is inert. Cover a literal dict, a dict(...)
# call, and -- for a shell child -- a non-literal mapping we cannot prove free
# of BASH_ENV / ENV (fail closed). Whether the child is a shell: shell=True, or
# the argv command word resolves to bash / sh.
_env_node = expanded_kwargs.get("env")
# A single-assignment env mapping (e = {'PATH': '.'}; run(['evil'], env=e)) reaches
# here as a Name; resolve it to its literal dict / dict() so the BASH_ENV and
# unsafe-PATH checks below still apply instead of silently passing.
if isinstance(_env_node, ast.Name) and _analyzer_on:
_renv = _scope_idx.resolve(_env_node.id, _env_node, "rhsnode")
if isinstance(_renv, (ast.Dict, ast.Call)):
_env_node = _renv
if _env_node is not None:
_is_shell_child = _shell_maybe_true
_is_git_child = False
if isinstance(_argv0_node, (ast.List, ast.Tuple)):
_elts0 = [_extract_string_from_node(_e) for _e in _argv0_node.elts]
_ci0 = _blocked_in_argv(_elts0)[1]
if _ci0 is not None and _ci0 < len(_elts0) and _elts0[_ci0]:
_cw0 = os.path.basename(_elts0[_ci0]).lower()
if not _is_shell_child:
_is_shell_child = _cw0 in _SHELL_BINARIES
_is_git_child = _cw0 == "git"
if isinstance(_env_node, ast.Dict):
_opaque_key = False
for _ek, _ev in zip(_env_node.keys, _env_node.values):
_ekey = _extract_string_from_node(_ek) if _ek is not None else None
_evstr = _extract_string_from_node(_ev)
if _ekey in ("BASH_ENV", "ENV") and _evstr != "":
blocked_in_args = blocked_in_args | {"shell-startup-env:" + _ekey}
elif (
_ekey == "PATH"
and isinstance(_evstr, str)
and _path_value_is_unsafe(_evstr)
):
# env={'PATH': '.'} lets a bare argv[0] resolve to a workdir exec.
blocked_in_args = blocked_in_args | {"unsafe-path-assign"}
elif (
_ekey in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE")
and _is_git_child
and isinstance(_evstr, str)
and _arg_escapes_workdir(_evstr)
):
# env={'GIT_DIR': '/tmp/x'} points git's repo outside the workdir.
blocked_in_args = blocked_in_args | {"git-write-outside"}
elif (
_ekey is not None
and _ekey.startswith("GIT_CONFIG")
and _is_git_child
):
# env={'GIT_CONFIG_COUNT': '0'} drops the sandbox hook suppression.
blocked_in_args = blocked_in_args | {"git-config-env-override"}
elif _ek is not None and _ekey is None:
_opaque_key = True # a computed key could be BASH_ENV / ENV
if _opaque_key and _is_shell_child:
blocked_in_args = blocked_in_args | {"shell-startup-env:opaque"}
# A git child whose literal env drops the sandbox's GIT_CONFIG_COUNT hook
# suppression (env={} / any dict without it and without a ** splat that
# could carry it) re-enables a planted .git/hooks/* in the unguarded child.
if (
_is_git_child
and not _opaque_key
and not any(
_extract_string_from_node(_k) == "GIT_CONFIG_COUNT"
for _k in _env_node.keys
if _k is not None
)
):
blocked_in_args = blocked_in_args | {"git-config-env-override"}
elif (
isinstance(_env_node, ast.Call)
and isinstance(_env_node.func, ast.Name)
and _env_node.func.id == "dict"
):
for _kw2 in _env_node.keywords:
if _kw2.arg in ("BASH_ENV", "ENV") and (
_extract_string_from_node(_kw2.value) != ""
):
blocked_in_args = blocked_in_args | {
"shell-startup-env:" + _kw2.arg
}
elif (
_kw2.arg == "PATH"
and isinstance(_extract_string_from_node(_kw2.value), str)
and _path_value_is_unsafe(_extract_string_from_node(_kw2.value))
):
blocked_in_args = blocked_in_args | {"unsafe-path-assign"}
elif _kw2.arg is None and _is_shell_child:
blocked_in_args = blocked_in_args | {"shell-startup-env:opaque"}
elif _is_shell_child:
# A non-literal env mapping (env=e, a comprehension) for a shell child
# cannot be proven free of BASH_ENV / ENV, so fail closed.
blocked_in_args = blocked_in_args | {"shell-startup-env:non-literal"}
# os.execl(path, a0, a1, ...) / os.execv(path, [a0, ...]) / os.spawnl(mode,
# path, a0, ...) / os.posix_spawn(path, argv, env) spread the child's argv across
# separate positional args (or a single list), so scanning each string alone
# misses a mutating tail like `sed -i ...`. posix_spawn(p) executes `path` while
# argv[0] is only cosmetic, so a literal-env form (env=() / a byte list) otherwise
# slips the non-literal-env fallback. Reconstruct the executed command line
# (program path + argv[1:], since argv[0] is cosmetic) and run the full scanner.
if (
shell_func.startswith("os.exec")
or shell_func.startswith("os.spawn")
or shell_func.startswith("os.posix_spawn")
):
_name = shell_func.split(".", 1)[1]
if _name.startswith("spawn"): # spawn*(mode, path, ...)
_path_node = node.args[1] if len(node.args) > 1 else None
_tail = node.args[2:]
else: # exec*(path, ...) / posix_spawn(path, argv, env)
_path_node = node.args[0] if node.args else None
_tail = node.args[1:]
_is_v = "execv" in _name or "spawnv" in _name or _name.startswith("posix_spawn")
if _is_v:
_argv = (
[_extract_string_from_node(e) for e in _tail[0].elts]
if _tail and isinstance(_tail[0], (ast.List, ast.Tuple))
else []
)
else:
_argv = [_extract_string_from_node(a) for a in _tail]
_parts = [p for p in ([_extract_string_from_node(_path_node)] + _argv[1:]) if p]
if _parts:
blocked_in_args = blocked_in_args | _find_blocked_commands(
" ".join(shlex.quote(p) for p in _parts)
)
if has_opaque_kwargs:
# Can't inspect dynamic **kwargs; flag as unsafe.
shell_escapes.append(
{
"type": "shell_escape_dynamic",
"line": node.lineno,
"description": (f"{shell_func}() called with dynamic **kwargs"),
}
)
elif blocked_in_args:
shell_escapes.append(
{
"type": "shell_escape",
"line": node.lineno,
"description": (
f"{shell_func}() invokes blocked command(s): "
f"{', '.join(sorted(blocked_in_args))}"
),
}
)
else:
# Only flag dynamic args for funcs that interpret strings as
# shell commands, or when shell= might be on. Any non-literal-
# False shell= is treated as potentially True (conservative).
_STRING_SHELL_FUNCS = frozenset(
{
"os.system",
"os.popen",
"os.popen2",
"os.popen3",
"os.popen4",
"subprocess.getoutput",
"subprocess.getstatusoutput",
}
)
shell_node = expanded_kwargs.get("shell")
shell_safe = shell_node is None or (
isinstance(shell_node, ast.Constant) and shell_node.value is False
)
# Dynamic shell-exec args (chr/format/concat bypasses).
if (
shell_func in _STRING_SHELL_FUNCS
or shell_func in _SHELL_EXEC_FUNCS
or not shell_safe
):
def _is_safe_literal(n):
if _extract_string_from_node(n) is not None:
return True
if isinstance(n, (ast.List, ast.Tuple)):
return all(_extract_string_from_node(e) is not None for e in n.elts)
return False
has_non_literal = any(not _is_safe_literal(a) for a in all_call_args)
if has_non_literal:
shell_escapes.append(
{
"type": "shell_escape_dynamic",
"line": node.lineno,
"description": (
f"{shell_func}() called with non-literal "
f"shell command (potential shell escape)"
),
}
)
# --- Dynamic execution / obfuscation primitives ---
# eval / exec / compile (bare builtin or a single-assignment alias).
exec_func_id = None
if isinstance(func, ast.Name):
if func.id in _DYNAMIC_EXEC_BUILTINS:
exec_func_id = func.id
elif func.id in self.exec_from_aliases:
exec_func_id = self.exec_from_aliases[func.id] # from builtins import exec as e
elif _analyzer_on:
# single-assignment `e = exec` alias, resolved in the call's scope.
exec_func_id = _scope_idx.resolve(func.id, func, "execb")
elif (
isinstance(func, ast.Attribute)
and func.attr in _DYNAMIC_EXEC_BUILTINS
and _ast_name_matches(func.value, self.builtins_aliases)
):
exec_func_id = func.attr # builtins.eval(...) / __builtins__.exec(...)
elif isinstance(func, ast.Attribute) and func.attr == "__call__":
# eval.__call__("...") / exec.__call__(...) / builtins.eval.__call__(...)
# invoke the builtin indirectly through its bound method; the payload is
# still node.args[0], so recover + recurse it exactly like a direct call.
_base = func.value
if isinstance(_base, ast.Name):
if _base.id in _DYNAMIC_EXEC_BUILTINS:
exec_func_id = _base.id
elif _base.id in self.exec_from_aliases:
exec_func_id = self.exec_from_aliases[_base.id]
elif _analyzer_on:
exec_func_id = _scope_idx.resolve(_base.id, _base, "execb")
elif (
isinstance(_base, ast.Attribute)
and _base.attr in _DYNAMIC_EXEC_BUILTINS
and _ast_name_matches(_base.value, self.builtins_aliases)
):
exec_func_id = _base.attr
elif (
_analyzer_on
and isinstance(func, ast.Attribute)
and isinstance(func.value, ast.Name)
):
# class-body alias reached as ClassName.attr (class C: e = eval; C.e('...')),
# or an instance-attribute alias (c.e = exec; c.e('...')).
exec_func_id = _scope_idx.resolve_class_attr(func.value.id, func.attr, "execb")
if exec_func_id is None:
exec_func_id = _scope_idx.resolve_instance_attr(
func.value.id, func.attr, "execb"
)
elif isinstance(func, ast.Subscript):
# ({'e': exec}['e'])(...) / [exec][0](...): an inline container hides the
# sink from the bare-name / attribute checks above.
exec_func_id = self._resolve_container_exec(func)
if exec_func_id is not None:
if _analyzer_on:
# Stage 2: recover + recurse the payload instead of a blanket ban,
# so eval("2+2") passes while obfuscated escapes still block.
_analyze_exec_call(node, exec_func_id)
else:
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": f"dynamic code execution via {exec_func_id}()",
}
)
else:
dynamic_desc = None
# A dangerous sink passed as a first-class VALUE (not called here) runs its
# payloads through a higher-order applier the recursive analyzer never sees:
# list(map(eval, ["..."])), functools.reduce(exec, ...),
# list(map(os.system, ['rm -rf /'])), functools.partial(subprocess.getoutput,
# 'wget ...')(), list(map(pickle.loads, [blob])). Flag any bare reference to a
# dynamic-exec / shell / import / deserializer sink appearing as a call
# argument, unpacking a literal *[...] / *(...) starred arg too.
_cand_args = []
for _a in list(node.args) + [k.value for k in node.keywords]:
if isinstance(_a, ast.Starred) and isinstance(_a.value, (ast.List, ast.Tuple)):
_cand_args.extend(_a.value.elts)
elif isinstance(_a, ast.Starred):
_cand_args.append(_a.value)
else:
_cand_args.append(_a)
_indirect_sink = None
for _t in _cand_args:
_indirect_sink = self._sink_ref_desc(_t)
if _indirect_sink is not None:
break
if _indirect_sink is not None:
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": (
f"{_indirect_sink} passed as a value to a higher-order call "
"(indirect execution of an un-analyzed payload)"
),
}
)
# An attribute-access call whose (receiver, attr-name) pair is the same
# obfuscation as getattr(): the builtin getattr/setattr, or the dunder
# forms object.__getattribute__(obj, 'name') / type.__getattribute__(...)
# / obj.__getattr__('name') that fetch an attribute without matching the
# bare getattr name. Normalized here so the gadget + sensitive-module
# checks below cover all of them.
_attr_call = None
_attr_dunder = False # True when reached via __getattribute__/__getattr__
if (
isinstance(func, ast.Name)
and func.id in ("getattr", "setattr")
and len(node.args) >= 2
):
_attr_call = (node.args[0], node.args[1])
elif isinstance(func, ast.Attribute) and func.attr in (
"__getattribute__",
"__getattr__",
):
# Unbound form object.__getattribute__(obj, 'name') carries the receiver
# as arg0; the BOUND form obj.__getattribute__('name') carries it as the
# attribute's own value (builtins.open.__getattribute__('__closure__')).
_attr_dunder = True
if len(node.args) >= 2:
_attr_call = (node.args[0], node.args[1])
elif len(node.args) == 1:
_attr_call = (func.value, node.args[0])
elif self._attrgetter_name(func) is not None and len(node.args) == 1:
# operator.attrgetter('name')(obj) evaluates to obj.name -- the same
# attribute-fetch obfuscation as getattr(obj, 'name'). Detect the
# attrgetter APPLICATION call itself (node.func is the attrgetter,
# node.args[0] is the object) so it is caught whether or not the result
# is immediately invoked: attrgetter('__closure__')(open)[0] and the
# chained attrgetter('system')(os)('rm -rf /') both normalize here.
_attr_call = (node.args[0], ast.Constant(value = self._attrgetter_name(func)))
elif self._methodcaller_getattr_name(func) is not None and len(node.args) == 1:
# operator.methodcaller('__getattribute__', 'name')(obj) fetches obj.name,
# the same attribute obfuscation as attrgetter/getattr.
_attr_call = (
node.args[0],
ast.Constant(value = self._methodcaller_getattr_name(func)),
)
is_dynamic_import = (
_ast_name_matches(_ecf, _DYNAMIC_IMPORT_FUNCS)
or (
isinstance(_ecf, ast.Name)
and (
_ecf.id in ("__import__", "import_module")
or _ecf.id in self.import_func_aliases
)
)
or (
isinstance(_ecf, ast.Attribute)
and _ecf.attr in ("import_module", "reload", "__import__")
and _ast_name_matches(_ecf.value, self.importlib_aliases)
)
or (
# builtins.__import__('os') / __builtins__.__import__(...), incl. a
# single-assignment alias (b = __builtins__; b.__import__('os')).
isinstance(_ecf, ast.Attribute)
and _ecf.attr == "__import__"
and self._is_builtins_ref(_ecf.value)
)
or (
# single-assignment `im = importlib.import_module` in scope.
_analyzer_on
and isinstance(_ecf, ast.Name)
and bool(_scope_idx.resolve(_ecf.id, _ecf, "impf"))
)
)
# Deserialization sinks reconstruct arbitrary objects/code from bytes.
# Resolve aliased imports (from pickle import loads as l), module aliases
# (import pickle as p; p.loads) and the file-based *.load variants -- not
# just the exact pickle.loads name. Uses the __call__-stripped effective
# callee so pickle.loads.__call__(blob) resolves like pickle.loads(blob).
_deser_fq = None
if isinstance(_ecf, ast.Attribute) and isinstance(_ecf.value, ast.Name):
_canon = self.deserialize_module_aliases.get(_ecf.value.id)
if _canon is not None:
_cand = f"{_canon}.{_ecf.attr}"
if _cand in _CODE_DESERIALIZE_SINKS:
_deser_fq = _cand
if _deser_fq is None and _analyzer_on:
# class-body alias reached as ClassName.attr (class C: l = pickle.loads).
_deser_fq = _scope_idx.resolve_class_attr(_ecf.value.id, _ecf.attr, "deser")
elif isinstance(_ecf, ast.Name):
_deser_fq = self.deserialize_aliases.get(_ecf.id)
if _deser_fq is None and _analyzer_on:
# single-assignment `l = pickle.loads` in the call's scope.
_deser_fq = _scope_idx.resolve(_ecf.id, _ecf, "deser")
elif _analyzer_on and isinstance(_ecf, ast.Subscript):
# ([pickle.loads][0])(payload) / {'k': pickle.loads}['k'](payload):
# an inline container hides the sink from the attribute / name checks.
_deser_fq = self._resolve_container_deser(_ecf)
if _deser_fq is None:
_fq_func = _fq_attr_name(_ecf)
if _fq_func in _CODE_DESERIALIZE_SINKS:
_deser_fq = _fq_func
if _deser_fq is None and isinstance(_ecf, ast.Attribute):
# yaml.load(...) / yaml.load_all(...) reconstruct arbitrary objects unless
# handed a safe loader. Resolve the (possibly module-aliased) yaml receiver
# and only flag when no explicit SafeLoader is passed, so yaml.load(data,
# Loader=yaml.SafeLoader) and yaml.safe_load(data) stay allowed.
if _ecf.attr in _YAML_LOAD_METHODS and isinstance(_ecf.value, ast.Name):
if self.deserialize_module_aliases.get(_ecf.value.id) == "yaml":
if not _yaml_call_has_safe_loader(node):
_deser_fq = "yaml." + _ecf.attr
# pickle.Unpickler(f).load() / dill.Unpickler(f).load(): the reduce payload
# runs on .load(); the sink-name check misses it because the callee is a
# method on an Unpickler instance, not a *.load module function.
if (
_deser_fq is None
and _ecf.attr in ("load", "load_all")
and self._is_unpickler_ctor(_ecf.value)
):
_deser_fq = "pickle.Unpickler.load"
# torch.load(f, weights_only=False) / np.load(f, allow_pickle=True) /
# joblib.load(f) run a pickle reduce payload; flag only the unsafe forms so
# the safe defaults (torch.load(f), np.load(f)) stay allowed.
if (
_deser_fq is None
and _ecf.attr == "load"
and isinstance(_ecf.value, ast.Name)
):
_pcanon = self.pickle_loader_module_aliases.get(_ecf.value.id)
if _pcanon is not None:
_plfq = f"{_pcanon}.load"
if _pickle_loader_is_unsafe(_plfq, node):
_deser_fq = _plfq
if _deser_fq is None and isinstance(_ecf, ast.Name):
# from yaml import load; load(data): apply the same safe-loader check to the
# bare-name alias so importing the function directly is not a bypass.
_ym = self.yaml_load_aliases.get(_ecf.id)
if _ym is not None and not _yaml_call_has_safe_loader(node):
_deser_fq = "yaml." + _ym
# from joblib import load; load(f): the bare-name conditional pickle loader.
if _deser_fq is None:
_plf = self.pickle_loader_func_aliases.get(_ecf.id)
if _plf is not None and _pickle_loader_is_unsafe(_plf, node):
_deser_fq = _plf
if _analyzer_on and _deser_fq is not None:
dynamic_desc = f"{_deser_fq}() deserializes an unverifiable code payload"
elif is_dynamic_import:
# Computed module name (obfuscation) or a dangerous target is unsafe; a
# benign literal import (huggingface_hub, json, ...) passes. With the
# analyzer on, the name is constant-folded first so `__import__(
# "hugging"+"face_hub")` resolves to a real module instead of blocking.
if node.args:
if _analyzer_on:
folded = _const_fold(node.args[0], _const_env)
mod = folded if isinstance(folded, str) else None
else:
mod = _extract_string_from_node(node.args[0])
else:
mod = None
_mod_top = mod.split(".")[0] if mod else None
if (
mod is None
or _mod_top in _DANGEROUS_IMPORT_NAMES
or _mod_top in _DESERIALIZE_MODULES
):
# Deserializer modules (pickle/marshal/...) are dangerous import
# targets too: __import__('pickle').loads(blob) executes a reduce
# payload even though a plain `import pickle` is benign.
dynamic_desc = "dynamic import of a computed or sensitive module name"
elif _attr_call is not None and (
(
isinstance(_const_fold(_attr_call[1], _const_env), str)
and _const_fold(_attr_call[1], _const_env) in _GADGET_DUNDERS
)
or (
# A __getattribute__/__getattr__ dunder call whose attribute name is
# not constant-foldable hides a gadget dunder behind a runtime
# expression (open.__getattribute__(''.join(map(chr, ...)))), which
# recovers a guarded wrapper's __closure__/cell_contents. No program
# legitimately spells __getattribute__ with a computed name, so fail
# closed on the dynamic form.
_attr_dunder and not isinstance(_const_fold(_attr_call[1], _const_env), str)
)
):
# getattr(anything, '__globals__' / '__subclasses__' / ...) or the
# object.__getattribute__ equivalent reaches an introspection gadget
# with no ast.Attribute for visit_Attribute to catch. Direct
# x.__globals__ is already flagged for ANY receiver, so flag the
# dynamic-attr-name form regardless of receiver too. (Also closes the
# __closure__ recovery of a guarded wrapper's original callable.)
_gv = _const_fold(_attr_call[1], _const_env)
if isinstance(_gv, str):
dynamic_desc = (
f"dynamic attribute access of an introspection gadget dunder ({_gv})"
)
else:
dynamic_desc = (
"computed attribute name via __getattribute__/__getattr__ "
"(obfuscated introspection gadget)"
)
elif (
isinstance(func, ast.Name)
and func.id == "vars"
and node.args
and _ast_name_matches(node.args[0], self._attr_obfuscation_targets())
):
# vars(os) / vars(__builtins__) returns the module __dict__, the same
# obfuscation as os.__dict__['system'] but without the attribute access.
dynamic_desc = "vars() on a sensitive module (dict obfuscation)"
elif _attr_call is not None and _ast_name_matches(
_attr_call[0], self._attr_obfuscation_targets()
):
# Stage 2 refinement: a benign constant attr (getattr(os, "getpid"))
# is allowed; only a dynamic attr or a dangerous constant attr blocks.
# Covers getattr/setattr and object.__getattribute__(builtins, 'eval').
if _analyzer_on:
attr_val = _const_fold(_attr_call[1], _const_env)
if isinstance(attr_val, str):
if attr_val in _DANGEROUS_ATTR_NAMES or attr_val == "__dict__":
# getattr(__builtins__, '__dict__')['__import__'] exposes the
# module namespace the same way vars()/direct .__dict__ do, so
# a constant '__dict__' on a sensitive module is dangerous too.
dynamic_desc = (
"dynamic attribute access on a sensitive module "
"(attribute-name obfuscation)"
)
else:
dynamic_desc = (
"dynamic attribute access on a sensitive module "
"(attribute-name obfuscation)"
)
else:
dynamic_desc = (
"dynamic attribute access on a sensitive module "
"(attribute-name obfuscation)"
)
elif (
# sys.modules.get('os') -- the .get() twin of sys.modules['os'], incl. a
# single-assignment alias (m = sys.modules; m.get('os')).
isinstance(func, ast.Attribute)
and func.attr == "get"
and self._is_sys_modules(func.value)
and node.args
):
# Constant-fold the key so sys.modules.get('o' + 's') is caught, not
# just a bare literal (the module is already loaded by the prelude).
_key = _const_fold(node.args[0], _const_env)
if isinstance(_key, str) and _key.split(".")[0] in _DANGEROUS_IMPORT_NAMES:
dynamic_desc = "sys.modules.get(...) access to a sensitive module"
elif (
# sys.modules.pop('_io', None) / .clear() / .update(...) / .setdefault(...)
# mutate the loader table just like `del sys.modules[...]`: dropping a
# guarded module entry lets `import _io` / `import posix` reload a fresh,
# UNWRAPPED C module (the prelude patched only the old object), bypassing
# filesystem confinement. The subscript-Store/Del check misses method calls.
isinstance(func, ast.Attribute)
and func.attr
in (
"pop",
"popitem",
"clear",
"setdefault",
"update",
"__setitem__",
"__delitem__",
)
and self._is_sys_modules(func.value)
):
dynamic_desc = (
f"sys.modules.{func.attr}(...) mutates the loader table "
"(can drop a guarded module for reimport)"
)
elif (
# The same loader-table mutation through an UNBOUND dict method:
# dict.pop(sys.modules, '_io') / type(sys.modules).__delitem__(sys.modules,
# ...). The receiver is `dict` / `type(sys.modules)`, not `sys.modules`
# itself, so the check above misses it; here sys.modules is the first arg.
isinstance(func, ast.Attribute)
and func.attr
in (
"pop",
"popitem",
"clear",
"setdefault",
"update",
"__setitem__",
"__delitem__",
)
and node.args
and self._is_sys_modules(node.args[0])
and (
(isinstance(func.value, ast.Name) and func.value.id == "dict")
or (
isinstance(func.value, ast.Call)
and isinstance(func.value.func, ast.Name)
and func.value.func.id == "type"
)
# sys.modules.__class__.pop(sys.modules, ...): the receiver is the dict
# TYPE reached via .__class__, not the bare `dict` name / type(...) call.
or (
isinstance(func.value, ast.Attribute)
and func.value.attr == "__class__"
and self._is_sys_modules(func.value.value)
)
)
):
dynamic_desc = (
f"unbound dict.{func.attr}(sys.modules, ...) mutates the loader table "
"(can drop a guarded module for reimport)"
)
elif (
# sys.meta_path.pop(0) / .clear() / .remove(...) / .insert(...) / .append(...)
# / .extend(...) / .reverse() / .sort() removes or reorders the import finder
# chain, dropping the sandbox's workdir-module vetter so a planted workdir
# helper (import evil) loads without source review and runs an unguarded sink.
# No sandboxed compute legitimately mutates the import finder chain.
isinstance(func, ast.Attribute)
and func.attr
in (
"pop",
"clear",
"remove",
"insert",
"append",
"extend",
"reverse",
"sort",
"__setitem__",
"__delitem__",
"__iadd__",
)
and self._is_sys_meta_path(func.value)
):
dynamic_desc = (
f"sys.meta_path.{func.attr}(...) mutates the import finder chain "
"(can remove the sandbox workdir-module vetter)"
)
elif (
# The same mutation via an UNBOUND list method: list.pop(sys.meta_path, 0) /
# list.insert(sys.meta_path, ...). The receiver is `list`, not sys.meta_path,
# so the bound-method check above misses it; here sys.meta_path is the first arg.
isinstance(func, ast.Attribute)
and func.attr
in (
"pop",
"clear",
"remove",
"insert",
"append",
"extend",
"reverse",
"sort",
"__setitem__",
"__delitem__",
"__iadd__",
)
and node.args
and self._is_sys_meta_path(node.args[0])
and (
(isinstance(func.value, ast.Name) and func.value.id == "list")
or (
isinstance(func.value, ast.Call)
and isinstance(func.value.func, ast.Name)
and func.value.func.id == "type"
)
# sys.meta_path.__class__.pop(sys.meta_path, 0): the receiver is the list
# TYPE reached via .__class__, not the bare `list` name / type(...) call.
or (
isinstance(func.value, ast.Attribute)
and func.value.attr == "__class__"
and self._is_sys_meta_path(func.value.value)
)
)
):
dynamic_desc = (
f"unbound list.{func.attr}(sys.meta_path, ...) mutates the import finder "
"chain (can remove the sandbox workdir-module vetter)"
)
elif (
# globals().get('__builtins__') / locals().get(...) / vars().get(...)
# -- the .get() twin of the globals()['__builtins__'] subscript form.
isinstance(func, ast.Attribute)
and func.attr == "get"
and isinstance(func.value, ast.Call)
and isinstance(func.value.func, ast.Name)
and func.value.func.id in ("globals", "locals", "vars")
and not func.value.args
and node.args
):
_key = _const_fold(node.args[0], _const_env)
if isinstance(_key, str) and (
_key in ("__builtins__", "__builtin__")
or _key.split(".")[0] in _DANGEROUS_IMPORT_NAMES
):
dynamic_desc = (
"namespace-dict .get() access to builtins / a sensitive module"
)
elif (
# dict.__getitem__(globals(), '__builtins__') / dict.get(locals(), ...): the
# unbound dict-method twin of globals()['__builtins__'], pulling the builtins
# namespace (or a dangerous module) out of the namespace dict without a
# subscript node. The receiver is `dict`, not the namespace dict itself, so
# the subscript scan misses it; here the namespace dict is the first argument.
isinstance(func, ast.Attribute)
and func.attr in ("__getitem__", "get")
and isinstance(func.value, ast.Name)
and func.value.id == "dict"
and len(node.args) >= 2
and isinstance(node.args[0], ast.Call)
and isinstance(node.args[0].func, ast.Name)
and node.args[0].func.id in ("globals", "locals", "vars")
and not node.args[0].args
):
_key = _const_fold(node.args[1], _const_env)
if isinstance(_key, str) and (
_key in ("__builtins__", "__builtin__")
or _key.split(".")[0] in _DANGEROUS_IMPORT_NAMES
):
dynamic_desc = (
"unbound dict access to builtins / a sensitive module namespace dict"
)
elif (
(
# types.FunctionType(compile(src, ...), {})() runs a code object WITHOUT
# eval/exec, so a dynamic compile() payload reaches execution here even
# though compile() alone is allowed. Flag when a FunctionType call takes
# a compile()-derived code object as its first argument.
(
isinstance(func, ast.Attribute)
and func.attr == "FunctionType"
and _ast_name_matches(func.value, self.types_aliases)
)
or (isinstance(func, ast.Name) and func.id in self.functiontype_aliases)
or (
# The same constructor is reachable as type(lambda: None): the
# type of any function IS types.FunctionType, so
# type(lambda: None)(code, {})() executes a code object too.
isinstance(func, ast.Call)
and not func.keywords
and len(func.args) == 1
and isinstance(func.args[0], ast.Lambda)
and (
(isinstance(func.func, ast.Name) and func.func.id == "type")
or (
isinstance(func.func, ast.Attribute)
and func.func.attr == "type"
and _ast_name_matches(func.func.value, self.builtins_aliases)
)
)
)
)
and node.args
and not self._functiontype_arg_is_vetted(node.args[0])
):
dynamic_desc = (
"types.FunctionType() executes an unvetted code object "
"(bypasses the eval/exec gate)"
)
elif (
# runpy.run_path('evil.py') / runpy.run_module('evil') execute a
# file/module in the guarded interpreter WITHOUT the recursive source
# analysis exec/eval receive, so a sandboxed snippet can write a local
# evil.py and run it. Treat these as direct execution sinks. Covers the
# attribute form, a `from runpy import run_path` bare-name alias, and a
# single-assignment alias (r = runpy.run_path; r('evil.py')).
(
isinstance(func, ast.Attribute)
and func.attr in ("run_path", "run_module")
and _ast_name_matches(func.value, self.runpy_aliases)
)
or (isinstance(func, ast.Name) and func.id in self.runpy_func_aliases)
or self._rhs_module_attr(func, ("run_path", "run_module"), self.runpy_aliases)
):
if isinstance(func, ast.Attribute):
_rn = func.attr
elif func.id in self.runpy_func_aliases:
_rn = func.id
else:
_rn = self._rhs_module_attr(
func, ("run_path", "run_module"), self.runpy_aliases
)
dynamic_desc = f"runpy.{_rn}() executes a file/module without static analysis"
elif (
# pty.spawn([...]) / pty.fork() run an unguarded child process (typically a
# shell) outside the sandbox, the same escape as subprocess / os.system.
# Covers the attribute form, a `from pty import spawn` bare-name alias, and a
# single-assignment alias (s = pty.spawn; s([...])).
(
isinstance(func, ast.Attribute)
and func.attr in ("spawn", "fork")
and _ast_name_matches(func.value, self.pty_aliases)
)
or (isinstance(func, ast.Name) and func.id in self.pty_func_aliases)
or self._rhs_module_attr(func, ("spawn", "fork"), self.pty_aliases)
):
if isinstance(func, ast.Attribute):
_pn = func.attr
elif func.id in self.pty_func_aliases:
_pn = func.id
else:
_pn = self._rhs_module_attr(func, ("spawn", "fork"), self.pty_aliases)
dynamic_desc = f"pty.{_pn}() spawns an unguarded child process"
elif (
# inspect.getclosurevars(open).nonlocals['real'] recovers the original
# unguarded callable a guard wrapper closes over, without spelling
# __closure__ / cell_contents. Block the introspection primitive
# (attribute form plus a `from inspect import getclosurevars` alias).
(
isinstance(func, ast.Attribute)
and func.attr == "getclosurevars"
and _ast_name_matches(func.value, self.inspect_aliases)
)
or (isinstance(func, ast.Name) and func.id in self.getclosurevars_aliases)
):
dynamic_desc = "inspect.getclosurevars() recovers a guarded wrapper's closure"
elif (
# gc.get_referents / get_referrers / get_objects walk the object graph to a
# guard wrapper's closure cell (the original unguarded open/os.* callable)
# without spelling __closure__ / cell_contents, so a recovered original can
# then write/read outside the workdir. Block the graph-traversal APIs.
(
isinstance(func, ast.Attribute)
and func.attr in ("get_referents", "get_referrers", "get_objects")
and _ast_name_matches(func.value, self.gc_aliases)
)
or (isinstance(func, ast.Name) and func.id in self.gc_walk_aliases)
):
_gn = func.attr if isinstance(func, ast.Attribute) else func.id
dynamic_desc = (
f"gc.{_gn}() walks the object graph to a guarded wrapper's closure"
)
elif (
# cls.mro().__getitem__(1) / .pop(1) / cls.__mro__.__getitem__(1): the
# method-call twin of the subscripted-mro base extraction
# (visit_Subscript). Same gadget shape (io.FileIO.mro().pop(1) recovers
# the original FileIO base), so flag an element-extraction method on an
# mro()/__mro__ receiver.
isinstance(func, ast.Attribute)
and func.attr in ("__getitem__", "pop")
and (
(
isinstance(func.value, ast.Call)
and isinstance(func.value.func, ast.Attribute)
and func.value.func.attr == "mro"
and not func.value.args
)
or (isinstance(func.value, ast.Attribute) and func.value.attr == "__mro__")
)
and (
# pop() / pop(i) always extract an element; __getitem__ only when the
# index is a plain integer (not a slice object).
func.attr == "pop"
or (
len(node.args) == 1
and isinstance(_const_fold(node.args[0], _const_env), int)
)
)
):
dynamic_desc = f"mro().{func.attr}(...) extracts a base class (gadget)"
elif isinstance(func, ast.Attribute) and func.attr in (
"runcode",
"runsource",
"load_module",
"exec_module",
):
# code.InteractiveInterpreter().runcode(c) / InteractiveConsole()
# .runsource(src) execute a code object / source string; an importlib file
# loader (SourceFileLoader(...).load_module() / spec.loader.exec_module(m))
# executes a local file. None run through the recursive analysis exec/eval
# receive, so an opaque payload (a written evil.py, a compile() result, or
# raw source) runs un-analyzed. These method names are unique to those
# interpreters / loaders, so flag the call regardless of receiver.
dynamic_desc = (
f"{func.attr}() executes code / a file without static analysis "
"(code interpreter / importlib file loader)"
)
if dynamic_desc:
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": dynamic_desc,
}
)
self.generic_visit(node)
def visit_Attribute(self, node):
# Introspection gadget dunders (``__subclasses__``, ``__globals__``, ...) are the
# standard way to walk from a benign object to os/builtins, bypassing the name-based
# checks. Flag the attribute access itself, then keep descending.
if node.attr in _GADGET_DUNDERS:
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": f"introspection gadget attribute {node.attr}",
}
)
elif node.attr == "__dict__" and _ast_name_matches(
node.value, self._attr_obfuscation_targets()
):
# os.__dict__['system']('id') reaches the sink with no getattr call for
# the name-based checks to see. __dict__ on ordinary objects stays allowed.
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": "__dict__ access on a sensitive module",
}
)
elif node.attr in ("__mro__", "mro") and self._is_fileclass_recovery_expr(node.value):
# io.FileIO.mro() / io.FileIO.__mro__ / open.__class__.mro(): the guard
# replaces io.FileIO with a confining subclass, but its MRO still exposes the
# UNGUARDED C base. Plain iteration recovers it (for c in io.FileIO.mro(): c(
# '/etc/x','w')) without ever indexing, so flag any whole-MRO access on a
# file-class-recovery receiver. Benign int.mro() / cls.__mro__ do not match.
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": "MRO access on a file class recovers an unguarded base (gadget)",
}
)
elif (
node.attr == "meta_path"
and isinstance(node.ctx, (ast.Store, ast.Del))
and _ast_name_matches(node.value, self.sys_aliases)
):
# Reassigning / deleting sys.meta_path (sys.meta_path = []) replaces the whole
# import finder chain, dropping the sandbox's workdir-module vetter. Reading it
# (Load) stays allowed; only Store / Del is a mutation.
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": (
"sys.meta_path reassignment can remove the sandbox workdir-module vetter"
),
}
)
self.generic_visit(node)
def _is_fileclass_recovery_expr(self, expr):
"""True when ``expr`` denotes a file/IO class whose MRO walk recovers an UNGUARDED
file primitive: the guarded ``io.FileIO`` / ``_io.FileIO`` (``.FileIO`` attribute)
or the type of a file object reached through ``.__class__`` (``open.__class__``,
``f.__class__``). Ordinary class receivers (``int``, ``cls``, ``type('X', (), {})``)
are plain Names / Calls and do not match, so benign MRO introspection stays allowed."""
return isinstance(expr, ast.Attribute) and expr.attr in ("FileIO", "__class__")
def _is_unbound_mro_gadget(self, node):
"""True when ``node`` is an UNBOUND MRO / getattribute call that recovers a file
class's MRO without spelling ``.mro`` / ``.__mro__`` on the receiver:
``type.mro(io.FileIO)``, ``type.__getattribute__(io.FileIO, '__mro__')``,
``object.__getattribute__(io.FileIO, 'mro')`` or ``getattr(io.FileIO, '__mro__')``.
Iterating the result exposes the unguarded ``_io.FileIO`` C base, so treat it as the
same recovery gadget as ``io.FileIO.__mro__``."""
f = node.func
# getattr(<fileclass>, 'mro' | '__mro__')
if (
isinstance(f, ast.Name)
and f.id == "getattr"
and len(node.args) >= 2
and self._is_fileclass_recovery_expr(node.args[0])
and _const_fold(node.args[1], _const_env) in ("mro", "__mro__")
):
return True
if not isinstance(f, ast.Attribute):
return False
# type.mro(<fileclass>)
if (
f.attr == "mro"
and isinstance(f.value, ast.Name)
and f.value.id == "type"
and node.args
and self._is_fileclass_recovery_expr(node.args[0])
):
return True
# type.__getattribute__(<fileclass>, 'mro' | '__mro__') / object.__getattribute__(...)
if (
f.attr in ("__getattribute__", "__getattr__")
and isinstance(f.value, ast.Name)
and f.value.id in ("type", "object")
and len(node.args) >= 2
and self._is_fileclass_recovery_expr(node.args[0])
and _const_fold(node.args[1], _const_env) in ("mro", "__mro__")
):
return True
return False
def visit_Subscript(self, node):
# An INTEGER-indexed __mro__ (cls.__mro__[1]) or the equivalent method call
# (cls.mro()[1]) extracts a specific base class the way __bases__[0] does -- the
# shape used to reach the original FileIO C base class (io.FileIO.mro()[1]) or
# walk to object/subclasses. Plain iteration (for c in cls.__mro__ / cls.mro())
# and slicing (cls.__mro__[1:]) yield the whole tuple/list for legitimate
# introspection, so only a non-slice index is flagged.
if (
isinstance(node.ctx, ast.Load)
and not isinstance(node.slice, ast.Slice)
and (
(isinstance(node.value, ast.Attribute) and node.value.attr == "__mro__")
or (
isinstance(node.value, ast.Call)
and isinstance(node.value.func, ast.Attribute)
and node.value.func.attr == "mro"
and not node.value.args
)
)
):
_mro_shape = "__mro__" if isinstance(node.value, ast.Attribute) else "mro()"
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": f"subscripted {_mro_shape} extracts a base class (gadget)",
}
)
# type(open).__dict__['__closure__'].__get__(open) / type(cell).__dict__[
# 'cell_contents'].__get__(cell): fetch a gadget descriptor from a type's __dict__
# BY NAME (a subscript, not an attribute node), then invoke __get__ to recover the
# guarded wrapper's original callable. Flag a __dict__ subscript keyed by a gadget
# dunder so the attribute-node gadget scan cannot be side-stepped this way. The same
# mapping is reachable through vars(type(obj))[...], so cover that form too.
_dunder_dict = (
isinstance(node.value, ast.Attribute) and node.value.attr == "__dict__"
) or (
isinstance(node.value, ast.Call)
and isinstance(node.value.func, ast.Name)
and node.value.func.id == "vars"
and node.value.args
)
if isinstance(node.ctx, ast.Load) and _dunder_dict:
_dk = _const_fold(node.slice, _const_env)
if isinstance(_dk, str) and _dk in _GADGET_DUNDERS:
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": f"__dict__[{_dk!r}] descriptor lookup (gadget)",
}
)
# sys.modules['os'] pulls an already-loaded dangerous module out of the
# loader table (os/subprocess are loaded by the host). Scope to a Load of a
# dangerous LITERAL key so legit uses ("x" in sys.modules, sys.modules.get(
# name), sys.modules[name] = ...) stay allowed.
v = node.value
# sys.modules[...] (attribute form), getattr(sys, 'modules')[...], or
# object.__getattribute__(sys, 'modules')[...] all index the loader table -- as
# does a single-assignment alias (m = sys.modules; m['os']), so use the alias-aware
# helper the mutation checks already use.
is_sys_modules = self._is_sys_modules(v)
if isinstance(node.ctx, ast.Load) and is_sys_modules:
# Constant-fold the key so sys.modules['o' + 's'] is caught, not just a
# bare literal; a truly dynamic key (sys.modules[name]) stays allowed.
key = _const_fold(node.slice, _const_env)
if isinstance(key, str) and key.split(".")[0] in _DANGEROUS_IMPORT_NAMES:
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": "sys.modules[...] access to a sensitive module",
}
)
if isinstance(node.ctx, (ast.Store, ast.Del)) and is_sys_modules:
# `del sys.modules['posix']; import posix` (or reassigning the entry) drops
# the guard-patched module object so a fresh, UNWRAPPED C module is imported,
# bypassing the Stage 5 wrappers. Mutating the loader table has no legitimate
# use in sandboxed compute, so deny any Store/Del on sys.modules[...].
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": "sys.modules mutation (del / assign) can drop a guarded module",
}
)
if isinstance(node.ctx, (ast.Store, ast.Del)) and self._is_sys_meta_path(v):
# `sys.meta_path[:] = []` / `del sys.meta_path[0]` / `sys.meta_path[0] = x`
# removes or reorders the import finder chain, dropping the sandbox's
# workdir-module vetter so a planted helper imports without source review.
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": (
"sys.meta_path mutation (del / assign) can remove the sandbox "
"workdir-module vetter"
),
}
)
# globals()['__builtins__'] / locals()[...] / vars()[...] pulls the builtins
# namespace (or a dangerous module) out of the namespace dict, e.g.
# getattr(globals()['__builtins__'], '__import__')('os'). Flag a Load of a
# dangerous literal key off a bare globals()/locals()/vars() call.
if isinstance(node.ctx, ast.Load) and self._is_namespace_dict_expr(v):
key = _const_fold(node.slice, _const_env)
if isinstance(key, str) and (
key in ("__builtins__", "__builtin__")
or key.split(".")[0] in _DANGEROUS_IMPORT_NAMES
):
dynamic_exec.append(
{
"type": "dynamic_exec",
"line": getattr(node, "lineno", -1),
"description": "namespace-dict access to builtins / a sensitive module",
}
)
self.generic_visit(node)
def visit_ExceptHandler(self, node):
if self.loop_depth == 0:
self.generic_visit(node)
return
if node.type is None:
exception_catching.append(
{
"type": "bare_except_in_loop",
"line": node.lineno,
"description": "Bare except in loop catches TimeoutError and continues looping",
}
)
elif isinstance(node.type, ast.Name):
# Flag BaseException/TimeoutError but NOT Exception: `except
# Exception` can't catch SystemExit/KeyboardInterrupt, so it
# can't suppress timeout enforcement.
if node.type.id in ("TimeoutError", "BaseException"):
exception_catching.append(
{
"type": f"catches_{node.type.id}_in_loop",
"line": node.lineno,
"description": f"Catches {node.type.id} in loop - may suppress timeout and continue",
}
)
elif isinstance(node.type, ast.Tuple):
for elt in node.type.elts:
if isinstance(elt, ast.Name):
if elt.id in ("TimeoutError", "BaseException"):
exception_catching.append(
{
"type": f"catches_{elt.id}_in_loop",
"line": node.lineno,
"description": f"Catches {elt.id} in loop - may suppress timeout and continue",
}
)
self.generic_visit(node)
visitor = SignalEscapeVisitor()
visitor.visit(tree)
if visitor.imports_signal and not signal_tampering:
warnings.append("Code imports 'signal' module - review manually for safety")
# Static host policy: block metadata hosts and any literal host outside the
# trusted allowlist; uploads blocked regardless of host. Dynamic hosts are
# caught by the bash blocklist.
network_calls: list[dict] = []
sensitive_file_reads: list[dict] = []
_NETWORK_FQ_PREFIXES = (
"socket.socket",
"socket.create_connection",
"socket.getaddrinfo",
"urllib.request.urlopen",
"urllib.request.urlretrieve",
"urllib3.",
"requests.get",
"requests.post",
"requests.put",
"requests.delete",
"requests.patch",
"requests.head",
"requests.request",
"requests.Session",
"http.client.HTTPConnection",
"http.client.HTTPSConnection",
"httpx.get",
"httpx.post",
"httpx.put",
"httpx.patch",
"httpx.delete",
"httpx.request",
"httpx.Client",
"httpx.AsyncClient",
"aiohttp.ClientSession",
)
_UPLOAD_HTTP_METHODS = (
"requests.post",
"requests.put",
"requests.patch",
"requests.delete",
"requests.request",
"httpx.post",
"httpx.put",
"httpx.patch",
"httpx.delete",
"httpx.request",
"urllib.request.urlopen",
"urllib.request.Request",
)
_UPLOAD_HF_FQ = (
"huggingface_hub.upload_file",
"huggingface_hub.upload_folder",
"huggingface_hub.upload_large_folder",
"huggingface_hub.create_commit",
)
_UPLOAD_HF_METHODS = frozenset(
{
"upload_file",
"upload_folder",
"upload_large_folder",
"create_commit",
}
)
# Cloud-metadata / link-local hosts.
_METADATA_HOST_LITERALS = {
"169.254.169.254",
"fd00:ec2::254",
"metadata.google.internal",
"metadata",
"metadata.tencentyun.com",
"100.100.100.200",
"100.100.100.110",
"169.254.170.2",
"169.254.170.23",
}
_METADATA_HOST_PREFIXES = (
"169.254.",
"100.64.",
)
# Allowlist kept explicit so each entry is auditable.
_TRUSTED_PUBLIC_HOST_LITERALS = frozenset(
{
# search
"www.google.com",
"google.com",
"www.bing.com",
"bing.com",
"duckduckgo.com",
"html.duckduckgo.com",
# encyclopedic / reference
"wikipedia.org",
"www.wikipedia.org",
"wikimedia.org",
"www.wikimedia.org",
"wikidata.org",
"www.wikidata.org",
"commons.wikimedia.org",
"www.britannica.com",
"openlibrary.org",
"www.openstreetmap.org",
# ML / dev / data
"huggingface.co",
"hf.co",
"github.com",
"api.github.com",
"raw.githubusercontent.com",
"gist.github.com",
"docs.github.com",
"pypi.org",
"files.pythonhosted.org",
"www.npmjs.com",
"registry.npmjs.org",
"crates.io",
"static.crates.io",
# docs
"docs.python.org",
"python.org",
"www.python.org",
"developer.mozilla.org",
"developer.apple.com",
"learn.microsoft.com",
"docs.docker.com",
"pytorch.org",
"docs.pytorch.org",
"tensorflow.org",
"www.tensorflow.org",
"numpy.org",
"pandas.pydata.org",
"scipy.org",
"scikit-learn.org",
"matplotlib.org",
"fastapi.tiangolo.com",
"starlette.io",
# academic
"arxiv.org",
"export.arxiv.org",
"scholar.google.com",
"openreview.net",
"semanticscholar.org",
"www.semanticscholar.org",
"biorxiv.org",
"www.biorxiv.org",
"medrxiv.org",
"www.medrxiv.org",
"pubmed.ncbi.nlm.nih.gov",
"www.ncbi.nlm.nih.gov",
# Q&A / community
"stackoverflow.com",
"stackexchange.com",
"askubuntu.com",
"superuser.com",
"serverfault.com",
# standards
"www.w3.org",
"tools.ietf.org",
"datatracker.ietf.org",
"www.rfc-editor.org",
# reputable news
"www.bbc.com",
"www.bbc.co.uk",
"www.reuters.com",
"apnews.com",
"www.nature.com",
"www.science.org",
# government / open data
"data.gov",
"catalog.data.gov",
"www.census.gov",
"www.nasa.gov",
"data.nasa.gov",
"www.cdc.gov",
"www.nih.gov",
"www.who.int",
# weather / time
"api.weather.gov",
"worldtimeapi.org",
}
)
_TRUSTED_PUBLIC_HOST_SUFFIXES = (
".wikipedia.org",
".wikimedia.org",
".wiktionary.org",
".wikibooks.org",
".wikiquote.org",
".wikisource.org",
".wikiversity.org",
".wikivoyage.org",
".stackexchange.com",
".hf.co",
".huggingface.co",
".githubusercontent.com",
".github.io",
".arxiv.org",
".readthedocs.io",
".readthedocs.org",
)
_SENSITIVE_FILE_PREFIXES = (
"/etc/passwd",
"/etc/shadow",
"/etc/sudoers",
"/etc/ssh/",
)
_SENSITIVE_FILE_RE = re.compile(r"^/proc/(?:self|\d+)/(?:environ|cmdline|task/\d+/environ)$")
def _normalize_host(host: str) -> str:
if not host:
return ""
h = host.strip().lower().rstrip(".")
if "@" in h:
h = h.split("@", 1)[1]
if h.startswith("[") and "]" in h:
h = h[1 : h.index("]")]
elif h.count(":") == 1:
h = h.split(":", 1)[0]
return h
def _is_metadata_host(host: str) -> bool:
h = _normalize_host(host)
if not h:
return False
if h in _METADATA_HOST_LITERALS:
return True
if any(h.startswith(p) for p in _METADATA_HOST_PREFIXES):
return True
return False
def _is_trusted_host(host: str) -> bool:
h = _normalize_host(host)
if not h:
return False
if h in _TRUSTED_PUBLIC_HOST_LITERALS:
return True
return any(h.endswith(s) for s in _TRUSTED_PUBLIC_HOST_SUFFIXES)
def _call_is_upload_shape(node: ast.Call, fq: str) -> bool:
"""True for statically obvious upload shapes (files=, data=open(), bytes literal)."""
if fq in _UPLOAD_HF_FQ:
return True
if fq not in _UPLOAD_HTTP_METHODS:
return False
for kw in node.keywords or []:
if kw.arg == "files":
return True
if kw.arg == "data":
v = kw.value
if isinstance(v, ast.Call) and isinstance(v.func, ast.Name) and v.func.id == "open":
return True
if isinstance(v, ast.Constant) and isinstance(v.value, (bytes, bytearray)):
return True
return False
# Bare method-name fallback (`x.upload_file(...)`) is fuzzy, so it fires only
# when huggingface_hub/hf_api is imported; else paramiko.upload_file,
# boto3.create_commit, etc. would false-positive. Pre-scan for the imports.
_HF_IMPORT_MODULES = (
"huggingface_hub",
"hf_api",
"huggingface_hub.hf_api",
)
def _module_has_hf_import(tree: ast.AST) -> bool:
for n in ast.walk(tree):
if isinstance(n, ast.Import):
for alias in n.names:
if alias.name.split(".", 1)[0] in _HF_IMPORT_MODULES:
return True
elif isinstance(n, ast.ImportFrom):
root = (n.module or "").split(".", 1)[0]
if root in _HF_IMPORT_MODULES:
return True
elif isinstance(n, ast.Call) and n.args:
# __import__('huggingface_hub'), importlib.import_module(...),
# and bare import_module(...) (via `from importlib import ...`).
arg0 = n.args[0]
if not (isinstance(arg0, ast.Constant) and isinstance(arg0.value, str)):
continue
if arg0.value.split(".", 1)[0] not in _HF_IMPORT_MODULES:
continue
func = n.func
if isinstance(func, ast.Name) and func.id in {
"__import__",
"import_module",
}:
return True
if isinstance(func, ast.Attribute) and func.attr == "import_module":
return True
return False
_hf_in_scope = _module_has_hf_import(tree)
def _method_call_hf_upload_name(node: ast.Call) -> str | None:
"""Return the HF upload method name (`upload_file`, ...) or None. Covers
the Attribute and bare-Name forms; the bare-name branch fires only when
an HF import is in scope so paramiko/boto3 don't false-positive."""
if not _hf_in_scope:
return None
f = node.func
if isinstance(f, ast.Attribute) and f.attr in _UPLOAD_HF_METHODS:
return f.attr
if isinstance(f, ast.Name) and f.id in _UPLOAD_HF_METHODS:
return f.id
return None
# Kwargs that ship a credential over the wire. The sandbox env strips
# credentials up front, so any value here is hard-coded or lifted from parent.
_HF_SENSITIVE_KWARGS = frozenset(
{
"token",
"hf_token",
"api_token",
"api_key",
"auth_token",
"access_token",
"password",
"secret",
}
)
def _is_os_environ(node: ast.AST) -> bool:
return (
isinstance(node, ast.Attribute)
and node.attr == "environ"
and isinstance(node.value, ast.Name)
and node.value.id == "os"
)
def _reads_env_or_secret(node: ast.AST | None) -> bool:
"""True if any node in the subtree resolves to an env/process read.
Walks the whole subtree (not just the root) to catch wrappers like
`str(os.environ)`. Covers os.environ[/.get]/os.getenv, bare getenv, and
subprocess.{run,check_output,...} that could lift parent env via printenv.
"""
if node is None:
return False
for sub in ast.walk(node):
if _is_os_environ(sub):
return True
if isinstance(sub, ast.Call):
f = sub.func
if isinstance(f, ast.Attribute):
if (
f.attr in {"getenv", "getenvb"}
and isinstance(f.value, ast.Name)
and f.value.id == "os"
):
return True
if (
f.attr
in {
"check_output",
"run",
"Popen",
"getoutput",
"getstatusoutput",
}
and isinstance(f.value, ast.Name)
and f.value.id in {"subprocess", "commands"}
):
return True
if isinstance(f, ast.Name) and f.id in {"getenv", "getenvb"}:
return True
return False
def _is_safe_relative_path(path: str) -> bool:
"""Relative path with no leading `/`, `~`, drive letter, or `..` segments."""
if not isinstance(path, str) or not path:
return False
if path[0] in ("/", "\\", "~"):
return False
if len(path) >= 2 and path[1] == ":":
return False
return ".." not in path.replace("\\", "/").split("/")
def _path_arg_is_sandbox_local(node: ast.AST | None) -> bool:
"""Whether the path argument resolves to a sandbox-local literal."""
if node is None:
return False
if isinstance(node, ast.Constant) and isinstance(node.value, (bytes, bytearray)):
return True # inline bytes, no file access
if isinstance(node, ast.Constant) and isinstance(node.value, str):
return _is_safe_relative_path(node.value)
if isinstance(node, ast.Call):
f = node.func
is_open = (isinstance(f, ast.Name) and f.id == "open") or (
isinstance(f, ast.Attribute) and f.attr == "open"
)
if is_open and node.args:
a0 = node.args[0]
return (
isinstance(a0, ast.Constant)
and isinstance(a0.value, str)
and _is_safe_relative_path(a0.value)
)
return False
def _hf_upload_violation(node: ast.Call, method_name: str) -> str | None:
"""Inspect an HF upload call; return a violation reason or None.
Policy: HF uploads are allowed only when (a) no sensitive kwarg is set,
(b) no positional / keyword value reads `os.environ` or related env
readers, and (c) the path arg is a sandbox-local literal: a relative
string with no `..`, an `open(<literal>)`, or inline bytes. Dynamic /
variable paths are rejected since safety can't be proven statically and
a wrong-allow means credential exfiltration.
"""
for kw in node.keywords or []:
if kw.arg in _HF_SENSITIVE_KWARGS:
return (
f"HF upload {kw.arg}= cannot be set from sandboxed code; "
"uploads run with the sandbox identity only"
)
all_values = list(node.args or []) + [kw.value for kw in (node.keywords or [])]
for v in all_values:
if _reads_env_or_secret(v):
return (
"HF upload cannot include os.environ / os.getenv / subprocess "
"env reads; secrets and tokens must not be exfiltrated"
)
if method_name == "create_commit":
for kw in node.keywords or []:
if kw.arg == "operations" and isinstance(kw.value, ast.List):
for elt in kw.value.elts:
if isinstance(elt, ast.Call):
inner = _hf_upload_violation(elt, "upload_file")
if inner:
return inner
return None
path_node: ast.AST | None = node.args[0] if node.args else None
for kw in node.keywords or []:
if kw.arg in ("path_or_fileobj", "folder_path"):
path_node = kw.value
break
if not _path_arg_is_sandbox_local(path_node):
return (
"HF upload path must be a sandbox-local relative-path literal "
"(no absolute paths, no '..' segments, no dynamic expressions)"
)
return None
# Network-module import aliases so the FQ prefix match sees the canonical module even
# when it is renamed: import requests as r -> {"r": "requests"}, import urllib.request as
# u -> {"u": "urllib.request"}, from urllib import request as req -> {"req":
# "urllib.request"}. Without this, r.get('http://169.254.169.254/') builds fq="r.get"
# and skips every metadata / allowlist / upload check.
_NET_TOP_MODULES = ("socket", "urllib", "urllib3", "requests", "http", "httpx", "aiohttp")
_net_aliases: dict[str, str] = {}
for _n in ast.walk(tree):
if isinstance(_n, ast.Import):
for _a in _n.names:
if _a.asname and _a.name.split(".")[0] in _NET_TOP_MODULES:
_net_aliases[_a.asname] = _a.name
elif isinstance(_n, ast.ImportFrom) and _n.module:
if _n.module.split(".")[0] in _NET_TOP_MODULES:
for _a in _n.names:
_net_aliases[_a.asname or _a.name] = f"{_n.module}.{_a.name}"
# The URL / address keyword arguments the stdlib + common HTTP clients accept, so a
# keyword host (requests.get(url=...), urlopen(url=...), create_connection(address=...))
# is extracted the same as a positional one.
_NET_URL_KWARGS = ("url",)
_NET_ADDR_KWARGS = ("address", "sock_addr")
class NetworkAndIoVisitor(ast.NodeVisitor):
def visit_Call(self, node):
parts: list[str] = []
cur = node.func
while isinstance(cur, ast.Attribute):
parts.insert(0, cur.attr)
cur = cur.value
if isinstance(cur, ast.Name):
parts.insert(0, cur.id)
# Resolve a renamed network module (import requests as r) to its canonical name
# so the FQ-prefix match below still fires.
if parts and parts[0] in _net_aliases:
parts = _net_aliases[parts[0]].split(".") + parts[1:]
fq = ".".join(parts) if parts else ""
hf_upload_name = _method_call_hf_upload_name(node)
if hf_upload_name is not None:
violation = _hf_upload_violation(node, hf_upload_name)
if violation is not None:
network_calls.append(
{
"type": "upload_blocked",
"line": getattr(node, "lineno", -1),
"description": f"Blocked: {violation}",
}
)
# Direct sock.connect((host, port)) bypasses the FQ-prefix branch. Only the
# (host, port) TUPLE form is an AF_INET network connect; a bare-string arg to
# .connect() is an AF_UNIX socket PATH or a DB connector path (sqlite3.connect(
# 'local.db'), duckdb.connect(':memory:')), not a network host, so restrict host
# classification to the tuple form (the bare-string branch only mis-flagged benign
# local database opens; filesystem escape for those is enforced at runtime instead).
if isinstance(node.func, ast.Attribute) and node.func.attr == "connect":
a0 = node.args[0] if node.args else None
if a0 is None:
for _kw in node.keywords or []:
if _kw.arg == "address":
a0 = _kw.value
break
host_lit = None
if isinstance(a0, ast.Tuple) and a0.elts:
e0 = a0.elts[0]
if isinstance(e0, ast.Constant) and isinstance(e0.value, str):
host_lit = e0.value
if host_lit:
if _is_metadata_host(host_lit):
network_calls.append(
{
"type": "metadata_host_blocked",
"line": getattr(node, "lineno", -1),
"description": "Blocked: cloud-metadata host",
}
)
elif not _is_trusted_host(host_lit):
network_calls.append(
{
"type": "untrusted_host_blocked",
"line": getattr(node, "lineno", -1),
"description": (
"Blocked: host not in sandbox allowlist; "
"use an allowed informational source"
),
}
)
if fq and any(fq.startswith(p) for p in _NETWORK_FQ_PREFIXES):
# 1) Upload-shape check (host-independent).
if _call_is_upload_shape(node, fq):
network_calls.append(
{
"type": "upload_blocked",
"line": getattr(node, "lineno", -1),
"description": ("Blocked: file upload disallowed in sandbox"),
}
)
# 2) Extract literal host (URL string or (host, port) tuple). The host may be
# a positional first arg OR a keyword (requests.get(url=...),
# urlopen(url=...), create_connection(address=(host, port))).
host_arg = None
url_arg = None
a0 = node.args[0] if node.args else None
if a0 is None:
for _kw in node.keywords or []:
if _kw.arg in _NET_URL_KWARGS:
a0 = _kw.value
break
if _kw.arg in _NET_ADDR_KWARGS:
a0 = _kw.value
break
if a0 is not None:
if isinstance(a0, ast.Constant) and isinstance(a0.value, str):
url_arg = a0.value
elif isinstance(a0, ast.Tuple) and a0.elts:
e0 = a0.elts[0]
if isinstance(e0, ast.Constant) and isinstance(e0.value, str):
host_arg = e0.value
if url_arg and host_arg is None:
m = re.match(r"^\w+://([^/?#]+)", url_arg)
if m:
host_arg = m.group(1)
if host_arg:
if _is_metadata_host(host_arg):
network_calls.append(
{
"type": "metadata_host_blocked",
"line": getattr(node, "lineno", -1),
"description": "Blocked: cloud-metadata host",
}
)
elif not _is_trusted_host(host_arg):
network_calls.append(
{
"type": "untrusted_host_blocked",
"line": getattr(node, "lineno", -1),
"description": (
"Blocked: host not in sandbox allowlist; "
"use an allowed informational source"
),
}
)
is_open_call = (
(isinstance(node.func, ast.Name) and node.func.id == "open")
or fq in ("io.open", "pathlib.Path.open")
or fq.endswith(".open")
)
if is_open_call and node.args:
a0 = node.args[0]
path_lit = None
if isinstance(a0, ast.Constant) and isinstance(a0.value, str):
path_lit = a0.value
if path_lit:
flagged = False
if any(path_lit.startswith(p) for p in _SENSITIVE_FILE_PREFIXES):
flagged = True
elif _SENSITIVE_FILE_RE.match(path_lit):
flagged = True
if flagged:
sensitive_file_reads.append(
{
"type": "sensitive_file_read",
"line": getattr(node, "lineno", -1),
"description": (
f"open({path_lit!r}) targets a host identity / "
"credential file; sandboxed code may not read it"
),
}
)
self.generic_visit(node)
def _fs_block(node, description):
filesystem_violations.append(
{
"type": "filesystem_violation",
"line": getattr(node, "lineno", -1),
"description": description,
}
)
# Read-only scanner: filesystem WRITES are confined at runtime by the Stage 5
# realpath backstop, so this static pass only blocks host-secret READS (the
# backstop leaves reads unpatched). A sensitive absolute / ~-rooted literal in
# ANY call arg is flagged -- this covers open()/os.open and library loaders that
# internally open the path (pandas.read_csv('/etc/shadow'), numpy.load('/etc/passwd')).
# The `..` / `~` traversal-escape form is flagged only for the dedicated
# open()/read callees, so benign relative-path building (os.path.join('..','x'))
# is not caught. Dynamic (non-foldable) paths are left to the runtime backstop.
_READ_METHODS = ("read_text", "read_bytes")
# Pathlib read methods carry the path on the RECEIVER, not in an argument:
# Path('../../.ssh/id_rsa').read_text() has no call args, so the constructor path
# must be inspected separately.
_PATHLIB_READ_METHODS = ("read_text", "read_bytes", "open")
_PATHLIB_CTORS = (
"Path",
"PurePath",
"PosixPath",
"PurePosixPath",
"WindowsPath",
"PureWindowsPath",
)
# shutil.copy*/move read their SOURCE (first arg) from the host, so a `..` traversal
# or ~ source copies a host secret into the workdir even though it is not an open()/
# read callee. Treat them as read callees so the traversal/sensitive check applies.
_SHUTIL_COPY_SINKS = (
"shutil.copy",
"shutil.copy2",
"shutil.copyfile",
"shutil.copytree",
"shutil.move",
)
_SHUTIL_COPY_METHODS = ("copy", "copy2", "copyfile", "copytree", "move")
# Import aliases so the traversal check still recognizes a renamed callee:
# from pathlib import Path as P -> P('../../etc/passwd').read_text()
# import shutil as sh -> sh.copy('../../etc/passwd', 'x')
_pathlib_ctor_aliases = set(_PATHLIB_CTORS)
_shutil_aliases = {"shutil"}
# from shutil import copy as c / copyfile / move -> bare-name aliases whose SOURCE (first
# arg) is a host read, e.g. c('../../../etc/passwd', 'x'). Tracked so the traversal check
# treats them as read callees like the attribute form shutil.copy(...).
_shutil_copy_from_aliases: set[str] = set()
# from os import open as oo / from io import open as X / from builtins import open as X:
# the read-only os.open is deliberately allowed OUTSIDE the workdir by the runtime
# guard, so a traversal read via such an alias must be caught statically.
_open_from_aliases: set[str] = set()
# Receiver-module aliases for the open() attribute form (import builtins as b; b.open(...),
# import io as i; i.open(...), import os as o; o.open(...)), so an aliased-module read is
# recognized like the literal builtins/io/os.open forms.
_open_mod_aliases = {"builtins", "__builtins__", "io", "os"}
# os/subprocess module aliases + from-import shell-name aliases, so a shell command
# string that reads a host secret (os.system('cat /etc/passwd')) is scanned even when
# os/subprocess is renamed.
_os_mod_aliases = {"os"}
_subprocess_mod_aliases = {"subprocess"}
_shell_name_aliases: dict[str, str] = {}
# from subprocess import run as r / call / check_call / check_output / Popen -> bare-name
# aliases that run an unguarded child, so r(['cat', '../../etc/shadow']) reads a host
# secret. Tracked so the read-callee traversal check recognizes them like subprocess.run.
_subprocess_exec_from_aliases: set[str] = set()
# from os.path import join as j / normpath / abspath -> {alias: 'join'} so a path builder
# folder recognizes the bare-name form open(join('/etc', 'passwd')).
_pathfunc_from_aliases: dict[str, str] = {}
# operator module + `from operator import methodcaller` aliases, so a deferred method
# applied to an os/subprocess receiver (methodcaller('popen', 'cat /etc/passwd')(os)) is
# rewritten to the direct call before the read scanner runs.
_operator_mod_aliases = {"operator"}
_methodcaller_from_aliases: set[str] = set()
for _imp in ast.walk(tree):
if isinstance(_imp, ast.ImportFrom) and _imp.module == "pathlib":
for _a in _imp.names:
if _a.name in _PATHLIB_CTORS:
_pathlib_ctor_aliases.add(_a.asname or _a.name)
elif isinstance(_imp, ast.ImportFrom) and _imp.module in ("os", "io", "builtins"):
for _a in _imp.names:
if _a.name == "open":
_open_from_aliases.add(_a.asname or "open")
# `from os import system as s` / popen: record the os shell-exec alias here too
# (this elif consumes the `os` module, so the subprocess branch below never sees
# it), else _scan_shell_string_reads skips s('cat /etc/passwd').
_fq = f"{_imp.module}.{_a.name}"
if _fq in _SHELL_EXEC_FUNCS:
_shell_name_aliases[_a.asname or _a.name] = _fq
elif isinstance(_imp, ast.ImportFrom) and _imp.module == "shutil":
for _a in _imp.names:
if _a.name in _SHUTIL_COPY_METHODS:
_shutil_copy_from_aliases.add(_a.asname or _a.name)
elif isinstance(_imp, ast.ImportFrom) and _imp.module == "subprocess":
for _a in _imp.names:
_fq = f"subprocess.{_a.name}"
if _fq in _SHELL_EXEC_FUNCS:
_shell_name_aliases[_a.asname or _a.name] = _fq
if _a.name in ("run", "call", "check_call", "check_output", "Popen"):
_subprocess_exec_from_aliases.add(_a.asname or _a.name)
elif isinstance(_imp, ast.ImportFrom) and _imp.module in (
"os.path",
"posixpath",
"ntpath",
):
for _a in _imp.names:
if _a.name in ("join", "normpath", "abspath"):
_pathfunc_from_aliases[_a.asname or _a.name] = _a.name
elif isinstance(_imp, ast.ImportFrom) and _imp.module == "operator":
for _a in _imp.names:
if _a.name == "methodcaller":
_methodcaller_from_aliases.add(_a.asname or _a.name)
elif isinstance(_imp, ast.Import):
for _a in _imp.names:
if _a.name == "shutil":
_shutil_aliases.add(_a.asname or "shutil")
elif _a.name == "operator":
_operator_mod_aliases.add(_a.asname or "operator")
elif _a.name == "os":
_os_mod_aliases.add(_a.asname or "os")
_open_mod_aliases.add(_a.asname or "os") # o.open(...)
elif _a.name in ("posix", "nt"):
# posix / nt are the os C backend (posix.system == os.system), so a shell
# string passed to them must be scanned for embedded secret reads too.
_os_mod_aliases.add(_a.asname or _a.name)
elif _a.name in ("io", "builtins"):
_open_mod_aliases.add(_a.asname or _a.name) # i.open(...) / b.open(...)
elif _a.name == "subprocess":
_subprocess_mod_aliases.add(_a.asname or "subprocess")
def _fold_pathjoin_call(call):
# Fold an os.path.join/normpath/abspath call that _const_fold's owner check misses
# because os is aliased (import os as o -> o.path.join) or the function is
# from-imported (from os.path import join -> join(...)). Recurses through
# _fold_read_arg so scope-local constants inside the args still resolve.
if not isinstance(call, ast.Call):
return None
fn = call.func
pname = None
if isinstance(fn, ast.Attribute) and fn.attr in ("join", "normpath", "abspath"):
owner = fn.value
if (
isinstance(owner, ast.Attribute)
and owner.attr == "path"
and isinstance(owner.value, ast.Name)
and owner.value.id in _os_mod_aliases
):
pname = fn.attr
elif isinstance(owner, ast.Name) and owner.id in ("posixpath", "ntpath"):
pname = fn.attr
elif isinstance(fn, ast.Name) and fn.id in _pathfunc_from_aliases:
pname = _pathfunc_from_aliases[fn.id]
if pname is None or not call.args:
return None
parts = []
for a in call.args:
v = _fold_read_arg(a)
if v is None:
return None
parts.append(v)
try:
if pname == "join":
return os.path.join(*parts)
if len(parts) == 1:
return (
os.path.normpath(parts[0]) if pname == "normpath" else os.path.abspath(parts[0])
)
except Exception:
return None
return None
def _unwrap_container_node(n):
# `[open][0]` / `(open,)[0]` / `{'k': open}['k']`: resolve an inline literal-container
# index to the element node so a container-hidden alias is seen through.
if not isinstance(n, ast.Subscript):
return n
container = n.value
ci = _const_fold(n.slice, _const_env)
if isinstance(container, (ast.List, ast.Tuple)) and isinstance(ci, int):
if -len(container.elts) <= ci < len(container.elts):
return container.elts[ci]
if isinstance(container, ast.Dict) and ci is not None:
for k, v in zip(container.keys, container.values):
if k is not None and _const_fold(k, _const_env) == ci:
return v
return n
def _resolves_to_open(fn):
# A callee that is `open`, a `from os/io/builtins import open as X` alias, a
# single-assignment alias (o = open; o('../../etc/passwd').read()), a
# container-hidden alias (o = [open][0]; o(...)), the attribute forms
# builtins.open / __builtins__.open / io.open / os.open, or any of these behind a
# trailing .__call__ (open.__call__('../../etc/passwd')).
while isinstance(fn, ast.Attribute) and fn.attr == "__call__":
fn = fn.value
if isinstance(fn, ast.Name):
if fn.id == "open" or fn.id in _open_from_aliases:
return True
rhs = _unwrap_container_node(_scope_idx.resolve(fn.id, fn, "rhsnode"))
if isinstance(rhs, ast.Name) and rhs.id == "open":
return True
if (
isinstance(rhs, ast.Attribute)
and rhs.attr == "open"
and isinstance(rhs.value, ast.Name)
and rhs.value.id in _open_mod_aliases
):
return True
if (
isinstance(fn, ast.Attribute)
and fn.attr == "open"
and isinstance(fn.value, ast.Name)
and fn.value.id in _open_mod_aliases
):
return True
return False
def _is_shutil_copy_callee(fn):
while isinstance(fn, ast.Attribute) and fn.attr == "__call__":
fn = fn.value
# A single-assignment alias (c = shutil.copy; c('../../etc/passwd', 'x')) hides the
# shutil.copy attribute form behind a bare Name, so resolve the RHS before matching.
if isinstance(fn, ast.Name):
fn = _unwrap_container_node(_scope_idx.resolve(fn.id, fn, "rhsnode"))
return (
isinstance(fn, ast.Attribute)
and fn.attr in _SHUTIL_COPY_METHODS
and isinstance(fn.value, ast.Name)
and fn.value.id in _shutil_aliases
)
def _is_subprocess_exec_callee(fn):
# subprocess.run/call/check_call/check_output/Popen run an unguarded child, so a
# `..` traversal in a literal argv (subprocess.run(['cat', '../../root/.ssh/id_rsa']))
# reads a host secret. Treat these as read callees so the traversal check fires on
# their argv path elements (absolute-sensitive elements already block regardless).
while isinstance(fn, ast.Attribute) and fn.attr == "__call__":
fn = fn.value
# A from-import (from subprocess import run as r -> r([...])) or a single-assignment
# alias (r = subprocess.run) both hide the exec attribute form behind a bare Name;
# resolve/recognize them before matching the attribute form.
if isinstance(fn, ast.Name):
if fn.id in _subprocess_exec_from_aliases:
return True
fn = _unwrap_container_node(_scope_idx.resolve(fn.id, fn, "rhsnode"))
return (
isinstance(fn, ast.Attribute)
and isinstance(fn.value, ast.Name)
and fn.value.id in _subprocess_mod_aliases
and fn.attr in ("run", "call", "check_call", "check_output", "Popen")
)
def _fold_read_arg(arg):
# Fold a read-path argument to a concrete string, resolving a module-level
# constant (via _const_env) OR a function-local single-assignment string
# constant (p = '/etc/passwd' inside a def) via the scope index.
v = _const_fold(arg, _const_env)
if isinstance(v, (str, bytes, bytearray)):
return _to_text(v)
if isinstance(arg, ast.Name):
sv = _scope_idx.resolve(arg.id, arg, "strconst")
if isinstance(sv, (str, bytes, bytearray)):
return _to_text(sv)
return None
# os-aliased / from-imported path builder (o.path.join(...), join(...)) that
# _const_fold's literal-`os` owner check misses.
pj = _fold_pathjoin_call(arg)
if isinstance(pj, (str, bytes, bytearray)):
return _to_text(pj)
# A path-builder call (os.path.join(p, 'passwd'), normpath, ...) whose arguments
# include function-local single-assignment string constants stays opaque to the
# module-level _const_env. Augment the fold env with those scope-local names' RHS
# NODES and re-fold so `p = '/etc'; open(os.path.join(p, 'passwd'))` is caught.
# (_const_fold maps names to RHS nodes, not values.)
_local_env = None
for _sub in ast.walk(arg):
if isinstance(_sub, ast.Name) and isinstance(_sub.ctx, ast.Load):
if _const_env is not None and _sub.id in _const_env:
continue
_svn = _scope_idx.resolve(_sub.id, _sub, "rhsnode")
if _svn is not None:
if _local_env is None:
_local_env = dict(_const_env or {})
_local_env[_sub.id] = _svn
if _local_env is not None:
v = _const_fold(arg, _local_env)
if isinstance(v, (str, bytes, bytearray)):
return _to_text(v)
return None
def _pathlib_receiver_path(recv, _seen = None):
# Resolve a pathlib receiver to a concrete path: Path(...) / pathlib.Path(...)
# (all constructor args joined), a `/` join (Path('/etc') / 'passwd'), a
# .joinpath(...) chain, or a single-assignment name bound to any of these
# (p = Path('..') / 'etc' / 'passwd'; p.read_text()).
if isinstance(recv, ast.Name):
# Resolve the name to its single-assignment RHS (cycle-guarded).
if _seen is None:
_seen = set()
if recv.id in _seen:
return None
_seen.add(recv.id)
rhs = _scope_idx.resolve(recv.id, recv, "rhsnode")
if rhs is None:
return None
return _pathlib_receiver_path(rhs, _seen)
if isinstance(recv, ast.BinOp) and isinstance(recv.op, ast.Div):
base = _pathlib_receiver_path(recv.left, _seen)
rv = _fold_read_arg(recv.right)
if base is None or rv is None:
return None
try:
return os.path.join(base, rv)
except Exception:
return None
if not isinstance(recv, ast.Call):
return None
rf = recv.func
# No-op path-identity methods (resolve/absolute/expanduser) return the same file, so
# look through them: Path('/etc').joinpath('passwd').resolve().read_text() still
# reads /etc/passwd. expanduser() only makes a leading ~ concrete, which the
# sensitive check already handles on the pre-expansion form.
if isinstance(rf, ast.Attribute) and rf.attr in ("resolve", "absolute", "expanduser"):
return _pathlib_receiver_path(rf.value, _seen)
if isinstance(rf, ast.Attribute) and rf.attr == "joinpath":
base = _pathlib_receiver_path(rf.value, _seen)
if base is None:
return None
parts = [base]
for a in recv.args:
v = _fold_read_arg(a)
if v is None:
return None
parts.append(v)
try:
return os.path.join(*parts)
except Exception:
return None
# A single-assignment alias of the constructor (P = pathlib.Path / P = Path) is not
# in _pathlib_ctor_aliases, so resolve a Name callee's RHS to see if it binds a
# pathlib constructor before giving up.
_ctor_name = isinstance(rf, ast.Name) and rf.id in _pathlib_ctor_aliases
if not _ctor_name and isinstance(rf, ast.Name):
_crhs = _unwrap_container_node(_scope_idx.resolve(rf.id, rf, "rhsnode"))
_ctor_name = (isinstance(_crhs, ast.Name) and _crhs.id in _pathlib_ctor_aliases) or (
isinstance(_crhs, ast.Attribute) and _crhs.attr in _PATHLIB_CTORS
)
ctor = _ctor_name or (isinstance(rf, ast.Attribute) and rf.attr in _PATHLIB_CTORS)
if not ctor or not recv.args:
return None
parts = []
for a in recv.args:
v = _fold_read_arg(a)
if v is None:
return None
parts.append(v)
if not parts:
return None
try:
return os.path.join(*parts)
except Exception:
return None
def _flag_read_path(node, s, is_read_callee):
norm = s.replace("\\", "/")
# Collapse redundant separators / '.' segments and resolve '..' so equivalent
# spellings (/etc//passwd, /etc/./passwd, /tmp/../etc/passwd) still match the
# sensitive exact / dir checks. Keep the raw form for the traversal check below.
try:
canon = os.path.normpath(norm)
except Exception:
canon = norm
if _is_sensitive_abs_path(norm) or _is_sensitive_abs_path(canon):
_fs_block(node, f"{s!r} is a sensitive host identity / credential file")
return True
if is_read_callee and (s[:1] == "~" or ".." in norm.split("/")):
_fs_block(node, f"{s!r} escapes the session workdir via path traversal")
return True
return False
# Shell sinks whose first argument is always interpreted as a shell command STRING
# (os.system('cat /etc/passwd') runs an unguarded child that leaks the file in stdout).
_STRING_SHELL_SINKS = frozenset(
{
"os.system",
"os.popen",
"os.popen2",
"os.popen3",
"os.popen4",
"subprocess.getoutput",
"subprocess.getstatusoutput",
}
)
def _shell_string_sink_fq(f):
# Resolve a callee to its fq shell-sink name honoring os/subprocess module aliases
# and from-import name aliases (from subprocess import getoutput as g), else None.
if isinstance(f, ast.Attribute):
v = f.value
# os / posix / nt receiver as a simple name (posix.system) or a module that
# re-exports os as an attribute (pathlib.os.system, tempfile.os.system).
if (isinstance(v, ast.Name) and v.id in _os_mod_aliases) or (
isinstance(v, ast.Attribute) and v.attr in ("os", "posix", "nt")
):
cand = f"os.{f.attr}"
elif (isinstance(v, ast.Name) and v.id in _subprocess_mod_aliases) or (
isinstance(v, ast.Attribute) and v.attr == "subprocess"
):
cand = f"subprocess.{f.attr}"
else:
cand = None
if cand in _SHELL_EXEC_FUNCS:
return cand
elif isinstance(f, ast.Name):
return _shell_name_aliases.get(f.id)
return None
def _rewrite_methodcaller_call(node):
# operator.methodcaller('popen', 'cat /etc/passwd')(os) applies a deferred method to a
# module receiver; rewrite it to the direct os.popen('cat /etc/passwd') call so the
# read scanner tokenizes the embedded secret read. Only os/subprocess receivers are
# rewritten, so a methodcaller aimed at a benign object is left untouched.
if len(node.args) != 1 or node.keywords:
return None
mc = node.func
while isinstance(mc, ast.Attribute) and mc.attr == "__call__":
mc = mc.value
if not isinstance(mc, ast.Call) or not mc.args:
return None
mf = mc.func
is_mc = (
isinstance(mf, ast.Attribute)
and mf.attr == "methodcaller"
and isinstance(mf.value, ast.Name)
and mf.value.id in _operator_mod_aliases
) or (isinstance(mf, ast.Name) and mf.id in _methodcaller_from_aliases)
if not is_mc:
return None
meth = _fold_read_arg(mc.args[0])
if not isinstance(meth, str) or not meth.isidentifier():
return None
receiver = node.args[0]
if not (
isinstance(receiver, ast.Name)
and (receiver.id in _os_mod_aliases or receiver.id in _subprocess_mod_aliases)
):
return None
synth = ast.Call(
func = ast.Attribute(value = receiver, attr = meth, ctx = ast.Load()),
args = list(mc.args[1:]),
keywords = list(mc.keywords),
)
ast.copy_location(synth, node)
ast.fix_missing_locations(synth)
return synth
def _is_exec_family_callee(f):
# os.execv / os.execl / os.spawnv / os.posix_spawn ... replace or fork the guarded
# process with an unguarded program, so a `..` traversal in their argv reads a host
# secret the same way subprocess argv does (os.execv('/bin/cat', ['cat',
# '../../etc/shadow'])). Treat them as read callees for the traversal check.
while isinstance(f, ast.Attribute) and f.attr == "__call__":
f = f.value
fq = _shell_string_sink_fq(f)
return fq is not None and (fq.startswith("os.exec") or fq.startswith("os.spawn"))
def _iter_call_kwargs(call):
# Yield (name, value_node) for every keyword argument, EXPANDING a literal **{...}
# unpack (subprocess.run(cmd, **{'shell': True, 'cwd': p})) so a shell / cwd argument
# smuggled through a dict unpack is seen exactly like an explicit shell= / cwd= kwarg.
for _kw in call.keywords or []:
if _kw.arg is not None:
yield _kw.arg, _kw.value
elif isinstance(_kw.value, ast.Dict):
for _dk, _dv in zip(_kw.value.keys, _kw.value.values):
if (
_dk is not None
and isinstance(_dk, ast.Constant)
and isinstance(_dk.value, str)
):
yield _dk.value, _dv
def _scan_shell_string_reads(node, f):
# os.system('cat /etc/passwd') / subprocess.run('cat /etc/passwd', shell=True): the
# read scanner otherwise treats the whole command as one opaque path candidate, and
# _is_sensitive_abs_path ignores strings with whitespace. Tokenize the command and
# check each token as a read path so an embedded host-secret read is caught.
def _first_cmd_arg():
# The command may be positional OR the public `args=` keyword
# (subprocess.run(args='cat /etc/passwd', shell=True) / run(args=['cat', p])),
# including a literal **{'args': ...} unpack.
if node.args:
return node.args[0]
for _name, _val in _iter_call_kwargs(node):
if _name == "args":
return _val
return None
# subprocess.run('cat passwd', shell=True, cwd='/etc') runs the payload in an unguarded
# shell whose cwd is /etc, so a relative reader arg reads /etc/passwd; a NON-literal cwd
# cannot be proven sandbox-local. Extract cwd= once and thread it into the payload scan.
_cwd_lit = None
_cwd_dyn = False
if _is_subprocess_exec_callee(f):
for _name, _val in _iter_call_kwargs(node):
if _name == "cwd":
_cv = _fold_read_arg(_val)
if isinstance(_cv, str):
_cwd_lit = _cv
elif not (isinstance(_val, ast.Constant) and _val.value is None):
_cwd_dyn = True
break
def _scan_one_command(cmd):
# Scan a shell command STRING (folded to a literal) for an embedded host-secret
# read and record a violation. Delegates to the shared scanner in strict-traversal
# mode (the os.system() shell-string policy blocks ANY .. / ~ read path), which also
# resolves the reader past assignment / wrapper prefixes and recurses nested shells.
if cmd is None:
return False
_r = _scan_command_string_for_reads(
cmd, strict_traversal = True, cwd = _cwd_lit, cwd_dynamic = _cwd_dyn
)
if _r is not None:
_fs_block(node, _r)
return True
return False
# A subprocess argv that invokes a shell with -c runs the payload in an unguarded
# child (subprocess.run(['sh', '-c', 'head -1 /etc/passwd'])). The blocked-command
# scanner finds no blocked command (head is benign), so scan the -c payload for
# sensitive reads here the same way a string shell sink is scanned.
if _is_subprocess_exec_callee(f):
argv = _first_cmd_arg()
if isinstance(argv, (ast.List, ast.Tuple)) and argv.elts:
_elts = [_fold_read_arg(_e) for _e in argv.elts]
# Resolve the executed command word past wrapper prefixes (env / timeout /
# nice / ...): subprocess.run(['env', 'bash', '-c', payload]) runs the nested
# shell just like subprocess.run(['bash', '-c', payload]), so scan the -c
# payload regardless of the wrapper hiding argv[0].
_ci = _blocked_in_argv(_elts)[1]
_sh = _elts[_ci] if _ci is not None and _ci < len(_elts) else None
if _sh is not None and os.path.basename(_sh).lower() in _SHELL_BINARIES:
# An env -C DIR earlier in the SAME argv chdirs the child before the nested
# shell runs (['env', '-C', '/etc', 'bash', '-c', 'cat passwd']), so the -c
# payload's relative reads resolve against DIR, not the workdir. Fold the
# argv env -C into the payload's cwd (or fail closed on a dynamic DIR).
_pl_cwd, _pl_dyn = _cwd_lit, _cwd_dyn
_envc = _argv_env_chdir(_elts)
if _envc is not None:
_rdir, _rdyn = _resolve_read_chdir(_envc, {})
if _rdyn:
_pl_dyn = True
else:
_pl_cwd = _join_chdir(_cwd_lit, _rdir)
for _k in range(_ci + 1, len(_elts)):
_ev = _elts[_k]
if _ev is not None and (
_ev == "-c"
or (
_ev.startswith("-")
and not _ev.startswith("--")
and _ev.endswith("c")
)
):
if _k + 1 < len(_elts) and _elts[_k + 1] is not None:
_rr = _scan_command_string_for_reads(
_elts[_k + 1],
strict_traversal = True,
cwd = _pl_cwd,
cwd_dynamic = _pl_dyn,
)
if _rr is not None:
_fs_block(node, _rr)
return True
break
# env -S 'payload' / --split-string in an argv (subprocess.run(['env', '-S',
# 'cat /etc/passwd'])) runs the split payload as the child; the -c block above
# only covers shell binaries, so reconstruct the argv and read-scan it when the
# resolved command word is env with a split-string flag.
if (
all(_x is not None for _x in _elts)
and any(os.path.basename(_x).lower() == "env" for _x in _elts)
and any(
_x in ("-S", "--split-string")
or _x.startswith("--split-string=")
or (_x.startswith("-S") and len(_x) > 2)
for _x in _elts
)
):
if _scan_one_command(" ".join(shlex.quote(_x) for _x in _elts)):
return True
_fq = _shell_string_sink_fq(f)
_is_str = _fq in _STRING_SHELL_SINKS
if not _is_str:
# subprocess.run/call/Popen/check_output/check_call(cmd, shell=True): a string
# command with shell=True runs through /bin/sh (these are in _SHELL_EXEC_FUNCS
# but not in _STRING_SHELL_SINKS, so check the shell= kwarg explicitly). Uses the
# subprocess-exec callee resolver so the attribute, from-import (from subprocess
# import run as r) and single-assignment (r = subprocess.run) forms are all seen.
if _is_subprocess_exec_callee(f):
for _name, _val in _iter_call_kwargs(node):
if _name == "shell" and not (
isinstance(_val, ast.Constant) and _val.value is False
):
_is_str = True
_cmd_node = _first_cmd_arg()
if not _is_str or _cmd_node is None:
return False
return _scan_one_command(_fold_read_arg(_cmd_node))
class _SensitiveReadVisitor(ast.NodeVisitor):
def visit_Call(self, node):
_rw = _rewrite_methodcaller_call(node)
if _rw is not None:
# methodcaller('popen', 'cat /etc/passwd')(os): analyze the direct call form.
self.visit_Call(_rw)
return
f = node.func
fq = _fq_attr_name(f)
method = (
f.attr
if isinstance(f, ast.Attribute)
else (f.id if isinstance(f, ast.Name) else "")
)
# A shell-command STRING sink: scan the command for embedded sensitive reads.
if _scan_shell_string_reads(node, f):
return
_is_child_exec = _is_subprocess_exec_callee(f) or _is_exec_family_callee(f)
is_read_callee = (
_resolves_to_open(f)
or fq in ("io.open", "os.open")
or fq in _SHUTIL_COPY_SINKS
or _is_shutil_copy_callee(f)
or (isinstance(f, ast.Name) and f.id in _shutil_copy_from_aliases)
or _is_child_exec
or method in _READ_METHODS
)
# subprocess.run(['cat', 'passwd'], cwd='/etc') reads /etc/passwd in an unguarded
# child: the argv entry is relative and /etc alone is not sensitive, so combine a
# literal cwd= with each relative argv path before the sensitivity check. A
# NON-literal cwd (cwd=P) cannot be proven sandbox-local, so a relative read under
# it fails closed (handled below).
_sub_cwd = None
_sub_cwd_dynamic = False
if _is_child_exec:
for _name, _val in _iter_call_kwargs(node):
if _name == "cwd":
_cv = _fold_read_arg(_val)
if isinstance(_cv, str):
_sub_cwd = _cv
elif not (isinstance(_val, ast.Constant) and _val.value is None):
_sub_cwd_dynamic = True
break
# An env -C DIR / --chdir=DIR at the front of the argv chdirs the child before
# the reader runs (['env', '-C', '/etc', 'cat', 'passwd']), just like the
# shell-string env -C case; fold that dir into the cwd used for relative reads.
_argv_for_cd = node.args[0] if node.args else None
if _argv_for_cd is None:
for _name, _val in _iter_call_kwargs(node):
if _name == "args":
_argv_for_cd = _val
break
if isinstance(_argv_for_cd, (ast.List, ast.Tuple)):
_ec = _argv_env_chdir([_fold_read_arg(_e) for _e in _argv_for_cd.elts])
if _ec is not None:
if _ec.startswith("/") or _ec.startswith("~"):
_sub_cwd = _ec # absolute env -C overrides the ambient cwd
_sub_cwd_dynamic = False
elif not _sub_cwd_dynamic:
_sub_cwd = _join_chdir(_sub_cwd, _ec)
# A file-reading child (cat / head / ...) with a relative argv path under a
# non-literal cwd could read a host secret (cwd=P; P evaluates to /etc); the child
# is unguarded, so fail closed unless the cwd is proven sandbox-local. The argv may
# be positional OR the public args= keyword (run(args=['cat', p], cwd=P)).
if _sub_cwd_dynamic:
_argv0 = None
_argv_node = node.args[0] if node.args else None
if _argv_node is None:
for _name, _val in _iter_call_kwargs(node):
if _name == "args":
_argv_node = _val
break
if isinstance(_argv_node, (ast.List, ast.Tuple)):
_av = _argv_node.elts
# Resolve the real command word past wrappers (timeout / env / nice / ...), so
# a wrapper-hidden reader (['timeout', '1', 'cat', 'passwd']) is checked, not
# just argv[0]. Then scan the reader's own relative args for a host read.
_folded = [_fold_read_arg(_e) for _e in _av]
_ci = _argv_command_word_index(_folded)
_p0 = _folded[_ci] if _ci is not None else None
if (
isinstance(_p0, str)
and os.path.basename(_p0).lower() in _SHELL_READ_COMMANDS
):
for _ae in _av[_ci + 1 :]:
_av_s = _fold_read_arg(_ae)
if (
isinstance(_av_s, str)
and _av_s
and not _av_s.startswith("-")
and not _av_s.startswith("/")
and not _av_s.startswith("~")
):
_fs_block(
node,
"child reader with a relative path under a non-literal cwd",
)
_argv0 = True
break
if _argv0:
return
# A `find` child-exec argv (subprocess.run(['find','/etc','-name','passwd','-exec',
# 'cat','{}',';'])) reads host files the flat per-element scan misses: the {} placeholder
# loses the escaping search root. Reconstruct the argv from the find command word into a
# shell string and run it through the read scanner, which carries the find-root + -exec
# logic. Only when every reconstructed element folds to a literal (else best-effort skip).
if _is_child_exec:
_fargv = node.args[0] if node.args else None
if _fargv is None:
for _name, _val in _iter_call_kwargs(node):
if _name == "args":
_fargv = _val
break
if isinstance(_fargv, (ast.List, ast.Tuple)):
_ffolded = [_fold_read_arg(_e) for _e in _fargv.elts]
_fci = _argv_command_word_index(_ffolded)
if (
_fci is not None
and isinstance(_ffolded[_fci], str)
and os.path.basename(_ffolded[_fci]).lower() == "find"
and all(isinstance(_x, str) for _x in _ffolded[_fci:])
):
_freason = _command_reads_sensitive(shlex.join(_ffolded[_fci:]))
if _freason is not None:
_fs_block(node, f"find child-exec reads a host file ({_freason})")
return
# Pathlib read on a Path(...) / join receiver: check the resolved path.
if isinstance(f, ast.Attribute) and f.attr in _PATHLIB_READ_METHODS:
rp = _pathlib_receiver_path(f.value)
if rp is not None and _flag_read_path(node, rp, True):
return
# Build the arg list, expanding a literal **{...} unpack so its path value is
# scanned (open(**{'file': '../../etc/passwd'}) reads the same file that
# open('../../etc/passwd') would, which is otherwise treated as opaque).
scan_args = list(node.args)
for kw in node.keywords or []:
if kw.arg is None and isinstance(kw.value, ast.Dict):
scan_args.extend(v for v in kw.value.values if v is not None)
else:
scan_args.append(kw.value)
# Descend into a literal list/tuple argv so a sensitive path element is
# scanned: subprocess.run(['cat', '/etc/passwd']) reads the host file in an
# unguarded child even though the top-level arg is a list, not a string.
# A literal *[...] / *(...) starred arg is unpacked positionally, so scan its
# elements too: open(*['/etc/passwd']) reads the same file open('/etc/passwd')
# would, and os.open(*['/etc/shadow', os.O_RDONLY]) is otherwise opaque.
_expanded = []
for a in scan_args:
inner = a.value if isinstance(a, ast.Starred) else a
if isinstance(inner, (ast.List, ast.Tuple)):
_expanded.extend(inner.elts)
else:
_expanded.append(inner)
scan_args = _expanded
for arg in scan_args:
s = _fold_read_arg(arg)
if s is None:
# A pathlib expression carries no foldable string constant
# (open(Path('/etc') / 'passwd')), so resolve it the same way a
# Path(...).read_text() receiver is resolved before skipping.
rp = _pathlib_receiver_path(arg)
if rp is not None and _flag_read_path(node, rp, is_read_callee):
break
# An opaque read path assembled from obfuscation primitives
# (open(''.join(map(chr, [...]))).read()) can still target a host
# secret, and reads are not runtime-confined. Apply the same
# fail-closed obfuscation policy exec payloads get: block a read
# callee whose path is built from chr/join(map)/decode/fetch/... .
if is_read_callee and _payload_has_obfuscation_primitive(arg):
_fs_block(node, "read path assembled from obfuscation primitives")
break
continue
if _flag_read_path(node, s, is_read_callee):
break
# Resolve a relative argv entry against a literal subprocess cwd= (cat passwd
# + cwd='/etc' -> /etc/passwd) so the combined host-secret read is caught.
if _sub_cwd is not None and not s.startswith("/") and not s.startswith("~"):
if _flag_read_path(node, os.path.join(_sub_cwd, s), is_read_callee):
break
self.generic_visit(node)
NetworkAndIoVisitor().visit(tree)
if _analyzer_on:
try:
_SensitiveReadVisitor().visit(tree)
except Exception: # pragma: no cover - never crashier than legacy
logger.warning("sandbox filesystem analyzer failed; skipping", exc_info = True)
filesystem_violations.clear()
is_safe = (
len(signal_tampering) == 0
and len(exception_catching) == 0
and len(shell_escapes) == 0
and len(dynamic_exec) == 0
and len(network_calls) == 0
and len(sensitive_file_reads) == 0
and len(filesystem_violations) == 0
)
return is_safe, {
"signal_tampering": signal_tampering,
"exception_catching": exception_catching,
"shell_escapes": shell_escapes,
"dynamic_exec": dynamic_exec,
"network_calls": network_calls,
"sensitive_file_reads": sensitive_file_reads,
"filesystem_violations": filesystem_violations,
"warnings": warnings,
}
def _check_code_safety(code: str) -> str | None:
"""Validate code safety via static analysis.
Returns an error message string if the code is unsafe, or None if OK.
"""
safe, info = _check_signal_escape_patterns(code)
if not safe:
# Let SyntaxError from ast.parse through so the subprocess produces a
# normal Python traceback instead of a misleading "unsafe code" message.
if info.get("error"):
return None
reasons = [item.get("description", "") for item in info.get("signal_tampering", [])]
shell_reasons = [item.get("description", "") for item in info.get("shell_escapes", [])]
exception_reasons = [
item.get("description", "") for item in info.get("exception_catching", [])
]
dynamic_reasons = [item.get("description", "") for item in info.get("dynamic_exec", [])]
network_reasons = [item.get("description", "") for item in info.get("network_calls", [])]
file_reasons = [
item.get("description", "") for item in info.get("sensitive_file_reads", [])
]
fs_reasons = [item.get("description", "") for item in info.get("filesystem_violations", [])]
all_reasons = [
r
for r in reasons
+ shell_reasons
+ exception_reasons
+ dynamic_reasons
+ network_reasons
+ file_reasons
+ fs_reasons
if r
]
if all_reasons:
return (
f"Error: unsafe code detected ({'; '.join(all_reasons)}). "
f"Please remove unsafe patterns from your code."
)
return None
def _kill_process_tree(proc) -> None:
"""SIGKILL the setsid process group; fall back to single-pid kill."""
if proc.poll() is not None:
return
try:
pgid = os.getpgid(proc.pid)
except (ProcessLookupError, PermissionError):
pgid = None
if pgid is not None:
try:
os.killpg(pgid, signal.SIGKILL)
return
except (ProcessLookupError, PermissionError):
pass
try:
proc.kill()
except (ProcessLookupError, PermissionError):
pass
def _cancel_watcher(
proc,
cancel_event,
poll_interval = 0.2,
):
"""Daemon thread that kills a process when cancel_event is set."""
while proc.poll() is None:
if cancel_event is not None and cancel_event.is_set():
_kill_process_tree(proc)
return
cancel_event.wait(poll_interval) if cancel_event else None
def _truncate(text: str, limit: int = _MAX_OUTPUT_CHARS) -> str:
if len(text) > limit:
return text[:limit] + f"\n\n... (truncated, {len(text)} chars total)"
return text
# --------------------------------------------------------------------------
# Stage 5: runtime realpath backstop injected into sandboxed Python.
#
# This child-side guard resolves the true realpath (following symlinks) of every
# MUTATING file op and refuses it unless it lands inside the session workdir. It is
# the PRIMARY filesystem-write boundary: it sees dynamic paths, pre-existing
# symlinks, and library writers that funnel through builtins.open / io.open / os.open
# (numpy.save, torch.save, pandas.to_csv, savefig, ...). Reads are left unpatched
# here -- host-secret reads are handled by the static sensitive-read scanner. Skipped
# entirely under disable_sandbox (Bypass Permissions).
#
# Accepted residuals (a Python monkeypatch layer cannot close these; OS-level
# isolation is the real boundary): native-C writers that never call a patched Python
# entry point (cv2.imwrite, some pyarrow/zipfile writers), ctypes/cffi direct syscalls,
# and the realpath-before-open TOCTOU window under adversarial in-sandbox threading.
# --------------------------------------------------------------------------
_SANDBOX_GUARD_SRC = r"""
import sys as _sys
# The exec script lives INSIDE the workdir, so Python prepends the workdir to sys.path[0].
# A malicious workdir/os.py / io.py / pathlib.py / re.py (dropped by a prior run or upload)
# would otherwise shadow the guard's OWN imports below and execute unguarded at import time,
# before any patch is installed. Import the guard's stdlib deps with the workdir / cwd
# stripped from the path, then restore it so ordinary user imports still resolve (os / io /
# pathlib / re are now cached as the real, patched modules). `import sys` is safe: sys is a
# built-in module, never loaded from a file.
_saved_path = list(_sys.path)
_sys.path = [_p for _p in _sys.path if _p not in ("", ".", __WORKDIR__, __WORKDIR__ + "/")]
import os as _os, builtins as _bi, io as _io, pathlib as _pl, re as _re
# Pin the builtins the guard predicates consult (isinstance / int / bytes / str / any) into
# THIS namespace so a sandboxed `builtins.isinstance = lambda *a: True` (etc.) cannot make a
# guard check lie -- e.g. isinstance(path, int) treating an outside path as an fd and
# approving an absolute write. Every guard function below resolves these names from here, not
# the mutable builtins module.
isinstance = _bi.isinstance
int = _bi.int
bytes = _bi.bytes
str = _bi.str
any = _bi.any
# NOTE: sys.path stays stripped for the WHOLE guard setup below (it also imports shutil,
# which is pure-Python and equally shadowable); it is restored at the very END of this
# prelude, just before user code runs, so ordinary user imports still resolve.
# io + pathlib are imported BEFORE any patching on purpose: on Python <= 3.11
# pathlib._NormalAccessor captures io.open / os.* into class attributes at import
# time. A C builtin captured there does not bind on instance access, but a Python
# wrapper does (self shifts into the next arg), which would corrupt Path.open /
# Path.write_text. Importing here makes the accessor capture the originals; the
# confinement is applied on the public os / io / Path.* APIs below instead.
# Capture the path helpers/separator into IMMUTABLE guard-local names BEFORE user code
# runs. _within() otherwise reads os.path.realpath / os.fspath / os.sep off the live
# module every call, so sandboxed code could reassign os.path.realpath (e.g. to a lambda
# that echoes an in-workdir path) right before a write and have the guard approve an
# outside target while the real open() still writes there. These references cannot be
# rebound by mutating the os module.
_realpath = _os.path.realpath
_fspath = _os.fspath
_fsdecode = _os.fsdecode
_sep = _os.sep
# os.path.realpath resolves symlinks by consulting the LIVE os.lstat / os.readlink (and
# os.getcwd for relative paths). Capture the originals so a monkeypatch of any of them --
# e.g. os.lstat raising so realpath stops FOLLOWING an in-workdir symlink that points
# outside -- cannot make _within() approve a path the real open() then escapes through.
_lstat = _os.lstat
_readlink = _os.readlink
_getcwd = _os.getcwd
_stat = _os.stat
# posixpath.realpath decides whether to FOLLOW a component by calling os.path.stat.S_ISLNK
# on the live stat module. Sandboxed code can set os.path.stat.S_ISLNK = lambda mode: False
# (or reassign os.path.stat) so realpath stops following an in-workdir symlink that escapes,
# leaving the target under _WD while the real open() follows it outside. Capture the module +
# S_ISLNK so both can be re-pinned before each resolution.
_stat_mod = _os.path.stat
_S_ISLNK = _stat_mod.S_ISLNK
_WD = _realpath(__WORKDIR__)
# Standard device sinks cannot persist data outside the workspace, so a write to one is
# allowed (mirrors the terminal shell redirect allowlist); benign patterns like
# open('/dev/null', 'w') to suppress output would otherwise be denied by the workdir check.
_SAFE_DEV_SINKS = frozenset(
{"/dev/null", "/dev/zero", "/dev/full", "/dev/stdout", "/dev/stderr", "/dev/tty"}
)
def _within(p):
try:
if isinstance(p, int):
return True
# Allow a write to an exact device sink. Checked on the REQUESTED path, not its
# realpath, so /dev/stdout is not followed to a redirected outside file. Normalize
# via the base str.replace (not p.replace): a str subclass could override replace()
# to return "/dev/null" while its real value is an outside file, so call the genuine
# method on the underlying buffer, which yields a plain str immune to the override.
_ps = p if isinstance(p, str) else (_fsdecode(p) if isinstance(p, (bytes, bytearray)) else None)
if _ps is not None and str.replace(_ps, "\\", "/") in _SAFE_DEV_SINKS:
return True
# os.path.realpath internally calls the LIVE os.fspath (posixpath.realpath does
# `filename = os.fspath(filename)`) and os.lstat / os.readlink / os.getcwd, so a
# sandboxed reassignment of any of them would poison the resolution even though we
# hold the original realpath. Re-pin them to the captured originals before
# resolving; the target then resolves truthfully (symlinks followed, cwd honest).
# Re-pinning per check keeps it self-healing if user code re-patches; the real
# open() receives the already-materialized str and does not route through these,
# so restoring them has no effect on the write itself.
_os.fspath = _fspath
_os.lstat = _lstat
_os.readlink = _readlink
_os.getcwd = _getcwd
_os.stat = _stat
_os.path.stat = _stat_mod
_stat_mod.S_ISLNK = _S_ISLNK
rp = _realpath(_fspath(p))
# A bytes path resolves to bytes; normalize to str so the prefix compare against
# the str _WD does not raise (which would deny a legitimate in-workdir bytes write
# such as open(b'local.txt', 'w')).
if isinstance(rp, bytes):
rp = _fsdecode(rp)
except Exception:
return False
return rp == _WD or rp.startswith(_WD + _sep)
def _deny(p, what):
raise PermissionError(
"sandbox: %s outside the session workdir is not permitted: %r" % (what, p)
)
def _gwraps(real):
# Like functools.wraps but WITHOUT publishing __wrapped__: functools.wraps stores
# the ORIGINAL unguarded callable on w.__wrapped__, and sandboxed code could reach
# it (builtins.open.__wrapped__('/etc/x', 'w'), os.rename.__wrapped__(...)) to call
# straight through every confinement below. Copy only the cosmetic metadata.
def _deco(w):
for _a in ("__module__", "__name__", "__qualname__", "__doc__"):
try:
setattr(w, _a, getattr(real, _a))
except Exception:
pass
return w
return _deco
def _fspath1(p):
# Materialize a path-like ONCE so a stateful __fspath__ cannot return a workdir
# path for the _within() check and a different path for the real syscall (TOCTOU).
# Uses the captured _fspath so a reassigned os.fspath cannot interpose here.
if isinstance(p, int):
return p
try:
return _fspath(p)
except Exception:
return p
def _mode_is_write(mode):
# Coerce through the *base* str: a str-subclass __contains__/__str__ must not be
# able to lie about whether the mode requests a write.
m = str.__str__(mode) if isinstance(mode, str) else "r"
return any(c in m for c in "wax+")
# Runtime sensitive-read backstop. The static scanner cannot fold every read path
# (open(globals()['x']), open(fetch_name()), open(''.join(...))), and reads are otherwise
# unconfined, so an opaque path could name a host secret. Deny a read whose REALPATH
# resolves to a known-sensitive host file OUTSIDE the workdir. In-workdir files are the
# sandbox's own and always allowed. The loose 'credentials' / '.pem' / '/root/' signals the
# static layer uses are intentionally NOT applied here: importing common libraries reads
# site-packages files such as google/auth/credentials.py and certifi/cacert.pem (and, under
# a root home, /root/.local/.../site-packages), so matching them at runtime would break
# imports. The specific SSH / cloud / kube / netrc / HF-token signals stay.
_SENS_EXACT = frozenset({
"/etc/passwd", "/etc/shadow", "/etc/sudoers", "/etc/gshadow", "/etc/master.passwd",
})
_SENS_DIRS = (
"/etc/ssh/", "/.ssh/", "/.aws/", "/.config/gcloud", "/.kube/", "/.docker/",
"/var/run/secrets/kubernetes.io/", "/run/secrets/kubernetes.io/",
)
_SENS_TOKENS = (
"id_rsa", "id_ed25519", ".netrc", ".git-credentials", "/.huggingface/token", ".kube/config",
)
_SENS_PROC = _re.compile(r"^/proc/(?:self|\d+)/(?:environ|cmdline|maps|mem|task/\d+/environ)$")
def _is_sensitive_read(rp):
n = rp.replace("\\", "/")
if n in _SENS_EXACT:
return True
# _SENS_DIRS entries carry a trailing slash to match a file UNDER the dir
# (/root/.ssh/id_rsa). Append one to n so the sensitive directory ITSELF
# (os.listdir('/root/.ssh') -> '/root/.ssh', no trailing slash) matches too.
if any(part in (n + "/") for part in _SENS_DIRS):
return True
if _SENS_PROC.match(n):
return True
# Dotfiles / caches under a root home hold credentials (/root/.bashrc, /root/.cache/...);
# an opaque path the static /root/ rule cannot fold could read them at runtime. Restore
# the /root/ protection here (including the root home ITSELF, /root, which a directory
# reader would enumerate), but carve out package / library trees so importing a library
# installed under a root home (site-packages, the stdlib) is not broken.
if (n == "/root" or n.startswith("/root/")) and not any(
_seg in n
for _seg in ("/site-packages/", "/dist-packages/", "/lib/python", "/lib64/python")
):
return True
low = n.lower()
return any(tok in low for tok in _SENS_TOKENS)
def _read_realpath(p):
# Resolve to a truthful realpath the same self-healing way _within does, so a
# sandboxed reassignment of os.fspath / os.lstat / os.readlink / os.getcwd cannot
# poison the resolution.
try:
_os.fspath = _fspath
_os.lstat = _lstat
_os.readlink = _readlink
_os.getcwd = _getcwd
_os.stat = _stat
_os.path.stat = _stat_mod
_stat_mod.S_ISLNK = _S_ISLNK
rp = _realpath(_fspath(p))
if isinstance(rp, bytes):
rp = _fsdecode(rp)
return rp
except Exception:
return None
def _deny_sensitive_read(p):
if isinstance(p, int):
return
rp = _read_realpath(p)
if rp is None:
return
# In-workdir files are the sandbox's own; never treat them as host secrets.
if rp == _WD or rp.startswith(_WD + _sep):
return
if _is_sensitive_read(rp):
raise PermissionError(
"sandbox: reading a sensitive host path is not permitted: %r" % (rp,)
)
def _guard_open_like(real):
@_gwraps(real)
def w(file, mode="r", *a, **k):
f = _fspath1(file)
if _mode_is_write(mode):
if not _within(f):
_deny(f, "write")
else:
_deny_sensitive_read(f)
return real(f, mode, *a, **k)
return w
_bi.open = _guard_open_like(_bi.open)
# Low-level os.open: builtins.open does not route through it, so it needs its own
# guard. Any mutating open flag confines the target; ANY dir_fd call (read or write) fails
# closed -- a string realpath against cwd is wrong for an fd-relative path, and a read-only
# dir_fd open can still read a host file under a directory fd opened outside the workdir
# (d = os.open('/etc', O_RDONLY); os.open('passwd', O_RDONLY, dir_fd=d)).
_WRITE_OFLAGS = (
getattr(_os, "O_WRONLY", 0) | getattr(_os, "O_RDWR", 0)
| getattr(_os, "O_CREAT", 0) | getattr(_os, "O_TRUNC", 0) | getattr(_os, "O_APPEND", 0)
)
def _make_osopen_guard(real_open):
@_gwraps(real_open)
def _guarded(path, flags, *a, **k):
if k.get("dir_fd") is not None:
_deny(path, "os.open (dir_fd)")
try:
fi = int.__index__(flags) # base int: an int-subclass __and__ must not lie
except Exception:
fi = None
mutating = (fi is None) or bool(fi & _WRITE_OFLAGS)
if mutating:
p = _fspath1(path)
if not _within(p):
_deny(p, "os.open write")
return real_open(p, flags, *a, **k)
# Read-only os.open: reads are unconfined, but a host secret is still off limits.
p = _fspath1(path)
_deny_sensitive_read(p)
return real_open(p, flags, *a, **k)
return _guarded
_os.open = _make_osopen_guard(_os.open)
def _wrap1(mod, name, what):
orig = getattr(mod, name, None)
if orig is None:
return
@_gwraps(orig)
def w(*a, **k):
# The path may be positional OR a public keyword: os.mkdir(path=...),
# os.makedirs(name=...), os.removedirs(name=...). Extract it from whichever slot it
# arrived in so a keyword call is not broken (missing positional) while still confined.
_pk = None
if a:
path = a[0]
elif "path" in k:
path, _pk = k["path"], "path"
elif "name" in k:
path, _pk = k["name"], "name"
else:
return orig(*a, **k) # let the original raise its own TypeError
if any(k.get(_f) is not None for _f in ("dir_fd", "src_dir_fd", "dst_dir_fd")):
_deny(path, what + " (dir_fd)") # fd-relative target: a realpath check is meaningless
if isinstance(path, int):
# A mutating op given an fd (os.chmod(fd), os.truncate(fd), ...) can hit a
# file opened read-only outside the workdir; a string realpath cannot
# confine an fd, so deny it (fchmod/fchown are already denied separately).
_deny(path, what + " (fd)")
p = _fspath1(path)
if not _within(p):
_deny(p, what)
# Pass the MATERIALIZED path back in the same slot it arrived (a stateful __fspath__
# cannot then return a different outside path to the real call).
if a:
return orig(p, *a[1:], **k)
k = dict(k)
k[_pk] = p
return orig(*a, **k)
setattr(mod, name, w)
# Path-first single-arg mutators. mkfifo/utime/setxattr/removexattr create or mutate
# host files/metadata; the platform-specific ones no-op via _wrap1 when absent.
_OS_MUTATORS1 = (
"remove", "unlink", "rmdir", "removedirs", "truncate", "chmod", "chown",
"mkdir", "makedirs", "mknod", "mkfifo", "utime", "setxattr", "removexattr",
"lchmod", "lchown", "chflags", "lchflags",
)
for _n in _OS_MUTATORS1:
_wrap1(_os, _n, _n)
# Directory readers (os.listdir / os.scandir) enumerate a directory's names/entries
# WITHOUT routing through open(), so a sensitive host directory whose path is opaque to
# the static scanner (P = globals()['P']; os.listdir(P)) would leak its contents past the
# open-like backstop. Apply the same sensitive-read check to the directory path. A bare
# call (cwd), in-workdir paths, and an fd argument (os.open already screens the fd's read)
# stay allowed.
def _guard_dir_reader(name, mod=_os):
orig = getattr(mod, name, None)
if orig is None:
return
@_gwraps(orig)
def w(path=".", *a, **k):
if isinstance(path, int):
return orig(path, *a, **k)
# Materialize the path ONCE and pass that same value to the real call, so a stateful
# __fspath__ cannot return an in-workdir path for the check and a sensitive outside
# directory for the real listdir/scandir.
p = _fspath1(path)
_deny_sensitive_read(p)
return orig(p, *a, **k)
setattr(mod, name, w)
for _n in ("listdir", "scandir"):
_guard_dir_reader(_n)
def _wrap2(mod, name, both):
orig = getattr(mod, name, None)
if orig is None:
return
@_gwraps(orig)
def w(src, dst, *a, **k):
if any(k.get(_f) is not None for _f in ("dir_fd", "src_dir_fd", "dst_dir_fd")):
_deny(dst, name + " (dir_fd)") # fd-relative target: a realpath check is meaningless
s, d = _fspath1(src), _fspath1(dst)
if both and not _within(s):
_deny(s, name + " source")
if not _within(d):
_deny(d, name + " destination")
return orig(s, d, *a, **k)
setattr(mod, name, w)
for _n in ("rename", "renames", "replace", "link", "symlink"):
_wrap2(_os, _n, True)
# posix (POSIX) / nt (Windows) is the low-level C module os re-exports from; patching
# os.* leaves posix.open / posix.rename / ... importable with the originals, so apply
# the same guards to that module too.
for _lowosname in ("posix", "nt"):
try:
_lowos = __import__(_lowosname)
except Exception:
_lowos = None
if _lowos is not None:
try:
if hasattr(_lowos, "open"):
_lowos.open = _make_osopen_guard(_lowos.open)
for _n in _OS_MUTATORS1:
_wrap1(_lowos, _n, _lowosname + "." + _n)
for _n in ("rename", "renames", "replace", "link", "symlink"):
_wrap2(_lowos, _n, True)
except Exception:
pass
# io.open is a separate reference from the (now patched) builtins.open -- guard
# direct io.open() writers (e.g. zipfile-based) the same way. (Path.open is handled
# explicitly below, not via this patch.)
try:
_io.open = _guard_open_like(_io.open)
except Exception:
pass
# The low-level C module _io is where io.open / builtins.open originate; patching the
# io alias above leaves _io.open untouched, so `import _io; _io.open(p, 'w')` would
# escape. Guard the underlying entry point too.
try:
import _io as _lowio
_lowio.open = _guard_open_like(_lowio.open)
except Exception:
_lowio = None
# io.FileIO / _io.FileIO is a C constructor that opens a file WITHOUT routing through
# open(), so `io.FileIO('/tmp/escape', 'w')` bypasses _guard_open_like. Subclass it to
# confine mutating modes (subclassing keeps guard-built objects real FileIO instances).
def _guard_fileio(_realcls):
class _GuardedFileIO(_realcls):
def __init__(self, name, mode="r", *a, **k):
f = _fspath1(name)
if _mode_is_write(mode):
if not _within(f):
_deny(f, "FileIO write")
else:
_deny_sensitive_read(f)
# Pass the MATERIALIZED path so a stateful __fspath__ cannot return a
# different (outside) path to the real constructor than we checked.
super().__init__(f, mode, *a, **k)
return _GuardedFileIO
for _iomod in (_io, _lowio):
try:
if _iomod is not None and hasattr(_iomod, "FileIO"):
_iomod.FileIO = _guard_fileio(_iomod.FileIO)
except Exception:
pass
# The guards above patch the EXISTING posix / nt / _io module objects, but sandboxed code
# can mint a FRESH copy with the original unwrapped C functions via
# _imp.create_builtin(posix.__spec__) (or the BuiltinImporter path) and call its open()
# directly. Wrap _imp.create_builtin / create_dynamic so a freshly created guard-relevant
# module (posix / nt with os.open-style open + mutators, _io / io with open + FileIO) gets
# the same wrappers re-applied before it is handed back. Other builtin modules carry no file
# primitives, so they pass through unchanged and ordinary lazy imports keep working.
def _reguard_created(m):
try:
_nm = getattr(m, "__name__", "") or ""
except Exception:
return m
try:
if _nm in ("posix", "nt"):
if hasattr(m, "open"):
m.open = _make_osopen_guard(m.open)
for _rn in _OS_MUTATORS1:
_wrap1(m, _rn, _nm + "." + _rn)
for _rn in ("rename", "renames", "replace", "link", "symlink"):
_wrap2(m, _rn, True)
# A fresh posix/nt module also re-exposes the ORIGINAL fd metadata mutators and
# directory readers; reapply the same fd deniers + read confinement applied to the
# already-loaded module (else fresh fchmod(fd, ...) / fresh listdir('/root') slip).
if hasattr(m, "chdir"):
_wrap1(m, "chdir", _nm + ".chdir")
for _rn in ("fchmod", "fchown"):
if hasattr(m, _rn):
setattr(m, _rn, _make_fd_denier(_nm + "." + _rn, getattr(m, _rn)))
for _rn in ("listdir", "scandir"):
if hasattr(m, _rn):
_guard_dir_reader(_rn, m)
elif _nm in ("_io", "io"):
if hasattr(m, "open"):
m.open = _guard_open_like(m.open)
if hasattr(m, "FileIO"):
m.FileIO = _guard_fileio(m.FileIO)
except Exception:
pass
return m
try:
import _imp as _lowimp
def _guard_create(_orig):
@_gwraps(_orig)
def w(spec, *a, **k):
return _reguard_created(_orig(spec, *a, **k))
return w
for _cn in ("create_builtin", "create_dynamic"):
_co = getattr(_lowimp, _cn, None)
if _co is not None:
setattr(_lowimp, _cn, _guard_create(_co))
except Exception:
pass
# Confine the current working directory: os.chdir to a dir outside the workdir would
# let a later relative write/read (which the static read scan treats as local) escape.
# os.fchdir takes an fd whose target we cannot cheaply realpath, so deny it outright.
_wrap1(_os, "chdir", "chdir")
try:
_real_fchdir = _os.fchdir
@_gwraps(_real_fchdir)
def _guarded_fchdir(fd):
_deny(fd, "fchdir")
_os.fchdir = _guarded_fchdir
except Exception:
pass
# fd-based metadata mutators operate on an already-open descriptor, so a read-only
# os.open of an outside file (allowed -- reads are not confined) could still be reused
# to mutate host metadata. Deny them; sandboxed compute has no need to chmod/chown by fd.
def _make_fd_denier(_name, _orig):
@_gwraps(_orig)
def _w(fd, *a, **k):
_deny(fd, _name)
return _w
for _n in ("fchmod", "fchown"):
try:
setattr(_os, _n, _make_fd_denier(_n, getattr(_os, _n)))
except Exception:
pass
# The low-level posix / nt modules re-export chdir / fchdir / fchmod / fchown with the
# ORIGINALS, so patching os.* leaves posix.chdir (cwd escape -> unconfined relative
# reads) and posix.fchmod / posix.fchown (host-metadata mutation on a read-only outside
# fd) reachable. Apply the same confinement / deniers to those module objects too.
for _lowosname in ("posix", "nt"):
try:
_lowos = __import__(_lowosname)
except Exception:
_lowos = None
if _lowos is None:
continue
try:
if hasattr(_lowos, "chdir"):
_wrap1(_lowos, "chdir", _lowosname + ".chdir")
if hasattr(_lowos, "fchdir"):
_lowos.fchdir = _make_fd_denier(_lowosname + ".fchdir", _lowos.fchdir)
for _n in ("fchmod", "fchown"):
if hasattr(_lowos, _n):
setattr(_lowos, _n, _make_fd_denier(_lowosname + "." + _n, getattr(_lowos, _n)))
# posix.listdir / posix.scandir re-export the ORIGINAL enumerators, so the os.* dir
# guard leaves them reachable (posix.listdir('/root')); apply the same sensitive-read
# confinement to the low-level module.
for _n in ("listdir", "scandir"):
if hasattr(_lowos, _n):
_guard_dir_reader(_n, _lowos)
except Exception:
pass
try:
import shutil as _sh
_wrap1(_sh, "rmtree", "rmtree")
_wrap1(_sh, "chown", "chown")
_wrap2(_sh, "move", True)
for _n in ("copy", "copy2", "copyfile", "copytree", "copymode", "copystat"):
_wrap2(_sh, _n, False)
except Exception:
pass
try:
# sqlite3.connect(database) CREATES / opens the DB file via the native _sqlite3 C
# extension, not builtins.open, so the open-like realpath backstop never sees it and an
# absolute / traversal / dynamically built path (sqlite3.connect(os.sep+'tmp/x.db'))
# would write a persistent database outside the session workdir. Confine the database
# path to the workdir at runtime; :memory: / an empty (private temp) / an in-memory URI
# stay allowed. Both public bindings (sqlite3.connect and sqlite3.dbapi2.connect) are the
# same re-exported _sqlite3.connect, so wrap once and reassign every reachable attribute.
import sqlite3 as _sq3
def _sqlite_path_ok(_db, _uri):
if isinstance(_db, str):
if _db == ":memory:" or _db == "":
return True
if _uri and _db[:5].lower() == "file:":
_rest = _db[5:]
_pth, _, _params = _rest.partition("?")
if _pth == ":memory:" or _pth == "" or "mode=memory" in _params.lower():
return True
# file://host/path -> /path (an empty authority is local); a bare file:path
# keeps _pth as-is. The realpath check then confines the concrete file.
if _pth.startswith("//"):
_slash = _pth.find("/", 2)
_pth = _pth[_slash:] if _slash != -1 else ""
# SQLite percent-decodes the URI filename (file:%2Ftmp%2Fx -> /tmp/x), so decode
# BEFORE the workdir check -- otherwise an encoded absolute path passes _within()
# as a relative-looking string while SQLite opens the escaping path. Use the
# captured _bi.chr / _bi.int so a sandboxed rebind of chr/int cannot skew the decode.
_pth = _re.sub(
"%([0-9A-Fa-f]{2})",
lambda _m: _bi.chr(_bi.int(_m.group(1), 16)),
_pth,
)
return _within(_pth)
return _within(_db)
def _guard_sqlite_connect(_orig):
@_gwraps(_orig)
def w(*a, **k):
if a:
_db = a[0]
elif "database" in k:
_db = k["database"]
else:
return _orig(*a, **k) # let sqlite3 raise its own TypeError
_uri = bool(k.get("uri", False))
# Materialize a path-like once so a stateful __fspath__ cannot pass the check
# with an in-workdir value and then hand sqlite a different outside path.
if not isinstance(_db, (str, bytes)):
_db = _fspath1(_db)
if not _sqlite_path_ok(_db, _uri):
_deny(_db, "sqlite3.connect")
if a:
return _orig(_db, *a[1:], **k)
k = dict(k)
k["database"] = _db
return _orig(**k)
return w
_sq3_orig_connect = _sq3.connect
_sq3_guarded_connect = _guard_sqlite_connect(_sq3_orig_connect)
_sq3.connect = _sq3_guarded_connect
try:
_sq3.dbapi2.connect = _sq3_guarded_connect
except Exception:
pass
# The native _sqlite3 C extension still exposes the ORIGINAL connect, and it is importable
# directly (import _sqlite3; _sqlite3.connect('/tmp/escape.db')), bypassing the two Python
# bindings above. Wrap it too so the low-level entry point is confined; module attribute
# assignment on a C extension is allowed, but guard it in case a build disallows it.
try:
import _sqlite3 as _lowsq3
_lowsq3.connect = _guard_sqlite_connect(_lowsq3.connect)
except Exception:
pass
except Exception:
pass
try:
# Path.open("w"): wrap the public method directly (mode-aware). Version-robust
# because pathlib's accessor holds the original io.open (captured at the top).
_real_path_open = _pl.Path.open
@_gwraps(_real_path_open)
def _guarded_path_open(self, mode="r", *a, **k):
# Coerce mode through the base str (a str-subclass __contains__ must not lie).
if _mode_is_write(mode):
if not _within(self):
_deny(str(self), "Path.open")
else:
# A dynamically assembled Path (Path(globals()['P']).read_text()) has no literal
# receiver for the static scanner. On Python <= 3.11 pathlib holds the ORIGINAL
# io.open, so the io.open sensitive-read backstop would not fire for pathlib
# reads; apply it here so Path reads are confined version-robustly. read_text /
# read_bytes route through this same self.open(), so they are covered too.
_deny_sensitive_read(self)
return _real_path_open(self, mode, *a, **k)
_pl.Path.open = _guarded_path_open
def _wrapp(name, targ):
orig = getattr(_pl.Path, name, None)
if orig is None:
return
@_gwraps(orig)
def w(self, *a, **k):
if not _within(self):
_deny(str(self), "Path." + name)
if targ:
# rename/replace/symlink_to/hardlink_to accept the target as the
# `target=` keyword too; on Python <= 3.10 pathlib routes through the
# accessor's ORIGINAL os.rename, so this wrapper is the only
# confinement -- check the keyword as well as the positional arg.
_t = a[0] if a else k.get("target")
if _t is not None:
# Materialize the target once so a stateful __fspath__ cannot return
# an in-workdir path here and an outside one to the real call.
_tm = _fspath1(_t)
if not _within(_tm):
_deny(str(_tm), "Path." + name + " target")
if a:
a = (_tm,) + tuple(a[1:])
else:
k = dict(k)
k["target"] = _tm
return orig(self, *a, **k)
setattr(_pl.Path, name, w)
for _n in ("write_text", "write_bytes", "unlink", "mkdir", "rmdir", "chmod", "touch"):
_wrapp(_n, False)
for _n in ("rename", "replace", "symlink_to", "hardlink_to"):
_wrapp(_n, True)
# Path.iterdir / glob / rglob enumerate a directory; a dynamically built receiver
# (Path(globals()['P']).iterdir(), Path(P).glob('*')) has no literal path for the static
# scanner and, on some CPython versions, routes through pathlib's captured original
# os.scandir rather than the patched one, so screen the directory read on the RECEIVER dir.
def _guard_path_dir_reader(_name):
_real = getattr(_pl.Path, _name, None)
if _real is None:
return
@_gwraps(_real)
def w(self, *a, **k):
_deny_sensitive_read(self)
return _real(self, *a, **k)
setattr(_pl.Path, _name, w)
for _n in ("iterdir", "glob", "rglob"):
_guard_path_dir_reader(_n)
except Exception:
pass
# Gate workdir MODULE imports: user code may `import helper` a sibling .py it wrote, but that
# module's source was never seen by the static analyzer, so a planted workdir/evilmod.py could
# run os.system('cat /etc/passwd') / subprocess at import time in the guarded interpreter (the
# CHILD it spawns is unguarded). Install a meta-path finder that, for a module resolved FROM the
# workdir, parses its source and refuses the import if it reaches a command-execution sink or
# eval/exec/compile the runtime guard cannot confine. File reads/writes in the module are already
# runtime-guarded, and library imports (site-packages) are not workdir-sourced so they pass
# through untouched. (Direct sinks only; deeper obfuscation in a workdir module is an accepted
# residual -- OS isolation remains the real boundary.)
try:
import ast as _gast
import importlib.machinery as _gimach
import importlib.util as _gimportutil
_GUARD_WORKDIR_REAL = _os.path.realpath(__WORKDIR__)
# Network-capable modules: a workdir helper that opens a socket / HTTP client bypasses the
# static network policy (there is no runtime network backstop), so refuse importing one.
# (urllib / http bare tops are left out -- urllib.parse etc. are benign, OS isolation remains
# the boundary for the dotted network submodules.)
_GUARD_NET_MODS = frozenset({
"socket", "ssl", "ftplib", "smtplib", "telnetlib", "poplib", "imaplib", "nntplib",
"requests", "httpx", "aiohttp", "urllib3", "pycurl", "websocket", "websockets", "paramiko",
})
# Network-capable stdlib SUBMODULES whose bare top (urllib / http / xmlrpc) is benign
# (urllib.parse, http.cookies) but whose dotted form opens outbound connections the static
# network policy never saw (urllib.request.urlopen, http.client.HTTPConnection). Matched on
# the full dotted name so the benign siblings stay importable.
_GUARD_NET_DOTTED = frozenset({
"urllib.request", "urllib.robotparser", "http.client", "xmlrpc.client",
})
_GUARD_EXEC_ATTRS = frozenset({
"system", "popen", "popen2", "popen3", "popen4", "startfile",
"execl", "execle", "execlp", "execlpe", "execv", "execve", "execvp", "execvpe",
"spawnl", "spawnle", "spawnlp", "spawnlpe", "spawnv", "spawnve", "spawnvp", "spawnvpe",
"posix_spawn", "posix_spawnp",
})
_GUARD_EXEC_MODS = frozenset({"subprocess", "pty"})
# os / posix expose the exec-attr sinks (os.system, os.execv, ...); a sink attribute rooted at
# one of these is a command-exec sink even without a direct call (x = os.system; x('id')).
_GUARD_EXEC_RECEIVERS = frozenset({"os", "posix"})
# Deserializers that run an attacker-controlled reduce payload (which can spawn an unguarded
# child via posix.system in the reducer): pickle & friends, marshal, dill, cloudpickle,
# jsonpickle. The malicious bytes live OUTSIDE this source, so a workdir helper that calls one
# is refused. (yaml / torch / numpy have safe modes and are left to the top-level analyzer;
# importing pickle for pickle.dumps stays allowed -- only the load sinks are refused.)
_GUARD_DESER_MODS = frozenset(
{"pickle", "_pickle", "cpickle", "marshal", "dill", "cloudpickle", "jsonpickle"}
)
_GUARD_DESER_ATTRS = frozenset({"loads", "load", "Unpickler", "decode"})
# Native-code / dynamic-execution modules: a workdir helper importing one gets UNGUARDED native
# syscalls (ctypes libc write bypassing the patched open/os.open) or runs source / files outside
# the recursive analysis (runpy / code / codeop), so the import is refused too.
_GUARD_NATIVE_MODS = frozenset({"ctypes", "_ctypes", "cffi", "runpy", "code", "codeop"})
# Modules whose DYNAMIC import (importlib.import_module('subprocess')) re-obtains an otherwise
# denied module without a literal `import` statement.
_GUARD_IMPORT_DENIED = (
_GUARD_EXEC_MODS
| _GUARD_NET_MODS
| _GUARD_NATIVE_MODS
| _GUARD_EXEC_RECEIVERS
| {"sys", "builtins", "importlib"}
)
# Introspection / frame gadget attributes that recover a runtime guard wrapper's ORIGINAL
# unguarded callable (open.__closure__[0].cell_contents, frame.f_locals['real']) or walk to
# os / builtins. Mirrors the top-level _GADGET_DUNDERS; refuse them in a workdir helper too.
_GUARD_GADGET_ATTRS = frozenset({
"__subclasses__", "__bases__", "__base__", "__globals__", "__builtins__",
"__closure__", "cell_contents", "f_locals", "f_globals", "f_back", "f_builtins",
"tb_frame", "tb_next", "gi_frame", "cr_frame", "ag_frame",
"settrace", "setprofile", "_getframe", "_current_frames", "currentframe",
})
# sys attributes that reach the import machinery: mutating them removes the guard's import
# vetter so a sibling `import evil` loads unscanned.
_GUARD_IMPORT_MACHINERY = frozenset({"meta_path", "path_hooks", "path_importer_cache"})
def _guard_attr_root(_v):
# Base Name id of an attribute chain (os.path -> 'os'); None if not Name-rooted.
while isinstance(_v, _gast.Attribute):
_v = _v.value
return _v.id if isinstance(_v, _gast.Name) else None
def _guard_str_fold(_n):
# A statically foldable string: a literal or a concatenation of literals ('ev' + 'al').
if isinstance(_n, _gast.Constant) and isinstance(_n.value, str):
return _n.value
if isinstance(_n, _gast.BinOp) and isinstance(_n.op, _gast.Add):
_l = _guard_str_fold(_n.left)
_r = _guard_str_fold(_n.right)
if _l is not None and _r is not None:
return _l + _r
return None
def _guard_subscript_key(_sub):
return _guard_str_fold(_sub.slice)
def _guard_module_src_unsafe(_src):
try:
_tree = _gast.parse(_src)
except _bi.BaseException:
return True # unparseable workdir module -> fail closed
# Pre-pass: record os / posix import ALIASES (import os as o) so an aliased sink reference
# that is only assigned (s = o.system) -- not directly called -- is still recognized.
# Also record builtins aliases (import builtins as b) so the execution builtins reached
# as an attribute (builtins.eval / b.exec) are recognized alongside the bare names.
_recv = set(_GUARD_EXEC_RECEIVERS)
_bi = {"builtins", "__builtins__"}
_deser = set(_GUARD_DESER_MODS)
_sysmod = {"sys"}
_implib = {"importlib"}
for _nd in _gast.walk(_tree):
if isinstance(_nd, _gast.Import):
for _al in _nd.names:
if _al.name in ("os", "posix"):
_recv.add(_al.asname or _al.name)
elif _al.name == "builtins":
_bi.add(_al.asname or _al.name)
elif _al.name in _GUARD_DESER_MODS:
_deser.add(_al.asname or _al.name)
elif _al.name == "sys":
_sysmod.add(_al.asname or _al.name)
elif _al.name == "importlib":
_implib.add(_al.asname or _al.name)
# Modules whose dynamic attribute / namespace-dict access (getattr / vars) is obfuscation.
_obf = _recv | _bi | _deser | _sysmod | _implib
for _nd in _gast.walk(_tree):
if isinstance(_nd, _gast.Import):
for _al in _nd.names:
_top = _al.name.split(".")[0]
if (
_top in _GUARD_EXEC_MODS
or _top in _GUARD_NET_MODS
or _top in _GUARD_NATIVE_MODS
):
return True
# import urllib.request / import http.client -- benign top, network submodule.
if _al.name in _GUARD_NET_DOTTED:
return True
elif isinstance(_nd, _gast.ImportFrom):
_mod = _nd.module or ""
_mroot = _mod.split(".")[0]
if (
_mroot in _GUARD_EXEC_MODS
or _mroot in _GUARD_NET_MODS
or _mroot in _GUARD_NATIVE_MODS
):
return True
# from urllib.request import urlopen -- the module itself is a network submodule.
if _mod in _GUARD_NET_DOTTED:
return True
# from urllib import request / from http import client -- the network submodule is
# bound by name, so the dotted target is (package + . + imported name).
for _al in _nd.names:
if (_mod + "." + _al.name) in _GUARD_NET_DOTTED:
return True
# `from os import system` / `from os import *` binds a BARE sink name into the
# module namespace; a later bare system('id') call has no os. attribute to catch.
if _mroot in _GUARD_EXEC_RECEIVERS:
for _al in _nd.names:
if _al.name == "*" or _al.name in _GUARD_EXEC_ATTRS:
return True
# `from pickle import loads` / `from pickle import *` binds a bare deserializer
# sink; a later bare loads(evil) has no module attribute to catch. (dumps stays
# allowed -- only the load sinks are refused.)
if _mroot in _GUARD_DESER_MODS:
for _al in _nd.names:
if _al.name == "*" or _al.name in _GUARD_DESER_ATTRS:
return True
elif isinstance(_nd, _gast.Call):
# An ACTUAL invocation of a sink-named method on any receiver (os.system(...),
# or an aliased o.system(...)) is a command-exec call regardless of receiver.
if isinstance(_nd.func, _gast.Attribute) and _nd.func.attr in _GUARD_EXEC_ATTRS:
return True
if isinstance(_nd.func, _gast.Name) and _nd.func.id in (
"eval", "exec", "compile", "__import__"):
return True
# builtins.eval(...) / b.exec(...) -- the execution builtins reached as an
# attribute of the builtins module (or an alias). Require a builtins root so a
# benign .compile()/.eval() on some other object (model.compile, df.eval) is
# not misread as a sink.
if (
isinstance(_nd.func, _gast.Attribute)
and _nd.func.attr in ("eval", "exec", "compile", "__import__")
and _guard_attr_root(_nd.func.value) in _bi
):
return True
# A deserializer call (pickle.loads / marshal.load / dill.load /
# pickle.Unpickler(f) / jsonpickle.decode) runs an attacker-controlled reduce
# payload whose bytes live outside this source, so refuse it -- rooted at a
# deserializer module / alias so a benign json.load / config.load is untouched.
if (
isinstance(_nd.func, _gast.Attribute)
and _nd.func.attr in _GUARD_DESER_ATTRS
and _guard_attr_root(_nd.func.value) in _deser
):
return True
# importlib.import_module('subprocess') / importlib.reload(subprocess) dynamically
# re-obtain a denied module without a literal `import`. Refuse when the target is a
# denied module (constant name or module Name); a dynamic import_module target
# (non-constant) fails closed.
if (
isinstance(_nd.func, _gast.Attribute)
and _nd.func.attr in ("import_module", "reload")
and _guard_attr_root(_nd.func.value) in _implib
and _nd.args
):
_a0 = _nd.args[0]
if isinstance(_a0, _gast.Constant) and isinstance(_a0.value, str):
if _a0.value.split(".")[0] in _GUARD_IMPORT_DENIED:
return True
elif isinstance(_a0, _gast.Name) and _a0.id in _GUARD_IMPORT_DENIED:
return True
elif _nd.func.attr == "import_module":
return True # dynamic import target -> fail closed
# getattr(os, 'system')(...) / getattr(sys, 'meta_path') / vars(sys)['meta_path']
# -- dynamic attribute / namespace-dict access is the obfuscated twin of the direct
# sink (the name-based checks above never see it). A constant sink name on a sink
# receiver is refused; a NON-constant name on such a receiver, and vars() of one,
# are refused too (the attribute cannot be proven benign).
if (
isinstance(_nd.func, _gast.Name)
and _nd.func.id == "getattr"
and len(_nd.args) >= 2
and isinstance(_nd.args[0], (_gast.Name, _gast.Attribute))
):
_grecv = _guard_attr_root(_nd.args[0])
_gname = (
_nd.args[1].value
if isinstance(_nd.args[1], _gast.Constant)
and isinstance(_nd.args[1].value, str)
else None
)
if _gname is None:
if _grecv in _obf:
return True
else:
# An introspection / frame gadget dunder via getattr reaches an escape on
# ANY receiver -- getattr(open, '__closure__'), getattr(cell,
# 'cell_contents') recover the guard wrapper's original unguarded open --
# so reject the gadget name regardless of receiver (mirrors the direct
# attribute check below).
if _gname in _GUARD_GADGET_ATTRS:
return True
if _grecv in _recv and _gname in _GUARD_EXEC_ATTRS:
return True
if _grecv in _bi and _gname in ("eval", "exec", "compile", "__import__"):
return True
if _grecv in _deser and _gname in _GUARD_DESER_ATTRS:
return True
if _grecv in _sysmod and _gname in _GUARD_IMPORT_MACHINERY:
return True
if _grecv in _implib and _gname in (
"import_module", "reload", "__import__"):
return True
# vars(sys) / vars(os) / vars(builtins) exposes the module namespace dict for
# indirect access (vars(sys)['meta_path'][:] = [...], vars(os)['system']).
if (
isinstance(_nd.func, _gast.Name)
and _nd.func.id == "vars"
and len(_nd.args) == 1
and isinstance(_nd.args[0], (_gast.Name, _gast.Attribute))
and _guard_attr_root(_nd.args[0]) in _obf
):
return True
elif isinstance(_nd, _gast.Subscript):
# __builtins__['eval'] / builtins.__dict__['exec'] -- imported helpers run with
# __builtins__ as a dict, so subscript access reaches the execution builtins the
# attribute checks miss. A constant exec-builtin key is refused; a NON-constant key
# on a builtins receiver fails closed.
_sroot = _guard_attr_root(_nd.value)
if _sroot in _bi:
_skey = _guard_subscript_key(_nd)
if _skey is None:
return True
if _skey in ("eval", "exec", "compile", "__import__"):
return True
elif isinstance(_nd, _gast.Attribute):
# An introspection / frame gadget attribute (open.__closure__[0].cell_contents,
# frame.f_locals['real'], ().__class__.__bases__[0].__subclasses__()) recovers a
# runtime guard wrapper's original unguarded callable or walks to os / builtins.
# These reach an escape on ANY receiver, so flag the attribute itself.
if _nd.attr in _GUARD_GADGET_ATTRS:
return True
# A sink-named attribute REFERENCE (even uncalled) rooted at os / posix / an
# os alias (x = os.system, s = o.system). A same-named attribute on an unrelated
# object (p.system = 'linux') is NOT a sink, so require a sink-module receiver.
if _nd.attr in _GUARD_EXEC_ATTRS and _guard_attr_root(_nd.value) in _recv:
return True
# A builtins-rooted execution-builtin REFERENCE (e = builtins.eval; e(...)),
# even uncalled, is the same sink as calling it directly.
if (
_nd.attr in ("eval", "exec", "compile", "__import__")
and _guard_attr_root(_nd.value) in _bi
):
return True
# A workdir module that touches the import machinery (sys.meta_path /
# sys.path_hooks / sys.path_importer_cache) can remove THIS vetter, then a
# sibling `import evil` loads unscanned. The top-level analyzer blocks such
# mutation in submitted code; refuse it inside a vetted workdir module too.
if _nd.attr in ("meta_path", "path_hooks", "path_importer_cache"):
return True
return False
def _guard_under_workdir(_p):
return _p == _GUARD_WORKDIR_REAL or _p.startswith(_GUARD_WORKDIR_REAL + _os.sep)
class _GuardVettedSourceLoader:
# Executes the EXACT source string the vetter scanned, so the loader can never satisfy
# the import from a planted bytecode cache (.pyc) or re-decode the file differently than
# it was vetted. __path__ for a package still comes from the spec's search locations.
def __init__(self, _name, _path, _src, _is_pkg):
self._n = _name
self._p = _path
self._s = _src
self._pkg = _is_pkg
def create_module(self, _spec):
return None
def exec_module(self, _module):
exec(compile(self._s, self._p, "exec"), _module.__dict__)
def get_filename(self, _name=None):
return self._p
def is_package(self, _name=None):
return self._pkg
def get_source(self, _name=None):
return self._s
class _GuardWorkdirImportVetter:
def find_spec(self, _name, _path=None, _target=None):
try:
_spec = _gimach.PathFinder.find_spec(_name, _path, _target)
except _bi.BaseException:
return None
_orig = getattr(_spec, "origin", None) if _spec is not None else None
if not _orig:
return None # namespace / builtin / frozen: no file to vet, not workdir-sourced
try:
_abs = _os.path.abspath(_orig)
_rp = _os.path.realpath(_orig)
except _bi.BaseException:
return None
_orig_in_wd = _guard_under_workdir(_abs) or _guard_under_workdir(
_os.path.dirname(_abs)
)
_rp_in_wd = _guard_under_workdir(_rp)
if not _orig_in_wd and not _rp_in_wd:
return None # genuinely not a workdir module; let the default finders load it
# A workdir-sourced origin whose REALPATH escapes the workdir (a symlink to an outside
# file) must fail closed, not be handed to the default loader unvetted.
if not _rp_in_wd:
raise _bi.ImportError(
"sandbox: refusing symlinked workdir module " + _name)
# A workdir module must be a .py we can read + scan. A sourceless .pyc / native .so /
# any other non-source file under the workdir cannot be statically vetted, so refuse
# it: a planted legacy evil.pyc would otherwise run its bytecode via the default
# sourceless loader, never reaching the source scan below.
if not _orig.endswith(".py"):
raise _bi.ImportError(
"sandbox: refusing to import non-source workdir module " + _name)
# Decode with Python's PEP 263 source encoding (importlib.util.decode_source), NOT a
# fixed utf-8: a `# coding: utf_7` module the loader would decode as UTF-7 must be
# vetted as UTF-7, or a payload hidden in what a UTF-8 scan sees as a comment runs.
try:
_fb = _io.open(_orig, "rb")
try:
_raw = _fb.read()
finally:
_fb.close()
_msrc = _gimportutil.decode_source(_raw)
except _bi.BaseException:
raise _bi.ImportError("sandbox: cannot vet workdir module " + _name)
if _guard_module_src_unsafe(_msrc):
raise _bi.ImportError(
"sandbox: refusing to import unvetted workdir module " + _name)
# Run the EXACT vetted source via our loader so a planted matching .pyc can never be
# executed instead (the default SourceFileLoader would satisfy the import from a
# __pycache__ .pyc whose header matches the harmless source).
_is_pkg = _spec.submodule_search_locations is not None or _os.path.basename(
_orig
) == "__init__.py"
_spec.loader = _GuardVettedSourceLoader(_name, _orig, _msrc, _is_pkg)
return _spec
_sys.meta_path.insert(0, _GuardWorkdirImportVetter())
except _bi.BaseException:
pass
# All guard dependencies are now imported (and cached as the real, patched modules) with the
# workdir kept off sys.path, so no workdir/*.py could shadow them. Restore the original path
# for user code so ordinary sibling imports still resolve (workdir modules are vetted above).
_sys.path = _saved_path
"""
def _sandbox_runtime_prelude(workdir: str) -> str:
"""One physical line that runs the realpath backstop before the user code.
The guard executes in its own namespace (helper names never leak into user
globals) while its monkeypatches persist on the os/shutil/pathlib/builtins
module objects. Emitting it on a single line keeps user traceback line numbers
shifted by exactly one."""
src = _SANDBOX_GUARD_SRC.replace("__WORKDIR__", repr(workdir))
return (
"exec(compile(%r, '<studio-sandbox-guard>', 'exec'), {'__builtins__': __builtins__})\n"
% src
)
def _inject_sandbox_guard(code: str, prelude: str) -> str:
"""Splice the runtime guard into ``code`` without displacing leading directives.
``from __future__`` imports must be the first statement of a module (only a
docstring and comments may precede them), so blindly prepending the guard line
turns any user program that opens with a future import into a SyntaxError. Parse
the code, keep a leading module docstring and any ``from __future__`` imports on
top, and insert the (inert compile-time) guard immediately after them -- it still
runs before the first real user statement, so the sandbox is established before
any file operation. Everything else (including unparsable code, where we want the
natural SyntaxError traceback) falls back to a plain prepend.
"""
try:
tree = ast.parse(code)
except SyntaxError:
return prelude + code
body = getattr(tree, "body", [])
idx = 0
split = 0
if (
body
and isinstance(body[0], ast.Expr)
and isinstance(getattr(body[0], "value", None), ast.Constant)
and isinstance(body[0].value.value, str)
):
idx = 1
split = body[0].end_lineno or 0
has_future = False
while (
idx < len(body)
and isinstance(body[idx], ast.ImportFrom)
and body[idx].module == "__future__"
):
has_future = True
split = body[idx].end_lineno or split
idx += 1
if not has_future or split <= 0:
return prelude + code
# Split at the last head statement's exact END COLUMN, not the whole physical line: a
# `from __future__ import annotations; open('/tmp/x','w')` puts a real statement on the SAME
# line as the future import, and a line-granular split would copy that write into the head
# (before the guard prelude) and run it unguarded. Slice the head line at end_col_offset so the
# `;`-separated tail moves AFTER the prelude, then drop the leading `; ` so the tail is a valid
# statement. (Only future-import lines reach here -- pure ASCII -- so character slicing matches
# the byte col_offset.)
_last = body[idx - 1]
_el = _last.end_lineno or split
_ec = _last.end_col_offset or 0
lines = code.splitlines(keepends = True)
head = "".join(lines[: _el - 1]) + lines[_el - 1][:_ec]
tail = lines[_el - 1][_ec:] + "".join(lines[_el:])
_m = re.match(r"[ \t]*;[ \t]*", tail)
if _m:
tail = tail[_m.end() :] # a same-line `; stmt` tail -> valid statement after the prelude
if head and not head.endswith(("\n", "\r")):
head += "\n"
if prelude and not prelude.endswith(("\n", "\r")):
prelude += "\n"
return head + prelude + tail
def _python_exec(
code: str,
cancel_event = None,
timeout: int = _EXEC_TIMEOUT,
session_id: str | None = None,
disable_sandbox: bool = False,
) -> str:
"""Execute Python code in a subprocess sandbox.
disable_sandbox (Bypass Permissions): skip the safety analysis and rlimit
pre-exec, and use the host env minus secrets.
"""
if not code or not code.strip():
return "No code provided."
# Validate imports and code safety (skipped when the sandbox is disabled)
if not disable_sandbox:
error = _check_code_safety(code)
if error:
return error
elif not _harden_parent_against_proc_env_leak():
# Close the /proc/<parent>/environ secret-recovery path first; if it
# cannot be applied, fail closed rather than leak the parent environ.
return (
"Execution error: could not harden the Studio process against "
"/proc environment reads; refusing bypass execution."
)
tmp_path = None
workdir = _get_workdir(session_id)
# Snapshot image mtimes to detect new and overwritten files.
_before: dict[str, int] = {}
if os.path.isdir(workdir):
for _name in os.listdir(workdir):
if os.path.splitext(_name)[1].lower() in _IMAGE_EXTS:
_p = os.path.join(workdir, _name)
if os.path.isfile(_p):
try:
_before[_name] = os.stat(_p).st_mtime_ns
except OSError:
pass
try:
fd, tmp_path = tempfile.mkstemp(suffix = ".py", prefix = "studio_exec_", dir = workdir)
# utf-8 so non-ASCII in model-written code survives the OS default codec
# (Windows cp1252 would otherwise raise UnicodeEncodeError).
# Sandboxed runs get the realpath backstop prepended (Stage 5); bypass
# runs execute the code verbatim.
file_body = (
code
if disable_sandbox
else _inject_sandbox_guard(code, _sandbox_runtime_prelude(workdir))
)
with os.fdopen(fd, "w", encoding = "utf-8") as f:
f.write(file_body)
safe_env = _build_bypass_env(workdir) if disable_sandbox else _build_safe_env(workdir)
if disable_sandbox:
# Match the sandboxed Python path without changing bypass shell I/O.
safe_env = dict(safe_env)
safe_env["PYTHONIOENCODING"] = "utf-8"
popen_kwargs = dict(
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
# Decode child output as utf-8 (it emits utf-8 via PYTHONIOENCODING);
# replace so non-ASCII output never crashes the read on Windows.
encoding = "utf-8",
errors = "replace",
cwd = workdir,
env = safe_env,
)
if sys.platform != "win32":
popen_kwargs["preexec_fn"] = _bypass_preexec if disable_sandbox else _sandbox_preexec
else:
popen_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
# -s disables the per-user site directory (~/.local/.../site-packages): the guard is
# injected into the SCRIPT body and runs after site initialization, so a prior run that
# dropped .local/.../site-packages/usercustomize.py (HOME points at the workdir) would
# otherwise import it during startup and run writers with unpatched stdlib. Bypass keeps
# the host default. The real site-packages stays available for user imports.
_py_argv = (
[sys.executable, tmp_path] if disable_sandbox else [sys.executable, "-s", tmp_path]
)
proc = subprocess.Popen(_py_argv, **popen_kwargs)
# Spawn cancel watcher if we have a cancel event
if cancel_event is not None:
watcher = threading.Thread(
target = _cancel_watcher, args = (proc, cancel_event), daemon = True
)
watcher.start()
try:
output, _ = proc.communicate(timeout = timeout)
except subprocess.TimeoutExpired:
_kill_process_tree(proc)
try:
proc.communicate(timeout = 5)
except subprocess.TimeoutExpired:
pass
return _truncate(f"Execution timed out after {timeout} seconds.")
if cancel_event is not None and cancel_event.is_set():
return "Execution cancelled."
result = output or ""
if proc.returncode != 0:
result = f"Exit code {proc.returncode}:\n{result}"
result = _truncate(result) if result.strip() else "(no output)"
# Detect new/overwritten images and append sentinel for the frontend
if session_id and os.path.isdir(workdir):
new_images = []
for _name in os.listdir(workdir):
if os.path.splitext(_name)[1].lower() not in _IMAGE_EXTS:
continue
_p = os.path.join(workdir, _name)
if not os.path.isfile(_p):
continue
try:
_mtime = os.stat(_p).st_mtime_ns
except OSError:
continue
if _name not in _before or _mtime != _before[_name]:
new_images.append(_name)
if new_images:
import json as _json
result += f"\n__IMAGES__:{_json.dumps(sorted(new_images))}"
return result
except Exception as e:
return f"Execution error: {e}"
finally:
if tmp_path and os.path.exists(tmp_path):
try:
os.unlink(tmp_path)
except OSError:
pass
def _bash_exec(
command: str,
cancel_event = None,
timeout: int = _EXEC_TIMEOUT,
session_id: str | None = None,
disable_sandbox: bool = False,
) -> str:
"""Execute a bash command in a subprocess sandbox.
disable_sandbox (Bypass Permissions): skip the command blocklist and rlimit
pre-exec, and use the host env minus secrets.
"""
if not command or not command.strip():
return "No command provided."
# Block dangerous commands (skipped when the sandbox is disabled)
if not disable_sandbox:
blocked = _find_blocked_commands(command)
if blocked:
return f"Blocked command(s) for safety: {', '.join(sorted(blocked))}"
# The command runs in an unguarded shell child, so the Python-tool open() backstop
# does not confine its reads; refuse an embedded host-secret read the same way an
# os.system('cat /etc/passwd') shell string is refused in the Python tool.
_read = _command_reads_sensitive(command)
if _read is not None:
return f"Blocked command for safety: sensitive file read ({_read})"
elif not _harden_parent_against_proc_env_leak():
# Close the /proc/<parent>/environ secret-recovery path first; if it
# cannot be applied, fail closed rather than leak the parent environ.
return (
"Execution error: could not harden the Studio process against "
"/proc environment reads; refusing bypass execution."
)
try:
workdir = _get_workdir(session_id)
safe_env = _build_bypass_env(workdir) if disable_sandbox else _build_safe_env(workdir)
popen_kwargs = dict(
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
cwd = workdir,
env = safe_env,
)
if sys.platform != "win32":
popen_kwargs["preexec_fn"] = _bypass_preexec if disable_sandbox else _sandbox_preexec
else:
popen_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
proc = subprocess.Popen(_get_shell_cmd(command), **popen_kwargs)
if cancel_event is not None:
watcher = threading.Thread(
target = _cancel_watcher, args = (proc, cancel_event), daemon = True
)
watcher.start()
try:
output, _ = proc.communicate(timeout = timeout)
except subprocess.TimeoutExpired:
_kill_process_tree(proc)
try:
proc.communicate(timeout = 5)
except subprocess.TimeoutExpired:
pass
return _truncate(f"Execution timed out after {timeout} seconds.")
if cancel_event is not None and cancel_event.is_set():
return "Execution cancelled."
result = output or ""
if proc.returncode != 0:
result = f"Exit code {proc.returncode}:\n{result}"
return _truncate(result) if result.strip() else "(no output)"
except Exception as e:
return f"Execution error: {e}"