8be5765b04
- Scan command substitutions regardless of the outer command and quoting: bash runs $(...) /
backtick payloads even inside double quotes, so echo "$(head -1 /etc/passwd)" read the file
while the scanner only inspected reader-command arguments. Extract every substitution payload
(new _extract_command_subs) and recurse the read scanner into it.
- Normalize .. before the system-bin exemption: _is_local_executable_path trusted any /usr/bin/
prefix, so /usr/bin/../../<workdir>/evil escaped the local-exec check and ran an unguarded
shebang. normpath the command path first (the leading-slash check on the original token keeps
./evil local).
- Block flock as a command-executing wrapper: flock [opts] <file>|<fd> <command> (or flock -c
<command>) runs an arbitrary command in an unguarded child; its file/fd operand + -c forms make
the command word hard to resolve, so block the wrapper outright.
- Treat coproc as a command-position keyword: coproc [NAME] command runs COMMAND asynchronously,
so coproc touch /tmp/escape must resolve touch as the command; add coproc to the keyword set.
- Scan trap handler payloads: trap 'CMD' SIGSPEC runs CMD in the unguarded shell on EXIT / a
signal, so recurse the blocked-command and sensitive-read scanners into the handler operand; a
reset (trap - EXIT) / ignore (trap '' EXIT) has nothing to run.
- Guard the low-level posix / nt directory readers: posix.listdir / posix.scandir re-export the
ORIGINAL enumerators, so an opaque sensitive path (posix.listdir('/root')) slipped past the
os.* dir guard; apply the same sensitive-read confinement to the low-level modules (via a
module-parametrized _guard_dir_reader).
- Reapply fd deniers + dir-reader guards to a freshly created posix / nt module: _reguard_created
only rewrapped open + path mutators, so a fresh module's fchmod / fchown (host-metadata mutation
on a read-only outside fd) and listdir / scandir were unguarded; reapply them too.
Adds TestRound33Bypasses plus runtime posix dir-reader / fresh-module fd-denier tests.
9316 lines
404 KiB
Python
9316 lines
404 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-only
|
|
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
|
|
|
"""Tool definitions and executors for LLM tool calling: web search
|
|
(DuckDuckGo), Python code execution, and terminal commands."""
|
|
|
|
import ast
|
|
import http.client
|
|
import os
|
|
import signal
|
|
|
|
os.environ["UNSLOTH_IS_PRESENT"] = "1"
|
|
|
|
import asyncio
|
|
import base64
|
|
import binascii
|
|
import codecs
|
|
import random
|
|
import re
|
|
import shlex
|
|
import ssl
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import urllib.request
|
|
import zlib
|
|
|
|
from core.inference.mcp_client import (
|
|
MCP_TOOL_PREFIX,
|
|
TOOL_CACHE_INVALIDATING_FIELDS,
|
|
cache_tools,
|
|
call_tool_sync,
|
|
get_cached_tools,
|
|
in_failure_cooloff,
|
|
is_stdio,
|
|
list_tools_async,
|
|
parse_server_headers,
|
|
probe_timeout,
|
|
record_probe_failure,
|
|
stdio_mcp_enabled,
|
|
)
|
|
from storage import mcp_servers_db
|
|
|
|
from loggers import get_logger
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
_EXEC_TIMEOUT = 300 # 5 minutes
|
|
|
|
# Splits the UI source-map from the result; loops strip it (like __IMAGES__).
|
|
RAG_SOURCES_SENTINEL = "\n__RAG_SOURCES__:"
|
|
|
|
# Import these at module level so the preexec_fn closure triggers no imports in
|
|
# the forked child (which can deadlock multi-threaded servers).
|
|
_libc = None
|
|
if sys.platform == "linux":
|
|
try:
|
|
import ctypes
|
|
import ctypes.util
|
|
|
|
_libc_name = ctypes.util.find_library("c")
|
|
if _libc_name:
|
|
_libc = ctypes.CDLL(_libc_name, use_errno = True)
|
|
except (OSError, AttributeError):
|
|
pass
|
|
|
|
_resource = None
|
|
if sys.platform != "win32":
|
|
try:
|
|
import resource as _resource
|
|
except ImportError:
|
|
pass
|
|
|
|
# Raster-image allowlist for sandbox file serving.
|
|
# No .svg (XSS via embedded scripts), no .html, no .pdf.
|
|
_IMAGE_EXTS = frozenset({".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp"})
|
|
_MAX_OUTPUT_CHARS = 8000 # truncate long output
|
|
_BLOCKED_COMMANDS_COMMON = frozenset(
|
|
{
|
|
"rm",
|
|
"dd",
|
|
"chmod",
|
|
"chown",
|
|
"mkfs",
|
|
"mount",
|
|
"umount",
|
|
"fdisk",
|
|
"sudo",
|
|
"su",
|
|
"doas",
|
|
"pkexec",
|
|
"shutdown",
|
|
"reboot",
|
|
"halt",
|
|
"poweroff",
|
|
"kill",
|
|
"killall",
|
|
"pkill",
|
|
"passwd",
|
|
"curl",
|
|
"wget",
|
|
"nc",
|
|
"ncat",
|
|
"netcat",
|
|
"socat",
|
|
"ssh",
|
|
"scp",
|
|
"sftp",
|
|
"rsync",
|
|
"eval",
|
|
"source",
|
|
"ln",
|
|
# flock [options] <file>|<fd> <command> (or flock -c <command>) runs an arbitrary
|
|
# command in an unguarded child while holding a lock; its file/fd operand + -c forms
|
|
# make the command word hard to resolve, so block the wrapper outright.
|
|
"flock",
|
|
}
|
|
)
|
|
# Language interpreters that run inline / file / stdin code in a FRESH child process.
|
|
# The runtime filesystem backstop only patches the current interpreter, so a spawned
|
|
# `python -c '...'`, `perl -e '...'`, `node -e '...'`, etc. runs with none of the guard
|
|
# monkeypatches and can write/delete outside the session workdir. Blocking the
|
|
# interpreter at shell command position closes that child-process escape; the sandbox's
|
|
# own python_execute tool is the supported way to run Python (it IS guarded). Argument
|
|
# position (`echo python`, `ls /usr/bin/python3`) is unaffected by the command-position
|
|
# scanner.
|
|
_INTERPRETER_COMMANDS = frozenset(
|
|
{
|
|
"python",
|
|
"python2",
|
|
"python3",
|
|
"pythonw",
|
|
"perl",
|
|
"ruby",
|
|
"node",
|
|
"nodejs",
|
|
"php",
|
|
"deno",
|
|
"lua",
|
|
"luajit",
|
|
"rscript",
|
|
# awk variants run an inline program that can write files (print > "/path")
|
|
# in an unguarded child without any shell redirection token the scanner sees.
|
|
"awk",
|
|
"gawk",
|
|
"mawk",
|
|
"nawk",
|
|
}
|
|
)
|
|
# File-creating / writing coreutils. Same rationale as the interpreters: a spawned child
|
|
# runs without the in-process realpath backstop, so subprocess.run(['touch', '/tmp/x']),
|
|
# tee, cp, mv, ... write / create / delete outside the session workdir. In-workdir file
|
|
# work should go through the guarded Python file APIs. (dd / ln / rm are already denied
|
|
# above.) Native / unknown binaries the sandbox cannot enumerate remain an OS-isolation
|
|
# residual.
|
|
_CHILD_WRITE_COMMANDS = frozenset(
|
|
{
|
|
"touch",
|
|
"tee",
|
|
"cp",
|
|
"mv",
|
|
"mkdir",
|
|
"install",
|
|
"truncate",
|
|
"mkfifo",
|
|
"mknod",
|
|
"shred",
|
|
"unlink",
|
|
# rmdir removes (empty) directories; a bash child gets no realpath guard, so
|
|
# rmdir /tmp/some-empty-dir deletes a host directory outside the workdir.
|
|
"rmdir",
|
|
# split / csplit slice a file into PREFIXaa, PREFIXab, ... at an arbitrary prefix
|
|
# path, creating files outside the workdir in an unguarded child.
|
|
"split",
|
|
"csplit",
|
|
# Archive / compression tools create files in an unguarded child (tar -cf out,
|
|
# zip out, unzip extracts, gzip file). In-workdir archiving should go through the
|
|
# guarded Python APIs.
|
|
"tar",
|
|
"zip",
|
|
"unzip",
|
|
"gzip",
|
|
"gunzip",
|
|
"bzip2",
|
|
"bunzip2",
|
|
"xz",
|
|
"unxz",
|
|
"zstd",
|
|
"7z",
|
|
"7za",
|
|
"rar",
|
|
"unrar",
|
|
"cpio",
|
|
"rsync",
|
|
# mktemp creates a file / dir at a caller-chosen template path (mktemp
|
|
# /tmp/x.XXXXXX, mktemp -d), writing outside the workdir in an unguarded child.
|
|
"mktemp",
|
|
}
|
|
)
|
|
_BLOCKED_COMMANDS_COMMON = _BLOCKED_COMMANDS_COMMON | _INTERPRETER_COMMANDS | _CHILD_WRITE_COMMANDS
|
|
_BLOCKED_COMMANDS_WIN = frozenset(
|
|
{
|
|
"rmdir",
|
|
"takeown",
|
|
"icacls",
|
|
"runas",
|
|
"powershell",
|
|
"pwsh",
|
|
}
|
|
)
|
|
_BLOCKED_COMMANDS = (
|
|
_BLOCKED_COMMANDS_COMMON | _BLOCKED_COMMANDS_WIN
|
|
if sys.platform == "win32"
|
|
else _BLOCKED_COMMANDS_COMMON
|
|
)
|
|
|
|
|
|
_SHELL_SEPARATORS = frozenset({";", "&&", "||", "|", "&", "\n", "(", ")", "`", "{", "}"})
|
|
# Bash keywords whose FOLLOWING word is a new command position: the compound-statement
|
|
# headers (if / while / until / elif run their CONDITION command) and the body markers
|
|
# (then / do / else). `if touch x; then :; fi` executes `touch` as the condition command, so
|
|
# these must reset command position -- otherwise the header word is mistaken for the command
|
|
# and the real command it precedes is skipped as an argument.
|
|
_SHELL_KEYWORDS_AS_SEP = frozenset({"then", "do", "else", "elif", "if", "while", "until", "coproc"})
|
|
# POSIX / common shell binaries. A shell without an inline `-c` payload runs unscanned
|
|
# code (a script file, -s / stdin, or a bare stdin-reading shell), so it is denied.
|
|
_SHELL_BINARIES = frozenset({"bash", "sh", "zsh", "dash", "ksh", "csh", "tcsh", "fish"})
|
|
# Utilities whose LATER argv elements are actions / write flags, not inert arguments
|
|
# (find -exec/-delete, sed -i / w, sort -o). A non-shell argv resolving to one of these is
|
|
# re-scanned as a reconstructed command line so those dangerous flags are caught.
|
|
_ARGV_TAIL_SCAN_COMMANDS = frozenset({"find", "sed", "gsed", "ssed", "perl", "sort"})
|
|
|
|
|
|
def _is_versioned_interpreter(base: str) -> bool:
|
|
"""True when ``base`` is a version-suffixed interpreter name (python3.14, python3.11,
|
|
perl5.36, ruby3.0) whose unversioned stem is a blocked interpreter. Those binaries are
|
|
commonly on the sandbox PATH and start the same unguarded child as the bare name."""
|
|
stem = re.sub(r"[0-9][0-9.]*$", "", base)
|
|
return stem != base and stem in _INTERPRETER_COMMANDS
|
|
|
|
|
|
# Absolute paths under a standard system bin dir are trusted as real system commands (their
|
|
# basename is still interpreter-checked separately); every OTHER explicit path is a local file.
|
|
_SYSTEM_BIN_PREFIXES = ("/bin/", "/usr/bin/", "/usr/local/bin/", "/sbin/", "/usr/sbin/")
|
|
|
|
|
|
def _is_local_executable_path(tok: str) -> bool:
|
|
"""True when a command word is an explicit path to a LOCAL executable file (./evil, ../x,
|
|
subdir/tool, /tmp/x). Running such a file executes whatever its shebang names in an
|
|
UNGUARDED child -- a sandboxed snippet can create + chmod ./evil with `#!/usr/bin/python3`
|
|
and run it, starting an interpreter the argv basename scan never sees. A bare command name
|
|
resolved via PATH (no slash) and an absolute system-bin path are not treated as local."""
|
|
t = tok.replace("\\", "/")
|
|
if "/" not in t:
|
|
return False
|
|
# Collapse .. before the system-bin exemption so a workdir shebang cannot masquerade as a
|
|
# trusted binary via /usr/bin/../../<workdir>/evil (normpath -> /<workdir>/evil, not exempt).
|
|
# normpath keeps the leading ./ -> bare-name collapse harmless: the "/" check above already
|
|
# ran on the original token, so ./evil (has a slash) still reaches here and stays local.
|
|
norm = os.path.normpath(t)
|
|
return not norm.startswith(_SYSTEM_BIN_PREFIXES)
|
|
|
|
|
|
# The only shell redirection targets trusted without a realpath check: standard device
|
|
# sinks that cannot escape the workdir. Every other target (relative or absolute) fails
|
|
# closed, because the unguarded child follows symlinks and resolves relative names against a
|
|
# cwd the static scanner cannot verify (a pre-existing `out -> /tmp/host` symlink escapes).
|
|
_SAFE_REDIRECT_TARGETS = frozenset(
|
|
{"/dev/null", "/dev/zero", "/dev/full", "/dev/stdout", "/dev/stderr", "/dev/tty"}
|
|
)
|
|
# Coreutils that read + print file contents. A shell-expanded ($VAR / `cmd`) path passed
|
|
# to one of these can exfiltrate a host secret whose name the static scan cannot resolve.
|
|
_SHELL_READ_COMMANDS = frozenset(
|
|
{
|
|
"cat",
|
|
"head",
|
|
"tail",
|
|
"less",
|
|
"more",
|
|
"od",
|
|
"xxd",
|
|
"hexdump",
|
|
"strings",
|
|
"nl",
|
|
"tac",
|
|
"cut",
|
|
"sort",
|
|
"uniq",
|
|
"wc",
|
|
"base64",
|
|
"base32",
|
|
"sed",
|
|
"grep",
|
|
"egrep",
|
|
"fgrep",
|
|
"rev",
|
|
"fold",
|
|
"paste",
|
|
"comm",
|
|
"tr",
|
|
"dd",
|
|
"readlink",
|
|
"realpath",
|
|
# diff-style utilities print file contents in their output: `diff SECRET /dev/null`
|
|
# (or `cmp -l SECRET /dev/null`) leaks the file line-by-line / byte-by-byte, so a
|
|
# shell-expanded ($VAR / glob / `cmd`) path handed to one exfiltrates a host secret.
|
|
"diff",
|
|
"sdiff",
|
|
"diff3",
|
|
"colordiff",
|
|
"cmp",
|
|
}
|
|
)
|
|
# Wrappers whose next non-flag argument is the command Bash will exec.
|
|
_COMMAND_PREFIXES = frozenset(
|
|
{
|
|
"env",
|
|
"command",
|
|
"builtin",
|
|
"exec",
|
|
"time",
|
|
"nohup",
|
|
"nice",
|
|
"setsid",
|
|
"stdbuf",
|
|
"timeout",
|
|
"ionice",
|
|
"chroot",
|
|
"sudo",
|
|
"doas",
|
|
"su",
|
|
"xargs",
|
|
# chrt [options] <priority> <command> [<arg>...]: util-linux scheduler wrapper that
|
|
# execs the following command, so chrt -o 0 touch /tmp/x must resolve to touch.
|
|
"chrt",
|
|
}
|
|
)
|
|
_ASSIGNMENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=")
|
|
# Per-wrapper option flags that take a SEPARATED operand (the NEXT token is the flag's value,
|
|
# not the command). Anything not listed -- a no-operand flag (env -i, xargs -0), a GLUED short
|
|
# flag (stdbuf -oL), or a --long=value -- does NOT consume the next token, so the real command
|
|
# after it is still analysed. Wrappers absent from the map default to no operand-taking flags
|
|
# (their numeric args, nice -n 5 / timeout 5, are skipped separately).
|
|
_WRAPPER_OPERAND_FLAGS = {
|
|
"env": frozenset({"-u", "--unset", "-C", "--chdir"}),
|
|
"nice": frozenset({"-n", "--adjustment"}),
|
|
"timeout": frozenset({"-s", "--signal", "-k", "--kill-after"}),
|
|
"stdbuf": frozenset({"-i", "--input", "-o", "--output", "-e", "--error"}),
|
|
"ionice": frozenset({"-c", "--class", "-n", "--classdata", "-p", "--pid"}),
|
|
"sudo": frozenset(
|
|
{
|
|
"-u",
|
|
"--user",
|
|
"-g",
|
|
"--group",
|
|
"-C",
|
|
"--close-from",
|
|
"-h",
|
|
"--host",
|
|
"-p",
|
|
"--prompt",
|
|
"-r",
|
|
"--role",
|
|
"-t",
|
|
"--type",
|
|
"-U",
|
|
"--other-user",
|
|
"-T",
|
|
"--command-timeout",
|
|
"-R",
|
|
"--chroot",
|
|
"-D",
|
|
"--chdir",
|
|
}
|
|
),
|
|
"xargs": frozenset(
|
|
{
|
|
"-n",
|
|
"--max-args",
|
|
"-P",
|
|
"--max-procs",
|
|
"-L",
|
|
"--max-lines",
|
|
"-s",
|
|
"--max-chars",
|
|
"-I",
|
|
"--replace",
|
|
"-E",
|
|
"-d",
|
|
"--delimiter",
|
|
"-a",
|
|
"--arg-file",
|
|
}
|
|
),
|
|
"time": frozenset({"-f", "--format", "-o", "--output"}),
|
|
"chrt": frozenset({"-T", "--sched-runtime", "-P", "--sched-period", "-D", "--sched-deadline"}),
|
|
}
|
|
|
|
|
|
def _wrapper_flag_takes_operand(wrapper, flag: str) -> bool:
|
|
"""True when a wrapper option FLAG consumes the NEXT token as a separated operand
|
|
(env -u NAME, nice -n 5, stdbuf -o L). A glued short flag (-oL), a --long=value, or any
|
|
flag not listed for the wrapper does NOT, so the command word after it is still analysed
|
|
(stdbuf -oL sed -i ..., xargs -0 sed ...)."""
|
|
if "=" in flag:
|
|
return False
|
|
if not flag.startswith("--") and len(flag) > 2:
|
|
return False # glued short flag: -oL already carries its value
|
|
return flag in _WRAPPER_OPERAND_FLAGS.get(wrapper, frozenset())
|
|
|
|
|
|
# GNU sed can WRITE files (`w FILE`, `W FILE`, `s///w FILE`) or EXECUTE shell commands
|
|
# (`e COMMAND`, `s///e`) straight from its SCRIPT even without -i, escaping the workdir in an
|
|
# unguarded child. The filename/command may follow immediately (GNU accepts `w/tmp/x`) or
|
|
# after whitespace. A plain `s/word/x/` has `w`/`e` inside the pattern/replacement (a letter or
|
|
# closing delimiter follows), so these patterns are shaped to skip that.
|
|
_SED_WRITE_RE = re.compile(r"(?<![A-Za-z])[wW](?:[ \t]|/|~)")
|
|
_SED_EXEC_RE = re.compile(r"(?:^|[;\n{}]|[0-9$])[ \t]*e(?:[ \t;}\n]|$)")
|
|
# A completed s/PATTERN/REPL/FLAGS whose FLAGS include w (write) or e (execute).
|
|
_SED_SFLAG_RE = re.compile(r"s(.)(?:(?!\1).)*\1(?:(?!\1).)*\1[A-Za-z0-9]*[we]")
|
|
_FIND_EXEC_FLAGS = frozenset({"-exec", "-execdir", "-ok", "-okdir"})
|
|
# find's -exec / -ok command runs up to a `;` or `+` terminator; everything between
|
|
# is a full command line (may itself begin with a wrapper like env/timeout or sh -c).
|
|
_FIND_EXEC_TERMINATORS = frozenset({";", "+"})
|
|
|
|
|
|
def _is_wrapper_numeric_arg(token: str) -> bool:
|
|
"""A wrapper's numeric argument (`nice -n 5`, `timeout 5m`, `timeout 0.5`).
|
|
|
|
Accepts a plain int/float, optionally with a single trailing GNU ``timeout``
|
|
duration unit (s/m/h/d). Used only to decide whether to skip a token while a
|
|
command-prefix wrapper is still awaiting its real command, so being permissive
|
|
keeps the scan on the following command rather than dropping out of command
|
|
position.
|
|
"""
|
|
t = token.lstrip("-")
|
|
if not t:
|
|
return False
|
|
if len(t) > 1 and t[-1] in "smhd":
|
|
t = t[:-1]
|
|
try:
|
|
float(t)
|
|
return True
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
_ANSI_C_ESCAPES = {
|
|
"a": "\a",
|
|
"b": "\b",
|
|
"e": "\x1b",
|
|
"E": "\x1b",
|
|
"f": "\f",
|
|
"n": "\n",
|
|
"r": "\r",
|
|
"t": "\t",
|
|
"v": "\v",
|
|
"\\": "\\",
|
|
"'": "'",
|
|
'"': '"',
|
|
"?": "?",
|
|
}
|
|
|
|
|
|
def _decode_ansi_c(body: str) -> str:
|
|
"""Decode the escape sequences bash resolves inside a $'...' word (\\n, \\t, \\xHH,
|
|
octal \\NNN, \\uHHHH, ...) so the resulting command word matches what actually runs."""
|
|
out = []
|
|
i, n = 0, len(body)
|
|
while i < n:
|
|
c = body[i]
|
|
if c != "\\" or i + 1 >= n:
|
|
out.append(c)
|
|
i += 1
|
|
continue
|
|
d = body[i + 1]
|
|
if d in _ANSI_C_ESCAPES:
|
|
out.append(_ANSI_C_ESCAPES[d])
|
|
i += 2
|
|
elif d == "x":
|
|
j, h = i + 2, ""
|
|
while j < n and len(h) < 2 and body[j] in "0123456789abcdefABCDEF":
|
|
h += body[j]
|
|
j += 1
|
|
if h:
|
|
out.append(chr(int(h, 16)))
|
|
i = j
|
|
else:
|
|
out.append(c)
|
|
out.append(d)
|
|
i += 2
|
|
elif d in "01234567":
|
|
j, o = i + 1, ""
|
|
while j < n and len(o) < 3 and body[j] in "01234567":
|
|
o += body[j]
|
|
j += 1
|
|
out.append(chr(int(o, 8) & 0xFF))
|
|
i = j
|
|
elif d in ("u", "U"):
|
|
width = 4 if d == "u" else 8
|
|
j, h = i + 2, ""
|
|
while j < n and len(h) < width and body[j] in "0123456789abcdefABCDEF":
|
|
h += body[j]
|
|
j += 1
|
|
if h:
|
|
out.append(chr(int(h, 16)))
|
|
i = j
|
|
else:
|
|
out.append(c)
|
|
out.append(d)
|
|
i += 2
|
|
else:
|
|
out.append(c)
|
|
out.append(d)
|
|
i += 2
|
|
return "".join(out)
|
|
|
|
|
|
def _normalize_ansi_c_quotes(command: str) -> str:
|
|
"""Rewrite bash ANSI-C ($'...') and locale ($"...") quoted words to plain quoted words
|
|
so shlex sees the token bash actually executes. shlex leaves `$'touch'` as the literal
|
|
`$touch`, so a writer/interpreter hidden behind ANSI-C quoting (`$'touch' x`,
|
|
`$'\\x74ouch' x`) never matches the command blocklist otherwise."""
|
|
if "$'" not in command and '$"' not in command:
|
|
return command
|
|
res = []
|
|
i, n = 0, len(command)
|
|
while i < n:
|
|
if command[i] == "$" and i + 1 < n and command[i + 1] == '"':
|
|
res.append('"') # locale translation: bash just strips the leading $
|
|
i += 2
|
|
continue
|
|
if command[i] == "$" and i + 1 < n and command[i + 1] == "'":
|
|
j, buf = i + 2, []
|
|
while j < n:
|
|
if command[j] == "\\" and j + 1 < n:
|
|
buf.append(command[j])
|
|
buf.append(command[j + 1])
|
|
j += 2
|
|
continue
|
|
if command[j] == "'":
|
|
break
|
|
buf.append(command[j])
|
|
j += 1
|
|
decoded = _decode_ansi_c("".join(buf))
|
|
# Re-emit as a single-quoted shlex token (escaping embedded single quotes).
|
|
res.append("'" + decoded.replace("'", "'\\''") + "'")
|
|
i = j + 1 # skip the closing quote
|
|
continue
|
|
res.append(command[i])
|
|
i += 1
|
|
return "".join(res)
|
|
|
|
|
|
_IFS_RE = re.compile(r"\$\{IFS[^}]*\}|\$IFS\b")
|
|
|
|
|
|
def _expand_ifs(command: str) -> str:
|
|
"""bash expands ${IFS} / $IFS to whitespace (default space/tab/newline) BEFORE word
|
|
splitting, so cat${IFS}/etc/shadow runs `cat /etc/shadow` in the child. Replace an IFS
|
|
reference with a space so the scanner tokenizes the command bash actually executes."""
|
|
if "IFS" not in command:
|
|
return command
|
|
return _IFS_RE.sub(" ", command)
|
|
|
|
|
|
def _rewrite_unquoted_newlines(command: str) -> str:
|
|
"""Rewrite only UNQUOTED newline runs to ` ; ` (a bash command separator). A newline INSIDE
|
|
quotes is data (echo "ok\\nrm" is one argument), so a blanket regex would split a quoted
|
|
multiline string into a spurious command position and mis-block the later line."""
|
|
out = []
|
|
q = None
|
|
esc = False
|
|
prev_nl = False
|
|
for ch in command:
|
|
if esc:
|
|
out.append(ch)
|
|
esc = False
|
|
prev_nl = False
|
|
continue
|
|
if q == "'":
|
|
out.append(ch)
|
|
if ch == "'":
|
|
q = None
|
|
prev_nl = False
|
|
continue
|
|
if q == '"':
|
|
out.append(ch)
|
|
if ch == "\\":
|
|
esc = True
|
|
elif ch == '"':
|
|
q = None
|
|
prev_nl = False
|
|
continue
|
|
if ch == "\\":
|
|
out.append(ch)
|
|
esc = True
|
|
prev_nl = False
|
|
continue
|
|
if ch in ("'", '"'):
|
|
out.append(ch)
|
|
q = ch
|
|
prev_nl = False
|
|
continue
|
|
if ch in ("\r", "\n"):
|
|
if not prev_nl:
|
|
out.append(" ; ")
|
|
prev_nl = True
|
|
continue
|
|
out.append(ch)
|
|
prev_nl = False
|
|
return "".join(out)
|
|
|
|
|
|
def _mask_quoted_separators(command: str) -> str:
|
|
"""Neutralize command-boundary characters that are DATA inside quotes (blank them to a
|
|
space) so the regex command-position scan does not treat a quoted separator -- echo
|
|
"ok\\nrm" or 'a;rm' -- as a fresh command word. Command substitution ($(...) / backticks)
|
|
still runs inside DOUBLE quotes, so those are preserved; single-quoted text is fully
|
|
literal. The result is used only for the boundary regex, not for tokenization."""
|
|
out = []
|
|
q = None
|
|
esc = False
|
|
i = 0
|
|
n = len(command)
|
|
while i < n:
|
|
ch = command[i]
|
|
if esc:
|
|
out.append(ch)
|
|
esc = False
|
|
i += 1
|
|
continue
|
|
if q == "'":
|
|
out.append(" " if ch in ";&|(\n\r`$" else ch)
|
|
if ch == "'":
|
|
q = None
|
|
i += 1
|
|
continue
|
|
if q == '"':
|
|
if ch == "\\":
|
|
out.append(ch)
|
|
esc = True
|
|
i += 1
|
|
continue
|
|
if ch == '"':
|
|
out.append(ch)
|
|
q = None
|
|
i += 1
|
|
continue
|
|
if ch == "$" and i + 1 < n and command[i + 1] == "(":
|
|
out.append("$(") # command substitution runs inside double quotes; keep it
|
|
i += 2
|
|
continue
|
|
if ch == "`":
|
|
out.append("`")
|
|
i += 1
|
|
continue
|
|
out.append(" " if ch in ";&|(\n\r" else ch)
|
|
i += 1
|
|
continue
|
|
if ch == "\\":
|
|
out.append(ch)
|
|
esc = True
|
|
i += 1
|
|
continue
|
|
if ch in ("'", '"'):
|
|
out.append(ch)
|
|
q = ch
|
|
i += 1
|
|
continue
|
|
out.append(ch)
|
|
i += 1
|
|
return "".join(out)
|
|
|
|
|
|
def _iter_unquoted_chars(s):
|
|
"""Yield (index, char) for every character OUTSIDE single / double quotes (a backslash
|
|
escape and the char it escapes are skipped inside double quotes / unquoted text). Used to
|
|
locate brace-expansion syntax that bash would act on, ignoring quoted braces."""
|
|
q = None
|
|
esc = False
|
|
for i, ch in enumerate(s):
|
|
if esc:
|
|
esc = False
|
|
continue
|
|
if q == "'":
|
|
if ch == "'":
|
|
q = None
|
|
continue
|
|
if q == '"':
|
|
if ch == "\\":
|
|
esc = True
|
|
elif ch == '"':
|
|
q = None
|
|
continue
|
|
if ch == "\\":
|
|
esc = True
|
|
yield i, ch
|
|
continue
|
|
if ch in ("'", '"'):
|
|
q = ch
|
|
continue
|
|
yield i, ch
|
|
|
|
|
|
def _brace_first_comma_group(s):
|
|
"""Return (open, close) indices of the first UNQUOTED ``{...}`` that contains a top-level
|
|
comma (the shape bash expands), else None. ``{}`` / ``${x}`` / ``{1..5}`` have no top-level
|
|
comma and are left untouched, as are quoted braces."""
|
|
idxset = {i: ch for i, ch in _iter_unquoted_chars(s)}
|
|
for o, ch in list(idxset.items()):
|
|
if ch != "{":
|
|
continue
|
|
depth = 0
|
|
has_comma = False
|
|
for i in range(o, len(s)):
|
|
c = idxset.get(i)
|
|
if c is None:
|
|
continue
|
|
if c == "{":
|
|
depth += 1
|
|
elif c == "}":
|
|
depth -= 1
|
|
if depth == 0:
|
|
if has_comma:
|
|
return o, i
|
|
break
|
|
elif c == "," and depth == 1:
|
|
has_comma = True
|
|
return None
|
|
|
|
|
|
def _brace_split_top_commas(content):
|
|
"""Split a brace group's inner text on top-level (unnested, unquoted) commas."""
|
|
parts = []
|
|
cur = []
|
|
depth = 0
|
|
q = None
|
|
esc = False
|
|
for ch in content:
|
|
if esc:
|
|
cur.append(ch)
|
|
esc = False
|
|
continue
|
|
if q == "'":
|
|
cur.append(ch)
|
|
if ch == "'":
|
|
q = None
|
|
continue
|
|
if q == '"':
|
|
cur.append(ch)
|
|
if ch == "\\":
|
|
esc = True
|
|
elif ch == '"':
|
|
q = None
|
|
continue
|
|
if ch == "\\":
|
|
cur.append(ch)
|
|
esc = True
|
|
continue
|
|
if ch in ("'", '"'):
|
|
cur.append(ch)
|
|
q = ch
|
|
continue
|
|
if ch == "{":
|
|
depth += 1
|
|
cur.append(ch)
|
|
continue
|
|
if ch == "}":
|
|
depth -= 1
|
|
cur.append(ch)
|
|
continue
|
|
if ch == "," and depth == 0:
|
|
parts.append("".join(cur))
|
|
cur = []
|
|
continue
|
|
cur.append(ch)
|
|
parts.append("".join(cur))
|
|
return parts
|
|
|
|
|
|
def _brace_expand_word(word, budget):
|
|
"""Recursively expand a single word's comma brace groups (bash-style, quote-aware,
|
|
cartesian across multiple groups), returning the list of expansions. Bounded by budget."""
|
|
grp = _brace_first_comma_group(word)
|
|
if grp is None:
|
|
return [word]
|
|
o, c = grp
|
|
pre, content, post = word[:o], word[o + 1 : c], word[c + 1 :]
|
|
out = []
|
|
for opt in _brace_split_top_commas(content):
|
|
for opt_exp in _brace_expand_word(opt, budget):
|
|
for post_exp in _brace_expand_word(post, budget):
|
|
out.append(pre + opt_exp + post_exp)
|
|
if len(out) >= budget[0]:
|
|
return out
|
|
return out
|
|
|
|
|
|
def _split_words_unquoted_ws(s):
|
|
"""Split ``s`` into words on UNQUOTED space / tab; emit an unquoted newline as its own
|
|
token so it survives as a command separator. Quotes and their contents stay intact."""
|
|
words = []
|
|
cur = []
|
|
q = None
|
|
esc = False
|
|
for ch in s:
|
|
if esc:
|
|
cur.append(ch)
|
|
esc = False
|
|
continue
|
|
if q == "'":
|
|
cur.append(ch)
|
|
if ch == "'":
|
|
q = None
|
|
continue
|
|
if q == '"':
|
|
cur.append(ch)
|
|
if ch == "\\":
|
|
esc = True
|
|
elif ch == '"':
|
|
q = None
|
|
continue
|
|
if ch == "\\":
|
|
cur.append(ch)
|
|
esc = True
|
|
continue
|
|
if ch in ("'", '"'):
|
|
cur.append(ch)
|
|
q = ch
|
|
continue
|
|
if ch == "\n":
|
|
if cur:
|
|
words.append("".join(cur))
|
|
cur = []
|
|
words.append("\n")
|
|
continue
|
|
if ch in " \t":
|
|
if cur:
|
|
words.append("".join(cur))
|
|
cur = []
|
|
continue
|
|
cur.append(ch)
|
|
if cur:
|
|
words.append("".join(cur))
|
|
return words
|
|
|
|
|
|
def _expand_braces(command: str) -> str:
|
|
"""Model bash brace expansion (comma lists) before the block / read scans so a payload such
|
|
as ``{touch,/tmp/escape}`` or ``{python3,-c} '...'`` is seen as the writer / interpreter bash
|
|
would actually run, instead of a single opaque ``{...}`` token. Only unquoted groups with a
|
|
top-level comma are expanded; ``{}`` (find -exec), ``${VAR}`` parameter expansion, numeric
|
|
``{1..5}`` sequences and quoted braces are left intact. Expansion is bounded to avoid blowup;
|
|
if the bound is hit the (partial) expansion is still scanned."""
|
|
if "{" not in command:
|
|
return command
|
|
budget = [4096]
|
|
out = []
|
|
for w in _split_words_unquoted_ws(command):
|
|
if "{" in w and "}" in w and "," in w:
|
|
out.extend(_brace_expand_word(w, budget))
|
|
else:
|
|
out.append(w)
|
|
if len(out) >= 8192:
|
|
break
|
|
return " ".join(out)
|
|
|
|
|
|
def _find_blocked_commands(command: str) -> set[str]:
|
|
"""Detect blocked commands at shell command position only.
|
|
|
|
A token is at command position if it is the first token, or follows a
|
|
shell separator / brace-group opener / new-command keyword (`then`, `do`,
|
|
etc.), or a command-prefix wrapper like `env` / `time` / `xargs` (next
|
|
token is the real command). Tokens in argument position (`grep -r curl .`,
|
|
`echo source the data`, `ls /usr/bin/curl`) pass through. Also scans
|
|
`find ... -exec CMD` and recurses into bash -c / cmd /c.
|
|
"""
|
|
blocked: set[str] = set()
|
|
|
|
# Normalize bash ANSI-C ($'...') / locale ($"...") quoting first: shlex leaves
|
|
# `$'touch'` as `$touch`, so a writer/interpreter hidden behind ANSI-C quoting would
|
|
# never match the blocklist even though bash decodes and runs it. Then expand ${IFS} to
|
|
# whitespace so a separator-obfuscated command (rm${IFS}-rf${IFS}/) is tokenized.
|
|
command = _expand_ifs(_normalize_ansi_c_quotes(command))
|
|
# bash treats an unquoted newline as a command separator, but shlex's whitespace_split
|
|
# folds it into ordinary whitespace, so `echo ok\nsed -i ...` would read `sed` as an
|
|
# argument of `echo` and miss the write. Rewrite UNQUOTED newlines to `;` so each line
|
|
# starts a fresh command position; a newline inside quotes stays data (echo "ok\nrm" is one
|
|
# argument), so it is not turned into a spurious `; rm` command position.
|
|
command = _rewrite_unquoted_newlines(command)
|
|
# bash performs brace expansion before command lookup, so `{touch,/tmp/x}` /
|
|
# `{python3,-c} '...'` run the writer / interpreter even though the raw string has no
|
|
# blocked token. Expand comma brace groups so the produced command words are scanned.
|
|
command = _expand_braces(command)
|
|
|
|
# punctuation_chars splits separators into their own tokens, so command
|
|
# position is detected even in `echo done; rm -rf x` (no whitespace) or
|
|
# quote-split names (`r''m` collapses to `rm` after `;`). Including `<` splits an INPUT
|
|
# redirect / here-string glued to the command word (sh<<<'...', cat</etc/passwd) so the
|
|
# shell / reader is still recognized. (`>` is left out so the regex-based output-redirect
|
|
# scan keeps seeing `>&` as one operator.)
|
|
try:
|
|
if sys.platform == "win32":
|
|
tokens = shlex.split(command, posix = False)
|
|
else:
|
|
lexer = shlex.shlex(command, posix = True, punctuation_chars = ";&|()`<")
|
|
lexer.whitespace_split = True
|
|
tokens = list(lexer)
|
|
except ValueError:
|
|
tokens = command.split()
|
|
|
|
def _token_basename(tok: str) -> str:
|
|
# Strip glued-on meta-chars (`rm;`) so the basename still matches `rm`.
|
|
tok = tok.strip(";&|()`{}")
|
|
base = os.path.basename(tok).lower()
|
|
stem, ext = os.path.splitext(base)
|
|
if ext in {".exe", ".com", ".bat", ".cmd"}:
|
|
base = stem
|
|
return base
|
|
|
|
expect_command = True # start of string is a command position
|
|
prefix_pending = False # last cmd-position token was a wrapper (env/time/xargs/...)
|
|
prev_was_flag = False # previous token (while a wrapper is pending) takes an operand
|
|
cur_wrapper = None # the active wrapper's basename (drives per-wrapper option arity)
|
|
for token in tokens:
|
|
if token in _SHELL_SEPARATORS or token in _SHELL_KEYWORDS_AS_SEP:
|
|
expect_command = True
|
|
prefix_pending = False
|
|
prev_was_flag = False
|
|
cur_wrapper = None
|
|
continue
|
|
if token.startswith("-"):
|
|
# Flags belong to the active command, but keep expect_command while a
|
|
# wrapper prefix awaits its command. Only a flag that actually takes a SEPARATED
|
|
# operand (env -u NAME) marks the next token as its value; a glued / no-operand
|
|
# flag (stdbuf -oL sed, xargs -0 sed) does not, so the command that follows is
|
|
# still analysed.
|
|
if not prefix_pending:
|
|
expect_command = False
|
|
elif _wrapper_flag_takes_operand(cur_wrapper, token):
|
|
prev_was_flag = True
|
|
continue
|
|
if not expect_command:
|
|
continue
|
|
# A leading `!` negates the pipeline exit status, but the following word is still the
|
|
# command bash executes (`! touch x`, `! python3 -c ...`). Keep command position so the
|
|
# real command is scanned, rather than mistaking `!` for the command and its command for
|
|
# an argument.
|
|
if token == "!":
|
|
continue
|
|
# FOO=bar assignment prefix; next non-assignment token is the command.
|
|
if _ASSIGNMENT_RE.match(token):
|
|
continue
|
|
# Numeric wrapper arg: `timeout 1 cmd` / `nice -n 5 cmd`, plus GNU `timeout`
|
|
# duration forms (`5m`, `0.5`, `2h`). Skipping it keeps prefix_pending so the
|
|
# real command that follows is still analysed at command position; over-
|
|
# accepting a numeric-looking token is safe (we only skip, never stop scanning),
|
|
# whereas the old int-only check let `timeout 5m rm -rf /` slip through.
|
|
if prefix_pending and _is_wrapper_numeric_arg(token):
|
|
prev_was_flag = False
|
|
continue
|
|
base = _token_basename(token)
|
|
# A wrapper's separated option ARGUMENT (`stdbuf -o L cmd`, `ionice -c 2 cmd`):
|
|
# an operand right after a wrapper flag that is NOT itself a blocked command /
|
|
# prefix is the flag's value, so skip it and keep scanning for the real command
|
|
# instead of mistaking it for the command and stopping. If it IS a blocked
|
|
# command / prefix it is treated as the command below (never miss `env -i rm`).
|
|
if (
|
|
prefix_pending
|
|
and prev_was_flag
|
|
and base not in _BLOCKED_COMMANDS
|
|
and base not in _COMMAND_PREFIXES
|
|
):
|
|
prev_was_flag = False
|
|
continue
|
|
prev_was_flag = False
|
|
# An expansion sitting AT the resolved command word -- behind a wrapper
|
|
# (env $CMD -c ...) or after a leading assignment -- runs whatever it expands to as
|
|
# the command name and cannot be proven safe, so fail closed. The separator-anchored
|
|
# regex below misses the wrapper case because $CMD is not right after a separator.
|
|
if "$" in token or "`" in token:
|
|
blocked.add("command-expansion")
|
|
if base in _BLOCKED_COMMANDS or _is_versioned_interpreter(base):
|
|
blocked.add(base)
|
|
# The `.` builtin is bash's `source`: `. evil.sh` runs an unscanned script in the
|
|
# shell, the same escape as `source`, but its basename is not a blocklist word.
|
|
if base == ".":
|
|
blocked.add("source")
|
|
# An explicit path to a LOCAL executable at command position (./evil, subdir/tool) runs
|
|
# whatever its shebang names in an unguarded child, so treat it like a blocked command.
|
|
if _is_local_executable_path(token):
|
|
blocked.add("local-exec:" + base)
|
|
# Wrappers (env/time/xargs/sudo) consume one command; the next non-flag,
|
|
# non-numeric token is the real command. sudo is also in _BLOCKED_COMMANDS.
|
|
if base in _COMMAND_PREFIXES:
|
|
prefix_pending = True
|
|
cur_wrapper = base
|
|
continue
|
|
expect_command = False
|
|
prefix_pending = False
|
|
cur_wrapper = None
|
|
|
|
# `find ... -exec CMD ... ;` / `-execdir CMD ... +` invoke CMD directly. CMD may
|
|
# itself be a wrapper (`env rm`, `timeout 5 rm`) or a nested shell (`sh -c '...'`),
|
|
# so rescan the whole slice up to the `;`/`+` terminator through the full command-
|
|
# position analyzer instead of only basename-matching the immediate next token.
|
|
for i, tok in enumerate(tokens):
|
|
if tok in _FIND_EXEC_FLAGS:
|
|
seg = []
|
|
j = i + 1
|
|
while j < len(tokens) and tokens[j] not in _FIND_EXEC_TERMINATORS:
|
|
seg.append(tokens[j])
|
|
j += 1
|
|
if seg:
|
|
blocked |= _find_blocked_commands(" ".join(seg))
|
|
|
|
# Regex catches blocked words at command boundaries shlex misses: inside
|
|
# $(rm -rf), <(rm), backtick chains, or "foo;rm". Anchored to command-position
|
|
# delimiters, so it doesn't match in argument position. Quoted separators are neutralized
|
|
# first so a quoted multiline string (echo "ok\nrm") is not read as a command boundary.
|
|
lowered = _mask_quoted_separators(command).lower()
|
|
if _BLOCKED_COMMANDS:
|
|
words_alt = "|".join(re.escape(w) for w in sorted(_BLOCKED_COMMANDS))
|
|
pattern = (
|
|
rf"(?:^|[;&|`\n(]\s*|[$]\(\s*|<\(\s*)"
|
|
rf"(?:[\w./\\-]*/|[a-zA-Z]:[/\\][\w./\\-]*)?"
|
|
rf"({words_alt})(?:\.(?:exe|com|bat|cmd))?\b"
|
|
)
|
|
blocked.update(re.findall(pattern, lowered))
|
|
|
|
# Nested shell invocations (bash -c '...', bash -lc '...', cmd /c '...'):
|
|
# on a -c/-/c flag, look back for a shell name (skipping flags) and
|
|
# recursively scan the nested command string.
|
|
_SHELLS = _SHELL_BINARIES
|
|
_SHELLS_WIN = {"cmd", "cmd.exe"}
|
|
for i, token in enumerate(tokens):
|
|
tok_lower = token.lower()
|
|
# Match -c exactly, or combined flags ending in c (e.g. -lc, -xc)
|
|
is_unix_c = tok_lower == "-c" or (
|
|
tok_lower.startswith("-") and tok_lower.endswith("c") and not tok_lower.startswith("--")
|
|
)
|
|
is_win_c = tok_lower == "/c"
|
|
if not (is_unix_c or is_win_c) or i < 1 or i + 1 >= len(tokens):
|
|
continue
|
|
# Look back past flags for the shell binary. Windows flags and absolute
|
|
# paths both start with /, so only skip short /X flags (not /bin/bash).
|
|
for j in range(i - 1, -1, -1):
|
|
prev = tokens[j]
|
|
if prev.startswith("-"):
|
|
continue # skip Unix flags like --login, -l
|
|
if is_win_c and prev.startswith("/") and len(prev) <= 3:
|
|
continue # skip Windows flags like /s, /q (not /bin/bash)
|
|
prev_base = os.path.basename(prev).lower()
|
|
if is_unix_c and prev_base in _SHELLS:
|
|
blocked |= _find_blocked_commands(tokens[i + 1])
|
|
elif is_win_c and prev_base in _SHELLS_WIN:
|
|
blocked |= _find_blocked_commands(tokens[i + 1])
|
|
break # stop at first non-flag token
|
|
|
|
# `env -S 'cmd ...'` / `env --split-string='cmd'` splits the string and runs it as a
|
|
# fresh command, so a bare `env -S` operand is NOT just a flag value -- recurse into
|
|
# it (it can invoke an unguarded interpreter or another blocked command).
|
|
for i, token in enumerate(tokens):
|
|
tl = token.lower()
|
|
payload = None
|
|
if tl in ("-s", "--split-string") and i + 1 < len(tokens):
|
|
payload = tokens[i + 1]
|
|
elif tl.startswith("-s") and tl != "-s" and not tl.startswith("--"):
|
|
payload = token[2:] # glued short form: env -S'cmd' / -Scmd
|
|
elif tl.startswith("--split-string="):
|
|
payload = token[len("--split-string=") :]
|
|
if not payload:
|
|
continue
|
|
for j in range(i - 1, -1, -1):
|
|
prev = tokens[j]
|
|
if prev.startswith("-"):
|
|
continue
|
|
if os.path.basename(prev).lower() == "env":
|
|
blocked |= _find_blocked_commands(payload)
|
|
break
|
|
|
|
def _command_word_indices():
|
|
# Indices of the REAL command word at each command position, skipping FOO=bar
|
|
# assignments and wrapper prefixes (env / nice / timeout / xargs / ...) plus their
|
|
# numeric / separated-option arguments, so `env sed`, `timeout 5 bash` resolve to
|
|
# sed / bash. Mirrors the main command-position scan above.
|
|
out = []
|
|
expect = True
|
|
pending = False
|
|
prev_flag = False
|
|
wrapper = None
|
|
for _i, _tok in enumerate(tokens):
|
|
if _tok in _SHELL_SEPARATORS or _tok in _SHELL_KEYWORDS_AS_SEP:
|
|
expect = True
|
|
pending = False
|
|
prev_flag = False
|
|
wrapper = None
|
|
continue
|
|
if _tok.startswith("-"):
|
|
if not pending:
|
|
expect = False
|
|
elif _wrapper_flag_takes_operand(wrapper, _tok):
|
|
prev_flag = True
|
|
continue
|
|
if not expect:
|
|
continue
|
|
if _tok == "!":
|
|
continue # pipeline negation keeps command position (! bash s.sh)
|
|
if _ASSIGNMENT_RE.match(_tok):
|
|
continue
|
|
if pending and _is_wrapper_numeric_arg(_tok):
|
|
prev_flag = False
|
|
continue
|
|
_base = _token_basename(_tok)
|
|
if (
|
|
pending
|
|
and prev_flag
|
|
and _base not in _BLOCKED_COMMANDS
|
|
and _base not in _COMMAND_PREFIXES
|
|
):
|
|
prev_flag = False
|
|
continue
|
|
prev_flag = False
|
|
if _base in _COMMAND_PREFIXES:
|
|
pending = True
|
|
wrapper = _base
|
|
continue
|
|
out.append(_i)
|
|
expect = False
|
|
pending = False
|
|
wrapper = None
|
|
return out
|
|
|
|
_cmd_word_idx = _command_word_indices()
|
|
|
|
# trap 'CMD' SIGSPEC registers CMD to run (in the unguarded shell) on EXIT / a signal, so
|
|
# the quoted handler is unscanned shell code. Scan the handler operand of a command-position
|
|
# `trap` recursively; a reset (trap - EXIT) / ignore (trap '' EXIT) has nothing to run.
|
|
for i in _cmd_word_idx:
|
|
if _token_basename(tokens[i]) != "trap" or i + 1 >= len(tokens):
|
|
continue
|
|
_h = tokens[i + 1]
|
|
if _h and _h != "-" and _h not in _SHELL_SEPARATORS and _h not in _SHELL_KEYWORDS_AS_SEP:
|
|
blocked |= _find_blocked_commands(_h)
|
|
|
|
# A shell binary invoked with a SCRIPT FILE (`bash s.sh`) or `-s` (read the script from
|
|
# stdin) runs unscanned shell code in the same unguarded environment; only the inline
|
|
# `-c '...'` form is statically analyzable (handled above). Block a command-position
|
|
# shell whose operands include a non-flag argument (the script) and no -c/-lc flag. Using
|
|
# the wrapper-aware command-word indices so `env bash s.sh` / `timeout 5 bash s.sh` are
|
|
# not hidden behind the wrapper prefix.
|
|
for i in _cmd_word_idx:
|
|
tok = tokens[i]
|
|
if os.path.basename(tok).lower() not in _SHELLS:
|
|
continue
|
|
_has_c = False
|
|
_script = None
|
|
for k in range(i + 1, len(tokens)):
|
|
t = tokens[k]
|
|
if t in _SHELL_SEPARATORS or t in _SHELL_KEYWORDS_AS_SEP:
|
|
break
|
|
tl = t.lower()
|
|
if tl == "-c" or (tl.startswith("-") and not tl.startswith("--") and tl.endswith("c")):
|
|
_has_c = True
|
|
break
|
|
if tl in ("-s", "--"): # -s reads the script from stdin (unscanned)
|
|
_script = t
|
|
break
|
|
if t.startswith("-"):
|
|
continue # other shell flags: -l, -x, --login, --norc, ...
|
|
_script = t # first non-flag operand is the script file
|
|
break
|
|
# Any command-position shell WITHOUT an inline `-c` payload runs unscanned code:
|
|
# a script file (bash s.sh), stdin via -s, or a bare shell that reads stdin
|
|
# (`printf 'evil' | bash`). Only the `-c '...'` form is statically analyzable, so
|
|
# block everything else.
|
|
if not _has_c:
|
|
blocked.add("shell-script:" + (_script or _token_basename(tok)))
|
|
# BASH_ENV=script / ENV=script assignment prefix before a shell makes bash / sh SOURCE
|
|
# that workdir file before the scanned -c payload runs, executing unscanned commands in
|
|
# the unguarded child (BASH_ENV=env.sh bash -c 'echo ok', env BASH_ENV=env.sh bash -c).
|
|
# Scan the command segment before this shell word for a non-empty startup-env assignment.
|
|
for k in range(i - 1, -1, -1):
|
|
pk = tokens[k]
|
|
if pk in _SHELL_SEPARATORS or pk in _SHELL_KEYWORDS_AS_SEP:
|
|
break
|
|
if _ASSIGNMENT_RE.match(pk):
|
|
_an, _, _av = pk.partition("=")
|
|
if _an in ("BASH_ENV", "ENV") and _av != "":
|
|
blocked.add("shell-startup-env:" + _an)
|
|
|
|
# Output redirection (> / >> / &> / N>) runs in an unguarded child shell that follows
|
|
# symlinks before any Python guard, so no filename target can be trusted: a relative
|
|
# single-component name (> out) may be a pre-existing symlink to an outside file, a
|
|
# relative multi-component name (> sub/out) may traverse a symlinked subdir, an absolute
|
|
# / ~ / .. target is plainly outside, and a $ / backtick target can expand anywhere.
|
|
# Fail closed on every real-file target; only fd duplications (>&2) and the standard
|
|
# device sinks (/dev/null, ...) are allowed. Scanning tokens (not the raw string) avoids
|
|
# matching a `>` inside a quoted argument.
|
|
for i, tok in enumerate(tokens):
|
|
rm = re.search(r">{1,2}([^\s>]*)$", tok)
|
|
if rm is None:
|
|
continue
|
|
tgt = rm.group(1)
|
|
j = i
|
|
# `>|` (noclobber override) and `>&` (stdout+stderr / fd-or-file redirect) tokenize
|
|
# as `>` then `|` / `&`, so that punctuation is part of the redirect operator, not a
|
|
# pipeline / background op; skip it and take the real target after.
|
|
if not tgt and j + 1 < len(tokens) and tokens[j + 1] in ("|", "&"):
|
|
j += 1
|
|
if not tgt and j + 1 < len(tokens):
|
|
tgt = tokens[j + 1]
|
|
if not tgt:
|
|
continue
|
|
tn = tgt.replace("\\", "/")
|
|
# Allowed: a pure fd duplication (>&2, >&1 -> `&2` / a bare digit) and the safe
|
|
# device sinks. Everything else is a file target that fails closed.
|
|
if tgt.startswith("&") or tgt.isdigit() or tn in _SAFE_REDIRECT_TARGETS:
|
|
continue
|
|
blocked.add("redirect:" + tgt)
|
|
|
|
# `cd` / `pushd` to a dir OUTSIDE the workdir moves the child shell's cwd so a later
|
|
# relative redirect / write escapes (`cd /tmp; echo x > p`, `pushd /tmp; echo x > p`).
|
|
# Block a command-position cwd change to an absolute / .. / ~ / variable target; a
|
|
# relative in-workdir `cd data` stays allowed.
|
|
_at_cmd = True
|
|
for i, tok in enumerate(tokens):
|
|
if tok in _SHELL_SEPARATORS or tok in _SHELL_KEYWORDS_AS_SEP:
|
|
_at_cmd = True
|
|
continue
|
|
if _at_cmd and _token_basename(tok) in ("command", "builtin"):
|
|
# `command` / `builtin` run the following shell builtin with its args, so a
|
|
# `command cd /tmp` still changes the cwd. Stay at command position so the cd
|
|
# behind the wrapper is inspected (bash `help command`/`help builtin`).
|
|
continue
|
|
if _at_cmd and _token_basename(tok) in ("cd", "pushd"):
|
|
_cwd_kw = _token_basename(tok)
|
|
for k in range(i + 1, len(tokens)):
|
|
t = tokens[k]
|
|
if t.startswith("-") or t.startswith("+"):
|
|
continue # cd flags (-P/-L/-e/-@) and pushd rotation (+N/-N)
|
|
tnn = t.replace("\\", "/")
|
|
if (
|
|
t.startswith("~")
|
|
or tnn.startswith("/")
|
|
or ".." in tnn.split("/")
|
|
or "$" in t
|
|
or "`" in t
|
|
):
|
|
blocked.add(_cwd_kw + ":" + t)
|
|
break
|
|
_at_cmd = False
|
|
continue
|
|
if not tok.startswith("-"):
|
|
_at_cmd = False
|
|
|
|
# An EXPANSION in COMMAND POSITION runs whatever it expands to as the command name and
|
|
# cannot be proven safe: a command substitution ($(printf touch) / `printf touch`), a
|
|
# variable-expanded command word (p=python3; $p -c ...), or a ${VAR} parameter expansion.
|
|
# Fail closed. (An argument-position expansion -- echo $(date), echo $HOME, x=$(cmd) -- is
|
|
# not at command position, so it stays allowed. ${IFS} is already expanded to whitespace
|
|
# above, so a `cat${IFS}x` command word is not misread as an expansion here.)
|
|
if re.search(r"(?:^|[\n;&|(])\s*(?:\$|`)", command):
|
|
blocked.add("command-expansion")
|
|
|
|
# Some normally read-only utilities MUTATE files with certain flags (sed -i, sort -o
|
|
# FILE, find ... -delete, dd of=FILE, tee FILE, truncate), writing/deleting OUTSIDE the
|
|
# workdir in an unguarded child that no redirect token exposes. Treat the mutating
|
|
# invocation as a child writer. Uses the wrapper-aware command-word indices so a wrapper
|
|
# prefix (env sed -i ..., nice sed -i ...) does not hide the mutating utility.
|
|
for i in _cmd_word_idx:
|
|
tok = tokens[i]
|
|
_base = _token_basename(tok)
|
|
if _base not in (
|
|
"sed",
|
|
"gsed",
|
|
"ssed",
|
|
"perl",
|
|
"sort",
|
|
"find",
|
|
"dd",
|
|
"tee",
|
|
"truncate",
|
|
"history",
|
|
):
|
|
continue
|
|
if _base == "truncate":
|
|
blocked.add("mutating:truncate")
|
|
continue
|
|
for k in range(i + 1, len(tokens)):
|
|
a = tokens[k]
|
|
if a in _SHELL_SEPARATORS or a in _SHELL_KEYWORDS_AS_SEP:
|
|
break
|
|
al = a.lower()
|
|
_short = al.startswith("-") and not al.startswith("--")
|
|
if _base in ("sed", "gsed", "ssed", "perl"):
|
|
if al.startswith("--in-place") or (_short and "i" in al[1:]):
|
|
blocked.add("mutating:" + _base)
|
|
break
|
|
# A sed SCRIPT can write files (`w FILE` / `W FILE` / `s///w`) or execute shell
|
|
# commands (`e CMD` / `s///e`) even without -i: sed -n '1w /tmp/escape' file,
|
|
# sed -n 'w/tmp/probe' file (no space), sed '1e touch /tmp/x' file. Detect the
|
|
# write / execute commands and flags; a plain s/word/x/ is not matched.
|
|
if _base in ("sed", "gsed", "ssed") and not a.startswith("-"):
|
|
if _SED_WRITE_RE.search(a) or _SED_EXEC_RE.search(a) or _SED_SFLAG_RE.search(a):
|
|
blocked.add("mutating:" + _base)
|
|
break
|
|
elif _base == "sort":
|
|
if al.startswith("--output") or (_short and "o" in al[1:]):
|
|
blocked.add("mutating:sort")
|
|
break
|
|
elif _base == "find":
|
|
if al == "-delete" or al.startswith("-fprint"):
|
|
blocked.add("mutating:find")
|
|
break
|
|
elif _base == "dd":
|
|
if al.startswith("of="):
|
|
blocked.add("mutating:dd")
|
|
break
|
|
elif _base == "tee" and not a.startswith("-"):
|
|
blocked.add("mutating:tee")
|
|
break
|
|
elif _base == "history" and _short and any(_c in al[1:] for _c in "warn"):
|
|
# bash's history builtin reads/writes an arbitrary file: `history -w FILE`
|
|
# (or -a append) creates/overwrites an absolute host path, and `-r` / `-n`
|
|
# read a file into the history buffer. Even without a FILE operand it targets
|
|
# $HISTFILE, which the caller can point outside the workdir. -c / -d / -p / -s
|
|
# do not touch a file, so only w / a / r / n are blocked.
|
|
blocked.add("mutating:history")
|
|
break
|
|
|
|
return blocked
|
|
|
|
|
|
def _blocked_in_argv(str_elts: list[str | None]) -> tuple[set[str], int | None]:
|
|
"""Scan the command WORDS of a non-shell argv vector (subprocess.run(['rm', '-rf', '/'])).
|
|
Only element 0 -- and the real command after any wrapper prefix (env / nice / timeout /
|
|
xargs / ...) -- is executed by the OS; every later element is a literal argument that is
|
|
never run. Scanning just the command word keeps `env rm -rf /` blocked (rm resolved through
|
|
the wrapper) while a benign argument such as subprocess.run(['echo', 'python']) is not
|
|
misread as invoking `python`.
|
|
|
|
Returns (blocked_basenames, cmd_index): cmd_index is the position of the resolved command
|
|
word (or None), so the caller can hand a wrapper-hidden shell binary (env bash s.sh) to the
|
|
shell-argv analyzer."""
|
|
blocked: set[str] = set()
|
|
idx, n = 0, len(str_elts)
|
|
prefix_pending = False # a wrapper is awaiting its real command word
|
|
prev_was_flag = False # last token (under a wrapper) was an option flag with an operand
|
|
cur_wrapper = None # the active wrapper's basename (env / nice / timeout / ...)
|
|
while idx < n:
|
|
tok = str_elts[idx]
|
|
if tok is None:
|
|
return blocked, None # a non-literal element hides the command word; stop
|
|
# env FOO=bar assignments precede the command word.
|
|
if _ASSIGNMENT_RE.match(tok):
|
|
idx += 1
|
|
continue
|
|
if prefix_pending and tok.startswith("-"):
|
|
# env -S CMD / --split-string=CMD splits its operand into a command line, so
|
|
# scan that operand with the full command scanner (env -S 'bash -c ...').
|
|
if cur_wrapper == "env":
|
|
if tok in ("-S", "--split-string"):
|
|
_nxt = str_elts[idx + 1] if idx + 1 < n else None
|
|
if _nxt is not None:
|
|
blocked |= _find_blocked_commands(_nxt)
|
|
return blocked, None
|
|
if tok.startswith("--split-string="):
|
|
blocked |= _find_blocked_commands(tok[len("--split-string=") :])
|
|
return blocked, None
|
|
if tok.startswith("-S") and len(tok) > 2:
|
|
blocked |= _find_blocked_commands(tok[2:])
|
|
return blocked, None
|
|
# Only a flag that takes a SEPARATED operand (env -u NAME, nice -n 5) marks the
|
|
# next token as its value; a no-operand flag (env -i, xargs -0) or a glued short
|
|
# flag (stdbuf -oL) does not, so the real command after it is still analysed.
|
|
prev_was_flag = _wrapper_flag_takes_operand(cur_wrapper, tok)
|
|
idx += 1
|
|
continue
|
|
# A wrapper's numeric arg (`timeout 5 cmd`).
|
|
if prefix_pending and _is_wrapper_numeric_arg(tok):
|
|
prev_was_flag = False
|
|
idx += 1
|
|
continue
|
|
base = os.path.basename(tok).lower()
|
|
stem, ext = os.path.splitext(base)
|
|
if ext in {".exe", ".com", ".bat", ".cmd"}:
|
|
base = stem
|
|
# A wrapper flag's SEPARATED operand (`env -u FOO python3`, `env -C DIR cmd`): the
|
|
# token after a wrapper option flag that is not itself a blocked command / prefix /
|
|
# shell is the flag's value -- skip it and keep scanning so the real command (python3,
|
|
# bash) is not missed. A blocked command / prefix / shell is treated as the command.
|
|
if (
|
|
prefix_pending
|
|
and prev_was_flag
|
|
and base not in _BLOCKED_COMMANDS
|
|
and base not in _COMMAND_PREFIXES
|
|
and base not in _SHELL_BINARIES
|
|
):
|
|
prev_was_flag = False
|
|
idx += 1
|
|
continue
|
|
prev_was_flag = False
|
|
if base in _BLOCKED_COMMANDS or _is_versioned_interpreter(base):
|
|
blocked.add(base)
|
|
if base in _COMMAND_PREFIXES:
|
|
prefix_pending = True
|
|
cur_wrapper = base
|
|
idx += 1
|
|
continue # wrapper consumes one command; the next word is the real one
|
|
if _is_local_executable_path(tok):
|
|
blocked.add("local-exec:" + base) # runs an unguarded shebang interpreter
|
|
return blocked, idx # reached the executed command word
|
|
return blocked, None
|
|
|
|
|
|
def _build_safe_env(workdir: str) -> dict[str, str]:
|
|
"""Build a minimal, credential-free environment for sandboxed subprocesses.
|
|
|
|
Whitelist-built from scratch (parent env NOT inherited): only PATH/HOME/
|
|
TMPDIR/LANG/TERM/PYTHONIOENCODING (+VIRTUAL_ENV or Windows SystemRoot) reach
|
|
the child; all credential vars (HF_TOKEN, AWS_*, etc.) are absent. HOME
|
|
points at the sandbox workdir so SDKs can't read the operator's cached creds.
|
|
"""
|
|
# Start from the running interpreter's dir so 'python'/'pip' resolve to the
|
|
# same environment the Studio server runs in.
|
|
exe_dir = os.path.dirname(sys.executable)
|
|
path_entries = [exe_dir] if exe_dir else []
|
|
|
|
# If a virtualenv is active, include its bin/Scripts directory.
|
|
venv = os.environ.get("VIRTUAL_ENV")
|
|
if venv:
|
|
venv_bin = os.path.join(venv, "Scripts" if sys.platform == "win32" else "bin")
|
|
if venv_bin not in path_entries:
|
|
path_entries.append(venv_bin)
|
|
|
|
if sys.platform == "win32":
|
|
sysroot = os.environ.get("SystemRoot", r"C:\Windows")
|
|
path_entries.extend([os.path.join(sysroot, "System32"), sysroot])
|
|
else:
|
|
path_entries.extend(["/usr/local/bin", "/usr/bin", "/bin"])
|
|
|
|
# Deduplicate, preserving order.
|
|
deduped = list(dict.fromkeys(p for p in path_entries if p))
|
|
|
|
env = {
|
|
"PATH": os.pathsep.join(deduped),
|
|
"HOME": workdir,
|
|
"TMPDIR": workdir,
|
|
"LANG": os.environ.get("LANG", "C.UTF-8"),
|
|
"TERM": "dumb",
|
|
"PYTHONIOENCODING": "utf-8",
|
|
# HOME points at the workdir, so a prior run could plant
|
|
# .local/.../site-packages/usercustomize.py that runs (unguarded) at the next child's
|
|
# startup. Disable the per-user site directory here too (belt-and-suspenders with the
|
|
# interpreter's -s flag) so a sandboxed child never imports it.
|
|
"PYTHONNOUSERSITE": "1",
|
|
}
|
|
if venv:
|
|
env["VIRTUAL_ENV"] = venv
|
|
# Windows needs SystemRoot for Python/subprocess to work.
|
|
if sys.platform == "win32":
|
|
env["SystemRoot"] = os.environ.get("SystemRoot", r"C:\Windows")
|
|
return env
|
|
|
|
|
|
# Credential env vars dropped even in bypass mode so tool code cannot read the
|
|
# operator's keys. Over-strips on purpose (a benign var is harmless to lose).
|
|
_BYPASS_ENV_SECRET_NAMES = frozenset(
|
|
{
|
|
"HF_TOKEN",
|
|
"HF_HUB_TOKEN",
|
|
"HUGGING_FACE_HUB_TOKEN",
|
|
"HUGGINGFACE_TOKEN",
|
|
"HUGGINGFACEHUB_API_TOKEN",
|
|
"WANDB_API_KEY",
|
|
"GH_TOKEN",
|
|
"GITHUB_TOKEN",
|
|
"OPENAI_API_KEY",
|
|
"ANTHROPIC_API_KEY",
|
|
"GEMINI_API_KEY",
|
|
"GOOGLE_API_KEY",
|
|
"GROQ_API_KEY",
|
|
"OPENROUTER_API_KEY",
|
|
"REPLICATE_API_TOKEN",
|
|
"COHERE_API_KEY",
|
|
"MISTRAL_API_KEY",
|
|
"NGC_API_KEY",
|
|
"KAGGLE_KEY",
|
|
"MYSQL_PWD", # exact name: markers use PASSWD, not PWD (PWD is the cwd var)
|
|
"LD_PRELOAD",
|
|
# Auth brokers / capability handles: not secrets by value, but they
|
|
# hand the child the operator's live agent (ssh/gpg), kube config, or
|
|
# docker daemon. Names are listed because there is no value signal to
|
|
# key off. URL config vars (HTTP_PROXY, PIP_INDEX_URL, DATABASE_URL,
|
|
# ...) are intentionally NOT name-listed: a benign proxy/index without
|
|
# credentials must keep working in bypass mode, while a credentialed
|
|
# value is dropped by _is_secret_env_value() regardless of its name.
|
|
"SSH_AUTH_SOCK",
|
|
"SSH_AGENT_PID",
|
|
"GPG_AGENT_INFO",
|
|
"GNUPGHOME",
|
|
"KUBECONFIG",
|
|
"DOCKER_HOST",
|
|
}
|
|
)
|
|
_BYPASS_ENV_SECRET_PREFIXES = ("AWS_", "AZURE_", "GOOGLE_", "GCP_", "GCLOUD_", "DYLD_")
|
|
_BYPASS_ENV_SECRET_MARKERS = (
|
|
"TOKEN",
|
|
"API_KEY",
|
|
"APIKEY",
|
|
"SECRET",
|
|
"PASSWORD",
|
|
"PASSWD",
|
|
"CREDENTIAL",
|
|
"PRIVATE_KEY",
|
|
"AUTH", # e.g. NPM_CONFIG__AUTH (npm _auth), REDISCLI_AUTH
|
|
# Azure App Service connection strings: SQLCONNSTR_/CUSTOMCONNSTR_/... and
|
|
# WEBSITE_CONTENTAZUREFILECONNECTIONSTRING carry DB/storage credentials.
|
|
"CONNSTR",
|
|
"CONNECTIONSTRING",
|
|
)
|
|
# Non-secret hardening flags that match a secret prefix/marker but must be KEPT
|
|
# so bypass mode does not silently undo an operator's opt-out. AWS_EC2_METADATA_
|
|
# DISABLED tells the AWS SDK/CLI not to pull instance-role creds from IMDS;
|
|
# dropping it would re-open that path for a bypassed tool.
|
|
_BYPASS_ENV_KEEP_NAMES = frozenset(
|
|
{
|
|
"AWS_EC2_METADATA_DISABLED",
|
|
"AWS_EC2_METADATA_V1_DISABLED",
|
|
}
|
|
)
|
|
# Matches a URL that embeds userinfo before the host, covering both
|
|
# "scheme://user:pass@host" and token-only "scheme://token@host" (and
|
|
# percent-encoded variants). The userinfo must precede the first '/', so an '@'
|
|
# in a path or query does not false-positive. Used to scrub credential-bearing
|
|
# URL values regardless of the variable's name.
|
|
_URL_USERINFO_RE = re.compile(r"://[^/\s@]+@")
|
|
# Connection-string credential fields (ADO.NET / Azure storage / Service Bus):
|
|
# "...;Password=...", "...;AccountKey=...", "...;SharedAccessKey=...". Catches
|
|
# credential-bearing values whose names dodge the name classifier. "accesskey"
|
|
# also covers Shared/Secret AccessKey via substring; the Name fields (e.g.
|
|
# SharedAccessKeyName=) do not match since "=" must follow the keyword.
|
|
_SECRET_VALUE_RE = re.compile(r"(?i)(?:password|pwd|accountkey|accesskey)\s*=\s*[^\s;]")
|
|
|
|
# Names that hold no secret value but point SDKs at the operator's real
|
|
# home/cache/config (cached tokens, cred files), defeating the HOME repoint.
|
|
# Startup always sets HF_HOME (-> $HF_HOME/token), so this is the live leak.
|
|
# Dropped in bypass mode so tools fall back to the empty repointed HOME.
|
|
_BYPASS_ENV_CRED_LOCATION_NAMES = frozenset(
|
|
{
|
|
# HF cache roots (token lives under $HF_HOME/token)
|
|
"HF_HOME",
|
|
"HF_HUB_CACHE",
|
|
"HUGGINGFACE_HUB_CACHE",
|
|
"HF_XET_CACHE",
|
|
"TRANSFORMERS_CACHE",
|
|
"HF_DATASETS_CACHE",
|
|
"HF_ASSETS_CACHE",
|
|
# XDG base dirs (resolved before $HOME)
|
|
"XDG_CONFIG_HOME",
|
|
"XDG_CACHE_HOME",
|
|
"XDG_DATA_HOME",
|
|
# explicit cred/config file pointers honoured before $HOME
|
|
"NETRC",
|
|
"PGPASSFILE",
|
|
"BOTO_CONFIG",
|
|
"PIP_CONFIG_FILE",
|
|
"CLOUDSDK_CONFIG",
|
|
"KAGGLE_CONFIG_DIR",
|
|
"DOCKER_CONFIG",
|
|
"WANDB_DIR",
|
|
"WANDB_CONFIG_DIR",
|
|
"WANDB_CACHE_DIR",
|
|
# package-manager / git / cloud config pointers to real cred files
|
|
"NPM_CONFIG_USERCONFIG",
|
|
"NPM_CONFIG_GLOBALCONFIG",
|
|
"YARN_RC_FILENAME",
|
|
"GIT_CONFIG_GLOBAL",
|
|
"GIT_CONFIG_SYSTEM",
|
|
"CARGO_HOME",
|
|
"RCLONE_CONFIG",
|
|
# auth-helper scripts that hand creds to git/ssh
|
|
"GIT_ASKPASS",
|
|
"SSH_ASKPASS",
|
|
# shell startup hook: bash -c sources $BASH_ENV (can re-export secrets)
|
|
"BASH_ENV",
|
|
# Windows: HOMEDRIVE+HOMEPATH compose a home that bypasses HOME
|
|
"HOMEDRIVE",
|
|
"HOMEPATH",
|
|
}
|
|
)
|
|
# Windows profile dirs SDKs read creds under; repointed (not dropped) since
|
|
# callers expect them present.
|
|
_BYPASS_ENV_WINDOWS_PROFILE_VARS = ("USERPROFILE", "APPDATA", "LOCALAPPDATA")
|
|
|
|
|
|
def _is_secret_env_name(name: str) -> bool:
|
|
"""True if an env var name looks like it carries a credential."""
|
|
upper = name.upper()
|
|
if upper in _BYPASS_ENV_KEEP_NAMES:
|
|
return False # non-secret hardening flag; keep it
|
|
if upper in _BYPASS_ENV_SECRET_NAMES:
|
|
return True
|
|
if any(upper.startswith(p) for p in _BYPASS_ENV_SECRET_PREFIXES):
|
|
return True
|
|
return any(marker in upper for marker in _BYPASS_ENV_SECRET_MARKERS)
|
|
|
|
|
|
def _is_cred_location_env_name(name: str) -> bool:
|
|
"""True for vars that point SDKs at the real home/cache/config (cached creds)."""
|
|
return name.upper() in _BYPASS_ENV_CRED_LOCATION_NAMES
|
|
|
|
|
|
def _is_secret_env_value(value: str) -> bool:
|
|
"""True if a value embeds credentials regardless of its name.
|
|
|
|
Catches URL userinfo (``scheme://user:token@host`` in DATABASE_URL /
|
|
PIP_INDEX_URL / HTTP_PROXY) and connection-string credential fields
|
|
(``...;Password=...`` / ``...;AccountKey=...``) whose names dodge the name
|
|
classifier.
|
|
"""
|
|
if not value:
|
|
return False
|
|
return _URL_USERINFO_RE.search(value) is not None or _SECRET_VALUE_RE.search(value) is not None
|
|
|
|
|
|
def _build_bypass_env(workdir: str) -> dict[str, str]:
|
|
"""Env for bypass exec: full host env (unrestricted) minus credential vars,
|
|
with HOME/TMPDIR repointed at the workdir so SDKs cannot read cached creds.
|
|
|
|
Note: stripping the child env is necessary but not sufficient on its own -
|
|
a same-UID child can still read the parent's environment via procfs, so
|
|
callers also harden the parent (see _harden_parent_against_proc_env_leak).
|
|
"""
|
|
env = {
|
|
k: v
|
|
for k, v in os.environ.items()
|
|
if not _is_secret_env_name(k)
|
|
and not _is_secret_env_value(v)
|
|
and not _is_cred_location_env_name(k)
|
|
}
|
|
env["HOME"] = workdir
|
|
env["TMPDIR"] = workdir
|
|
# Windows tempfile / SDKs honour TEMP/TMP, not TMPDIR; repoint all three so
|
|
# the bypassed tool writes under the per-session sandbox dir on every OS.
|
|
env["TEMP"] = workdir
|
|
env["TMP"] = workdir
|
|
# Windows SDKs read creds under the profile dirs, not $HOME; repoint set
|
|
# ones to the workdir (HOMEDRIVE/HOMEPATH are dropped above).
|
|
for var in _BYPASS_ENV_WINDOWS_PROFILE_VARS:
|
|
if var in os.environ:
|
|
env[var] = workdir
|
|
return env
|
|
|
|
|
|
def _sandbox_preexec():
|
|
"""Best-effort sandbox setup for sandboxed subprocesses (modules are
|
|
resolved at import time so the forked child runs no imports)."""
|
|
try:
|
|
os.setsid()
|
|
except OSError:
|
|
pass
|
|
|
|
try:
|
|
os.umask(0o077)
|
|
except OSError:
|
|
pass
|
|
|
|
if _libc is not None:
|
|
try:
|
|
_libc.prctl(38, 1, 0, 0, 0) # PR_SET_NO_NEW_PRIVS
|
|
except (OSError, AttributeError):
|
|
pass
|
|
|
|
try:
|
|
_libc.prctl(1, 9, 0, 0, 0) # PR_SET_PDEATHSIG = SIGKILL
|
|
except (OSError, AttributeError):
|
|
pass
|
|
|
|
# CLONE_NEWNET not applied: with userns enabled it blocks all egress,
|
|
# including allowlisted hosts. Network policy is enforced by the AST
|
|
# host check and the bash blocklist.
|
|
|
|
if _resource is not None:
|
|
# RLIMIT_NPROC is per-real-UID, so the cap is well above normal usage.
|
|
try:
|
|
nproc = int(os.environ.get("UNSLOTH_STUDIO_SANDBOX_NPROC", "10000"))
|
|
_resource.setrlimit(_resource.RLIMIT_NPROC, (nproc, nproc))
|
|
except (ValueError, OSError, AttributeError):
|
|
pass
|
|
try:
|
|
_resource.setrlimit(_resource.RLIMIT_FSIZE, (100 * 1024 * 1024, 100 * 1024 * 1024))
|
|
except (ValueError, OSError):
|
|
pass
|
|
try:
|
|
as_bytes = int(os.environ.get("UNSLOTH_STUDIO_SANDBOX_AS_GB", "8")) * 1024 * 1024 * 1024
|
|
_resource.setrlimit(_resource.RLIMIT_AS, (as_bytes, as_bytes))
|
|
except (ValueError, OSError, AttributeError):
|
|
pass
|
|
try:
|
|
cpu_s = int(os.environ.get("UNSLOTH_STUDIO_SANDBOX_CPU_S", "600"))
|
|
_resource.setrlimit(_resource.RLIMIT_CPU, (cpu_s, cpu_s))
|
|
except (ValueError, OSError, AttributeError):
|
|
pass
|
|
try:
|
|
# High enough for multi-shard safetensors mmaps; tunable via env.
|
|
# Clamp to the inherited hard limit so setrlimit doesn't ValueError
|
|
# when the parent's hard cap is below the request.
|
|
nofile = int(os.environ.get("UNSLOTH_STUDIO_SANDBOX_NOFILE", "16384"))
|
|
_soft_cur, hard_cur = _resource.getrlimit(_resource.RLIMIT_NOFILE)
|
|
target = nofile if hard_cur == _resource.RLIM_INFINITY else min(nofile, hard_cur)
|
|
_resource.setrlimit(_resource.RLIMIT_NOFILE, (target, target))
|
|
except (ValueError, OSError, AttributeError):
|
|
pass
|
|
|
|
|
|
def _bypass_preexec():
|
|
"""Minimal pre-exec for bypass exec: os.setsid() only.
|
|
|
|
Required, not a restriction: _kill_process_tree does killpg(getpgid(child)),
|
|
so without a new session a timeout/cancel would kill the Studio server too.
|
|
"""
|
|
try:
|
|
os.setsid()
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
# Hardening the Studio parent is done once (PR_SET_DUMPABLE is process-global
|
|
# and sticky); guarded so repeated bypass calls do not re-issue the prctl.
|
|
_parent_proc_hardened = False
|
|
|
|
|
|
def _harden_parent_against_proc_env_leak() -> bool:
|
|
"""Make the Studio process's /proc/<pid>/environ unreadable to its children.
|
|
|
|
Stripping the child env is not enough on Linux: a bypassed same-UID child
|
|
runs unsandboxed and can read /proc/<getppid()>/environ to recover the
|
|
tool-executing process's *unfiltered* secrets (HF_TOKEN, cloud keys, ...).
|
|
Clearing the dumpable flag (PR_SET_DUMPABLE=0) reparents this process's
|
|
/proc entries to root, so a same-UID child can no longer read its environ.
|
|
|
|
Returns True when the process is hardened or hardening is unnecessary (no
|
|
/proc leak off Linux), and False when it is needed but could not be applied
|
|
(e.g. prctl denied by a seccomp policy). Callers must fail closed - refuse
|
|
the unsandboxed exec - when this returns False, rather than running with the
|
|
parent environ still readable.
|
|
|
|
Scope: this closes the direct parent read (the demonstrated leak). It is a
|
|
mitigation, not a full boundary - a bypassed tool is unsandboxed by design,
|
|
so it can still walk /proc to a same-UID *ancestor* (e.g. the launching
|
|
shell) or read on-disk credentials by absolute path. Complete isolation
|
|
needs a separate uid / PID+mount namespace, which is out of scope here; the
|
|
UI already warns the mode is dangerous. Applied lazily on first bypass exec
|
|
so non-bypass operation is unchanged.
|
|
"""
|
|
global _parent_proc_hardened
|
|
if _parent_proc_hardened:
|
|
return True
|
|
if sys.platform != "linux":
|
|
return True # no /proc/<pid>/environ same-UID leak to close
|
|
if _libc is None:
|
|
return False # on Linux but cannot issue prctl -> cannot harden
|
|
try:
|
|
# prctl(PR_SET_DUMPABLE=4, SUID_DUMP_DISABLE=0). ctypes returns the
|
|
# syscall result (-1 on failure) and does NOT raise, so check it.
|
|
ret = _libc.prctl(4, 0, 0, 0, 0)
|
|
except (OSError, AttributeError):
|
|
return False
|
|
if ret != 0:
|
|
return False
|
|
_parent_proc_hardened = True
|
|
return True
|
|
|
|
|
|
def _get_shell_cmd(command: str) -> list[str]:
|
|
"""Return the platform-appropriate shell invocation for a command string."""
|
|
if sys.platform == "win32":
|
|
return ["cmd", "/c", command]
|
|
return ["bash", "-c", command]
|
|
|
|
|
|
# Per-session working directories so each chat thread gets its own sandbox.
|
|
# Falls back to ~/studio_sandbox/_default for callers without a session_id.
|
|
_workdirs: dict[str, str] = {}
|
|
|
|
|
|
# Non-matching session_ids collapse to ``_invalid`` to block cross-session escapes.
|
|
_SESSION_ID_RE = re.compile(r"\A[A-Za-z0-9_\-]{1,64}\Z")
|
|
_PROJECT_SESSION_PREFIX = "project-"
|
|
|
|
|
|
def _get_project_workdir(session_id: str) -> str | None:
|
|
if not session_id.startswith(_PROJECT_SESSION_PREFIX):
|
|
return None
|
|
project_id = session_id[len(_PROJECT_SESSION_PREFIX) :]
|
|
if not project_id or not _SESSION_ID_RE.match(project_id):
|
|
return None
|
|
try:
|
|
from storage.studio_db import ensure_chat_project_workspace
|
|
project = ensure_chat_project_workspace(project_id)
|
|
except Exception:
|
|
logger.warning("Failed to resolve project sandbox for %s", session_id, exc_info = True)
|
|
return None
|
|
if not project:
|
|
return None
|
|
root_path = project.get("rootPath")
|
|
sandbox_path = project.get("sandboxPath")
|
|
if not root_path or not sandbox_path:
|
|
return None
|
|
root_real = os.path.realpath(root_path)
|
|
sandbox_real = os.path.realpath(sandbox_path)
|
|
if sandbox_real != root_real and not sandbox_real.startswith(root_real + os.sep):
|
|
return None
|
|
return sandbox_real
|
|
|
|
|
|
def _get_workdir(session_id: str | None = None) -> str:
|
|
"""Return a per-session sandbox dir at mode 0o700."""
|
|
global _workdirs
|
|
key = session_id or "_default"
|
|
if key not in _workdirs or not os.path.isdir(_workdirs[key]):
|
|
home = os.path.expanduser("~")
|
|
sandbox_root = os.path.join(home, "studio_sandbox")
|
|
project_workdir = (
|
|
_get_project_workdir(session_id)
|
|
if session_id and _SESSION_ID_RE.match(session_id)
|
|
else None
|
|
)
|
|
if project_workdir:
|
|
workdir = project_workdir
|
|
elif session_id and _SESSION_ID_RE.match(session_id):
|
|
workdir = os.path.join(sandbox_root, session_id)
|
|
if not os.path.realpath(workdir).startswith(os.path.realpath(sandbox_root) + os.sep):
|
|
workdir = os.path.join(sandbox_root, "_invalid")
|
|
elif session_id:
|
|
workdir = os.path.join(sandbox_root, "_invalid")
|
|
else:
|
|
workdir = os.path.join(sandbox_root, "_default")
|
|
os.makedirs(workdir, exist_ok = True)
|
|
try:
|
|
os.chmod(sandbox_root, 0o700)
|
|
except OSError:
|
|
pass
|
|
try:
|
|
os.chmod(workdir, 0o700)
|
|
except OSError:
|
|
pass
|
|
_workdirs[key] = workdir
|
|
return _workdirs[key]
|
|
|
|
|
|
def get_sandbox_workdir(session_id: str | None = None) -> str:
|
|
return _get_workdir(session_id)
|
|
|
|
|
|
WEB_SEARCH_TOOL = {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "web_search",
|
|
"description": (
|
|
"Search the web and fetch page content. Returns snippets for all results. "
|
|
"Use the url parameter to fetch full page text from a specific URL."
|
|
),
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"query": {
|
|
"type": "string",
|
|
"description": "The search query",
|
|
},
|
|
"url": {
|
|
"type": "string",
|
|
"description": "A URL to fetch full page content from (instead of searching). Use this to read a page found in search results.",
|
|
},
|
|
},
|
|
"required": [],
|
|
},
|
|
},
|
|
}
|
|
|
|
PYTHON_TOOL = {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "python",
|
|
"description": "Execute Python code in a sandbox and return stdout/stderr.",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"code": {
|
|
"type": "string",
|
|
"description": "The Python code to run",
|
|
}
|
|
},
|
|
"required": ["code"],
|
|
},
|
|
},
|
|
}
|
|
|
|
TERMINAL_TOOL = {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "terminal",
|
|
"description": "Execute a terminal command and return stdout/stderr.",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"command": {
|
|
"type": "string",
|
|
"description": "The command to run",
|
|
}
|
|
},
|
|
"required": ["command"],
|
|
},
|
|
},
|
|
}
|
|
|
|
RENDER_HTML_TOOL = {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "render_html",
|
|
"description": (
|
|
"Render a self-contained HTML/CSS/JavaScript canvas for the user. "
|
|
"Call this at most once per assistant response unless the user "
|
|
"explicitly asks for changes in that response. Future user requests "
|
|
"for new canvases may call render_html once. Put the entire document "
|
|
"in code, including any CSS in <style> tags and JavaScript in <script> tags."
|
|
),
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"code": {
|
|
"type": "string",
|
|
"description": "A complete self-contained HTML document.",
|
|
},
|
|
"title": {
|
|
"type": "string",
|
|
"description": "Short display title for the canvas.",
|
|
},
|
|
},
|
|
"required": ["code"],
|
|
},
|
|
},
|
|
}
|
|
|
|
# Duplicated (not imported from core.rag.tool) so the registry never pulls in
|
|
# the RAG stack; dispatch imports it lazily.
|
|
SEARCH_KNOWLEDGE_BASE_TOOL = {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "search_knowledge_base",
|
|
"description": (
|
|
"Search the user's uploaded documents and knowledge bases for "
|
|
"relevant passages. Use this whenever the question may be answered "
|
|
"by the attached documents, then cite the returned chunks."
|
|
),
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"query": {
|
|
"type": "string",
|
|
"description": "Natural-language search query.",
|
|
},
|
|
"top_k": {
|
|
"type": "integer",
|
|
"description": "Max chunks to return.",
|
|
},
|
|
},
|
|
"required": ["query"],
|
|
},
|
|
},
|
|
}
|
|
|
|
ALL_TOOLS = [
|
|
WEB_SEARCH_TOOL,
|
|
PYTHON_TOOL,
|
|
TERMINAL_TOOL,
|
|
RENDER_HTML_TOOL,
|
|
SEARCH_KNOWLEDGE_BASE_TOOL,
|
|
]
|
|
|
|
|
|
# OpenAI's function.name regex ^[a-zA-Z0-9_-]{1,64}$, enforced before streaming.
|
|
# MCP tool names with '.', '/', spaces, etc. would 400 the whole request, so we
|
|
# validate up front and skip with a warning.
|
|
_OPENAI_FN_NAME_RE = re.compile(r"^[a-zA-Z0-9_-]{1,64}$")
|
|
|
|
|
|
def _mcp_specs_for_server(server: dict, mcp_tools: list[dict]) -> list[dict]:
|
|
"""Convert an MCP server's tool list into OpenAI function specs."""
|
|
display = server.get("display_name") or server["id"]
|
|
specs: list[dict] = []
|
|
seen_names: set[str] = set()
|
|
for tool in mcp_tools:
|
|
raw_name = tool.get("name") or ""
|
|
if not raw_name:
|
|
logger.warning("Skipping MCP tool on '%s': empty name.", display)
|
|
continue
|
|
name = f"{MCP_TOOL_PREFIX}{server['id']}__{raw_name}"
|
|
# Bad chars or oversized names would 400 the whole request; skip + warn
|
|
# so the rest of the tools still ship.
|
|
if not _OPENAI_FN_NAME_RE.fullmatch(name):
|
|
logger.warning(
|
|
"Skipping MCP tool '%s' on '%s': composed name '%s' is not "
|
|
"valid OpenAI function.name (regex ^[a-zA-Z0-9_-]{1,64}$).",
|
|
raw_name,
|
|
display,
|
|
name,
|
|
)
|
|
continue
|
|
# Duplicate tool names would also 400 OpenAI; drop dupes.
|
|
if name in seen_names:
|
|
logger.warning("Skipping duplicate MCP tool '%s' on '%s'.", raw_name, display)
|
|
continue
|
|
seen_names.add(name)
|
|
specs.append(
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": name,
|
|
"description": f"[{display}] {tool.get('description') or ''}".strip(),
|
|
"parameters": tool.get("inputSchema") or {"type": "object", "properties": {}},
|
|
},
|
|
}
|
|
)
|
|
return specs
|
|
|
|
|
|
async def get_enabled_mcp_tools() -> list[dict]:
|
|
servers = [s for s in mcp_servers_db.list_servers() if s.get("is_enabled")]
|
|
# Never spawn stdio servers when stdio is disabled on this host (e.g. a DB
|
|
# carried from a desktop install onto a Colab/network deployment).
|
|
if not stdio_mcp_enabled():
|
|
servers = [s for s in servers if not is_stdio(s["url"])]
|
|
if not servers:
|
|
return []
|
|
|
|
# Skip servers still in their post-failure cool-off, otherwise a down
|
|
# server gets re-probed -- and blocks the send for the full timeout -- on
|
|
# every message.
|
|
uncached = [
|
|
s for s in servers if get_cached_tools(s["id"]) is None and not in_failure_cooloff(s["id"])
|
|
]
|
|
if uncached:
|
|
results = await asyncio.gather(
|
|
*(
|
|
list_tools_async(
|
|
url = s["url"],
|
|
headers = parse_server_headers(s),
|
|
timeout = probe_timeout(s["url"], bool(s.get("use_oauth"))),
|
|
use_oauth = bool(s.get("use_oauth")),
|
|
)
|
|
for s in uncached
|
|
),
|
|
return_exceptions = True,
|
|
)
|
|
# An edit/delete can land while we await a probe (up to 305 s for
|
|
# OAuth); its cache eviction is a no-op against an entry we haven't
|
|
# written yet. Re-read and drop a result whose server changed or
|
|
# was removed mid-probe, else a stale tool list caches indefinitely.
|
|
current = {s["id"]: s for s in mcp_servers_db.list_servers()}
|
|
for server, payload in zip(uncached, results):
|
|
# Guard the failure branch too: a stale failure must not park a
|
|
# cool-off on the fresh config, or the server the user just fixed
|
|
# is skipped for the whole window.
|
|
fresh = current.get(server["id"])
|
|
if fresh is None or any(
|
|
fresh.get(k) != server.get(k) for k in TOOL_CACHE_INVALIDATING_FIELDS
|
|
):
|
|
continue
|
|
if isinstance(payload, BaseException):
|
|
logger.warning(
|
|
"MCP server '%s' (%s) discovery failed: %s",
|
|
server.get("display_name") or server["id"],
|
|
server.get("url"),
|
|
payload,
|
|
)
|
|
# Failures aren't cached, but record one so a down server
|
|
# isn't re-probed every send during the cool-off.
|
|
record_probe_failure(server["id"], bool(fresh.get("use_oauth")))
|
|
continue
|
|
cache_tools(server["id"], payload)
|
|
|
|
specs: list[dict] = []
|
|
for server in servers:
|
|
payload = get_cached_tools(server["id"])
|
|
if payload is None:
|
|
continue
|
|
specs.extend(_mcp_specs_for_server(server, payload))
|
|
return specs
|
|
|
|
|
|
_TIMEOUT_UNSET = object()
|
|
|
|
|
|
def _render_html_result(arguments: dict) -> str:
|
|
code = arguments.get("code")
|
|
if not isinstance(code, str) or not code.strip():
|
|
return "Error: render_html requires a non-empty code string."
|
|
title = arguments.get("title")
|
|
if isinstance(title, str) and title.strip():
|
|
safe_title = title.strip()[:120]
|
|
return (
|
|
f"Rendered HTML canvas: {safe_title}. Do not call render_html "
|
|
"again in this response unless the user asks for changes. For a later "
|
|
"user request for a new canvas, call render_html once."
|
|
)
|
|
return (
|
|
"Rendered HTML canvas. Do not call render_html again in this response "
|
|
"unless the user asks for changes. For a later user request for a new "
|
|
"canvas, call render_html once."
|
|
)
|
|
|
|
|
|
def execute_tool(
|
|
name: str,
|
|
arguments: dict,
|
|
cancel_event = None,
|
|
timeout: int | None = _TIMEOUT_UNSET,
|
|
session_id: str | None = None,
|
|
rag_scope: dict | None = None,
|
|
disable_sandbox: bool = False,
|
|
) -> str:
|
|
"""Execute a tool by name with the given arguments; returns a string.
|
|
|
|
``timeout``: int seconds, ``None`` = no limit, unset = ``_EXEC_TIMEOUT``.
|
|
``session_id``: optional ID for per-conversation sandbox isolation.
|
|
``rag_scope``: hidden per-request RAG context the model never sees; consumed
|
|
by ``search_knowledge_base``.
|
|
``disable_sandbox``: Bypass Permissions; run python/terminal without the
|
|
safety checks, blocklist, or resource caps (secrets still stripped). Only
|
|
affects local code tools; web_search / MCP are unchanged.
|
|
"""
|
|
logger.info(f"execute_tool: name={name}, session_id={session_id}, timeout={timeout}")
|
|
effective_timeout = _EXEC_TIMEOUT if timeout is _TIMEOUT_UNSET else timeout
|
|
if name == "search_knowledge_base":
|
|
return _search_knowledge_base(arguments, rag_scope)
|
|
if name == "render_html":
|
|
return _render_html_result(arguments)
|
|
if name.startswith(MCP_TOOL_PREFIX):
|
|
try:
|
|
_, server_id, tool_name = name.split("__", 2)
|
|
except ValueError:
|
|
return f"Error: malformed MCP tool name '{name}'"
|
|
server = mcp_servers_db.get_server(server_id)
|
|
if not server:
|
|
return f"Error: MCP server '{server_id}' not found"
|
|
if not server.get("is_enabled"):
|
|
return f"Error: MCP server '{server_id}' is disabled"
|
|
if is_stdio(server["url"]) and not stdio_mcp_enabled():
|
|
return f"Error: stdio MCP server '{server_id}' is disabled on this host"
|
|
return call_tool_sync(
|
|
url = server["url"],
|
|
headers = parse_server_headers(server),
|
|
name = tool_name,
|
|
args = arguments,
|
|
timeout = effective_timeout,
|
|
use_oauth = bool(server.get("use_oauth")),
|
|
cancel_event = cancel_event,
|
|
)
|
|
if name == "web_search":
|
|
return _web_search(
|
|
arguments.get("query", ""),
|
|
url = arguments.get("url"),
|
|
timeout = effective_timeout,
|
|
)
|
|
if name == "python":
|
|
return _python_exec(
|
|
arguments.get("code", ""),
|
|
cancel_event,
|
|
effective_timeout,
|
|
session_id,
|
|
disable_sandbox = disable_sandbox,
|
|
)
|
|
if name == "terminal":
|
|
return _bash_exec(
|
|
arguments.get("command", ""),
|
|
cancel_event,
|
|
effective_timeout,
|
|
session_id,
|
|
disable_sandbox = disable_sandbox,
|
|
)
|
|
return f"Unknown tool: {name}"
|
|
|
|
|
|
def _opt_int(v) -> int | None:
|
|
try:
|
|
return int(v) if v is not None else None
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _scope_retrieval_kwargs(scope: dict) -> dict:
|
|
"""Retrieval mode from rag_scope; candidate pools and RRF come from config."""
|
|
mode = scope.get("mode")
|
|
return {"mode": mode if mode in ("hybrid", "dense", "lexical") else "hybrid"}
|
|
|
|
|
|
def _search_knowledge_base(arguments: dict, rag_scope: dict | None) -> str:
|
|
"""Run the RAG search bound to the hidden per-request ``rag_scope`` (the model
|
|
supplies only ``query``/``top_k``). Lazy import; missing sqlite-vec degrades
|
|
to a friendly message."""
|
|
scope = rag_scope or {}
|
|
query = (arguments or {}).get("query", "")
|
|
if not query or not str(query).strip():
|
|
return "Error: query is empty."
|
|
try:
|
|
from storage import rag_db
|
|
if not rag_db.RAG_AVAILABLE:
|
|
return "Knowledge base search is unavailable on this server."
|
|
from core.rag.tool import search_knowledge_base_with_sources
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("RAG tool unavailable: %s", exc)
|
|
return "Knowledge base search is unavailable on this server."
|
|
|
|
top_k = _opt_int((arguments or {}).get("top_k") or scope.get("default_top_k"))
|
|
text, sources = search_knowledge_base_with_sources(
|
|
query = str(query),
|
|
scope_kb_id = scope.get("kb_id"),
|
|
scope_thread_id = scope.get("thread_id"),
|
|
scope_project_id = scope.get("project_id"),
|
|
top_k = top_k,
|
|
**_scope_retrieval_kwargs(scope),
|
|
)
|
|
# Append the UI source-map after the sentinel; loops strip it before the model.
|
|
if sources:
|
|
import json as _json
|
|
return text + RAG_SOURCES_SENTINEL + _json.dumps(sources, ensure_ascii = False)
|
|
return text
|
|
|
|
|
|
# Forced first-pass RAG retrieval: a high cosine floor keeps it precise (fires on
|
|
# on-topic queries, skips weak ones) and helps small models that under-call the tool.
|
|
# Tunable via RAG_AUTOINJECT_MIN_SCORE.
|
|
_AUTOINJECT_DEFAULT_FLOOR = 0.70
|
|
|
|
|
|
def _autoinject_enabled() -> bool:
|
|
return os.environ.get("RAG_AUTOINJECT", "1").strip().lower() not in (
|
|
"0",
|
|
"false",
|
|
"no",
|
|
"off",
|
|
)
|
|
|
|
|
|
def _autoinject_floor() -> float:
|
|
raw = os.environ.get("RAG_AUTOINJECT_MIN_SCORE")
|
|
if raw is not None:
|
|
try:
|
|
return float(raw)
|
|
except ValueError:
|
|
pass
|
|
return _AUTOINJECT_DEFAULT_FLOOR
|
|
|
|
|
|
# Lean: injecting the full top_k every turn prefills thousands of tokens.
|
|
_AUTOINJECT_DEFAULT_TOP_K = 4
|
|
|
|
|
|
def _autoinject_top_k() -> int:
|
|
raw = os.environ.get("RAG_AUTOINJECT_TOP_K")
|
|
if raw is not None:
|
|
try:
|
|
return max(1, int(raw))
|
|
except ValueError:
|
|
pass
|
|
return _AUTOINJECT_DEFAULT_TOP_K
|
|
|
|
|
|
def _thread_whole_doc_enabled(scope: dict) -> bool:
|
|
"""Whether a thread-attached file should be injected in full rather than
|
|
retrieved top-K. ``rag_scope.whole_doc=False`` disables it for this request."""
|
|
override = scope.get("whole_doc")
|
|
if override is False:
|
|
return False
|
|
try:
|
|
from core.rag import config as _rag_config
|
|
except Exception: # noqa: BLE001
|
|
return True
|
|
return _rag_config.THREAD_WHOLE_DOC
|
|
|
|
|
|
_IMAGE_PART_TOKEN_ESTIMATE = 1024
|
|
|
|
|
|
def _message_token_estimate(conversation: list[dict]) -> int:
|
|
"""Cheap prompt-size estimate for budget guards; exact tokenization happens later."""
|
|
total = 0
|
|
for msg in conversation:
|
|
content = msg.get("content")
|
|
if isinstance(content, str):
|
|
total += max(1, len(content) // 4)
|
|
elif isinstance(content, list):
|
|
for part in content:
|
|
if isinstance(part, dict):
|
|
if part.get("type") in ("image_url", "input_image"):
|
|
total += _IMAGE_PART_TOKEN_ESTIMATE
|
|
else:
|
|
total += max(1, len(str(part.get("text") or "")) // 4)
|
|
total += 4 # chat-template role / separator overhead estimate
|
|
return total
|
|
|
|
|
|
def _whole_doc_budget(scope: dict | None = None, conversation: list[dict] | None = None) -> int:
|
|
try:
|
|
from core.rag import config as _rag_config
|
|
except Exception: # noqa: BLE001
|
|
budget = 6000
|
|
else:
|
|
budget = _rag_config.WHOLE_DOC_MAX_TOKENS
|
|
if not scope:
|
|
return budget
|
|
context = _opt_int(scope.get("context_length") or scope.get("max_context_tokens"))
|
|
if context is None or context <= 0:
|
|
return budget
|
|
headroom = _opt_int(scope.get("response_headroom"))
|
|
if headroom is None:
|
|
headroom = max(1024, context // 4)
|
|
used = _message_token_estimate(conversation or [])
|
|
# Leave room for tool XML wrappers, citation metadata, and chat-template overhead.
|
|
available = context - headroom - used - 512
|
|
return min(budget, max(0, available))
|
|
|
|
|
|
def _last_user_text(conversation: list[dict]) -> str:
|
|
"""Plain text of the most recent user turn (text parts only)."""
|
|
for msg in reversed(conversation):
|
|
if msg.get("role") != "user":
|
|
continue
|
|
content = msg.get("content")
|
|
if isinstance(content, str):
|
|
return content.strip()
|
|
if isinstance(content, list):
|
|
parts = [
|
|
p.get("text", "")
|
|
for p in content
|
|
if isinstance(p, dict) and p.get("type") in ("text", "input_text")
|
|
]
|
|
return " ".join(t for t in parts if t).strip()
|
|
return ""
|
|
return ""
|
|
|
|
|
|
def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> dict | None:
|
|
"""Pre-retrieve the latest user turn; if a hit clears the cosine floor return
|
|
``{"events": [...], "messages": [...]}`` to splice into the loop, else ``None``.
|
|
Toggle via ``rag_scope.autoinject`` (else env ``RAG_AUTOINJECT``); floor via
|
|
``rag_scope.autoinject_min_score`` (else env ``RAG_AUTOINJECT_MIN_SCORE``).
|
|
|
|
Also the small-model fallback: models below ~4B often answer from memory
|
|
instead of calling ``search_knowledge_base``, so forcing retrieval here keeps
|
|
attachments consulted regardless of model size."""
|
|
if not rag_scope:
|
|
return None
|
|
enabled = rag_scope.get("autoinject")
|
|
if enabled is None:
|
|
enabled = _autoinject_enabled()
|
|
thread_id = rag_scope.get("thread_id")
|
|
whole_doc_requested = (
|
|
bool(thread_id) and not rag_scope.get("kb_id") and _thread_whole_doc_enabled(rag_scope)
|
|
)
|
|
if not enabled and not whole_doc_requested:
|
|
return None
|
|
query = _last_user_text(conversation)
|
|
if not query:
|
|
return None
|
|
try:
|
|
from storage import rag_db
|
|
if not rag_db.RAG_AVAILABLE:
|
|
return None
|
|
from core.rag.tool import render_sources, search_for_autoinject, whole_document_context
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("RAG auto-inject unavailable: %s", exc)
|
|
return None
|
|
|
|
text: str | None = None
|
|
sources: list[dict] = []
|
|
|
|
floor_override = rag_scope.get("autoinject_min_score")
|
|
floor = float(floor_override) if floor_override is not None else _autoinject_floor()
|
|
# Cap at the lean top_k, but honor a lower user setting.
|
|
lean_k = _autoinject_top_k()
|
|
sidebar_k = _opt_int(rag_scope.get("default_top_k"))
|
|
top_k = min(sidebar_k, lean_k) if sidebar_k is not None else lean_k
|
|
|
|
# Whole-document mode: a thread-attached file under budget is injected in full so
|
|
# the model reads everything. A KB selection is exclusive, so whole-doc never
|
|
# preempts it; in a project chat the project sources are still retrieved top-K and
|
|
# appended under one citation numbering. Oversized files (or no thread doc) fall
|
|
# through to the combined top-K retrieval below.
|
|
if whole_doc_requested:
|
|
try:
|
|
budget = _whole_doc_budget(rag_scope, conversation)
|
|
|
|
whole = whole_document_context(
|
|
scope_thread_id = thread_id,
|
|
max_tokens = budget,
|
|
)
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("RAG whole-document context failed: %s", exc)
|
|
whole = None
|
|
if whole is not None:
|
|
text, sources = whole
|
|
project_id = rag_scope.get("project_id")
|
|
if project_id:
|
|
try:
|
|
proj = search_for_autoinject(
|
|
query = query,
|
|
scope_project_id = project_id,
|
|
top_k = top_k,
|
|
min_dense_score = floor,
|
|
**_scope_retrieval_kwargs(rag_scope),
|
|
)
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("RAG project retrieval (whole-doc companion) failed: %s", exc)
|
|
proj = None
|
|
if proj is not None:
|
|
merged = sources + proj[1]
|
|
merged_text = render_sources(merged)
|
|
if max(1, len(merged_text) // 4) <= budget:
|
|
sources = merged
|
|
text = merged_text
|
|
logger.info("RAG auto-inject: whole-document context (%d chunk(s))", len(sources))
|
|
|
|
if text is None and enabled:
|
|
try:
|
|
found = search_for_autoinject(
|
|
query = query,
|
|
scope_kb_id = rag_scope.get("kb_id"),
|
|
scope_thread_id = rag_scope.get("thread_id"),
|
|
scope_project_id = rag_scope.get("project_id"),
|
|
top_k = top_k,
|
|
min_dense_score = floor,
|
|
**_scope_retrieval_kwargs(rag_scope),
|
|
)
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("RAG auto-inject retrieval failed: %s", exc)
|
|
return None
|
|
if not found:
|
|
logger.info("RAG auto-inject: no passage >= %.2f; skipping", floor)
|
|
return None
|
|
text, sources = found
|
|
if text is None:
|
|
return None
|
|
|
|
import json as _json
|
|
import uuid as _uuid
|
|
|
|
call_id = "rag_auto_" + _uuid.uuid4().hex[:12]
|
|
args = {"query": query}
|
|
full_result = text + RAG_SOURCES_SENTINEL + _json.dumps(sources, ensure_ascii = False)
|
|
events = [
|
|
{"type": "status", "text": f"Searching documents: {query[:60]}"},
|
|
{
|
|
"type": "tool_start",
|
|
"tool_name": "search_knowledge_base",
|
|
"tool_call_id": call_id,
|
|
"arguments": args,
|
|
},
|
|
{
|
|
"type": "tool_end",
|
|
"tool_name": "search_knowledge_base",
|
|
"tool_call_id": call_id,
|
|
"result": full_result,
|
|
},
|
|
{"type": "status", "text": ""},
|
|
]
|
|
messages = [
|
|
{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{
|
|
"id": call_id,
|
|
"type": "function",
|
|
"function": {
|
|
"name": "search_knowledge_base",
|
|
"arguments": _json.dumps(args, ensure_ascii = False),
|
|
},
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"role": "tool",
|
|
"name": "search_knowledge_base",
|
|
"tool_call_id": call_id,
|
|
"content": text,
|
|
},
|
|
]
|
|
logger.info("RAG auto-inject: %d passage(s) for %r", len(sources), query[:80])
|
|
return {"events": events, "messages": messages}
|
|
|
|
|
|
_MAX_PAGE_CHARS = 16000 # cap fetched page text (after HTML-to-MD conversion)
|
|
# Raw download cap > _MAX_PAGE_CHARS because SSR pages embed large <head>
|
|
# sections stripped during conversion; 512 KB reaches article content even
|
|
# where <head> alone is ~200 KB.
|
|
_MAX_FETCH_BYTES = 512 * 1024
|
|
|
|
_USER_AGENTS = (
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15",
|
|
)
|
|
|
|
_tls_ctx = ssl.create_default_context()
|
|
|
|
|
|
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
return None
|
|
|
|
|
|
class _PinnedHTTPSConnection(http.client.HTTPSConnection):
|
|
"""HTTPS connection to a pinned IP, using a different hostname for SNI and
|
|
cert verification.
|
|
|
|
SSRF IP-pinning rewrites URLs to raw IPs; a normal HTTPSConnection would then
|
|
send no SNI and verify the cert against the IP (both fail). This splits the
|
|
concerns: TCP connects to the pinned IP (``host``), TLS uses ``sni_hostname``.
|
|
"""
|
|
|
|
def __init__(self, host: str, *, sni_hostname: str, **kwargs):
|
|
super().__init__(host, **kwargs)
|
|
self._sni_hostname = sni_hostname
|
|
|
|
def connect(self):
|
|
# TCP connect to the pinned IP in self.host.
|
|
http.client.HTTPConnection.connect(self)
|
|
# TLS handshake with the real hostname for SNI + cert verification.
|
|
self.sock = self._context.wrap_socket(
|
|
self.sock,
|
|
server_hostname = self._sni_hostname,
|
|
)
|
|
|
|
|
|
class _SNIHTTPSHandler(urllib.request.HTTPSHandler):
|
|
"""HTTPS handler sending the correct SNI hostname during TLS handshake.
|
|
|
|
SSRF IP-pinning breaks SNI and cert verification; this returns a
|
|
``_PinnedHTTPSConnection`` that connects to the pinned IP but verifies TLS
|
|
against the original hostname.
|
|
"""
|
|
|
|
def __init__(self, hostname: str):
|
|
super().__init__(context = _tls_ctx)
|
|
self._sni_hostname = hostname
|
|
|
|
def https_open(self, req):
|
|
return self.do_open(self._sni_connection, req)
|
|
|
|
def _sni_connection(self, host, **kwargs):
|
|
kwargs["context"] = _tls_ctx
|
|
return _PinnedHTTPSConnection(host, sni_hostname = self._sni_hostname, **kwargs)
|
|
|
|
|
|
def _validate_and_resolve_host(hostname: str, port: int) -> tuple[bool, str, str]:
|
|
"""Resolve *hostname*, reject non-public IPs, return a pinned IP string.
|
|
|
|
Returns ``(ok, reason_or_empty, resolved_ip)``. The caller should connect
|
|
to *resolved_ip* (with a ``Host`` header) to prevent DNS rebinding between
|
|
validation and the actual fetch.
|
|
"""
|
|
import ipaddress
|
|
import socket
|
|
|
|
try:
|
|
infos = socket.getaddrinfo(hostname, port, type = socket.SOCK_STREAM)
|
|
except OSError as e:
|
|
return False, f"Failed to resolve host: {e}", ""
|
|
|
|
if not infos:
|
|
return False, f"Failed to resolve host: no addresses for {hostname!r}", ""
|
|
|
|
for *_, sockaddr in infos:
|
|
ip = ipaddress.ip_address(sockaddr[0])
|
|
# `not ip.is_global` is the source of truth: it rejects every category
|
|
# below PLUS shared/CGNAT (100.64.0.0/10) and benchmarking/doc ranges
|
|
# Python marks is_private=False and is_global=False. The explicit
|
|
# predicates only give human-readable categories in the error message.
|
|
if (
|
|
not ip.is_global
|
|
or ip.is_private
|
|
or ip.is_loopback
|
|
or ip.is_link_local
|
|
or ip.is_multicast
|
|
or ip.is_reserved
|
|
or ip.is_unspecified
|
|
):
|
|
return False, f"Blocked: refusing to fetch non-public address {ip}.", ""
|
|
|
|
# Return the first resolved address for pinning.
|
|
first_ip = infos[0][4][0]
|
|
return True, "", first_ip
|
|
|
|
|
|
def _fetch_page_text(
|
|
url: str,
|
|
max_chars: int = _MAX_PAGE_CHARS,
|
|
timeout: int = 30,
|
|
) -> str:
|
|
"""Fetch a URL and return plain text content (HTML tags stripped).
|
|
|
|
Blocks private/loopback/link-local targets (SSRF protection) and caps
|
|
the download size to avoid unbounded memory usage.
|
|
"""
|
|
from urllib.parse import urlparse
|
|
|
|
parsed = urlparse(url)
|
|
if parsed.scheme not in ("http", "https"):
|
|
return f"Blocked: only http/https URLs are allowed (got {parsed.scheme!r})."
|
|
if not parsed.hostname:
|
|
return "Blocked: URL is missing a hostname."
|
|
|
|
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
|
ok, reason, pinned_ip = _validate_and_resolve_host(parsed.hostname, port)
|
|
if not ok:
|
|
return reason
|
|
|
|
try:
|
|
from urllib.error import HTTPError as _HTTPError
|
|
from urllib.parse import urljoin, urlunparse
|
|
|
|
max_bytes = _MAX_FETCH_BYTES
|
|
current_url = url
|
|
current_host = parsed.hostname
|
|
ua = random.choice(_USER_AGENTS)
|
|
|
|
for _hop in range(5):
|
|
# Pin to the validated IP (prevents DNS rebinding): rewrite URL to
|
|
# the IP, set the Host header.
|
|
cp = urlparse(current_url)
|
|
# Bracket IPv6 addresses so the netloc is valid in a URL.
|
|
ip_str = f"[{pinned_ip}]" if ":" in pinned_ip else pinned_ip
|
|
ip_netloc = f"{ip_str}:{cp.port}" if cp.port else ip_str
|
|
pinned_url = urlunparse(cp._replace(netloc = ip_netloc))
|
|
|
|
opener = urllib.request.build_opener(
|
|
_NoRedirect,
|
|
_SNIHTTPSHandler(current_host),
|
|
)
|
|
|
|
req = urllib.request.Request(
|
|
pinned_url,
|
|
headers = {
|
|
"User-Agent": ua,
|
|
"Host": current_host,
|
|
},
|
|
)
|
|
try:
|
|
resp = opener.open(req, timeout = timeout)
|
|
except _HTTPError as e:
|
|
if e.code not in (301, 302, 303, 307, 308):
|
|
return f"Failed to fetch URL: HTTP {e.code} {getattr(e, 'reason', '')}"
|
|
location = e.headers.get("Location")
|
|
if not location:
|
|
return "Failed to fetch URL: redirect missing Location header."
|
|
current_url = urljoin(current_url, location)
|
|
rp = urlparse(current_url)
|
|
if rp.scheme not in ("http", "https") or not rp.hostname:
|
|
return "Blocked: redirect target is not a valid http/https URL."
|
|
rp_port = rp.port or (443 if rp.scheme == "https" else 80)
|
|
ok2, reason2, pinned_ip = _validate_and_resolve_host(
|
|
rp.hostname,
|
|
rp_port,
|
|
)
|
|
if not ok2:
|
|
return reason2
|
|
current_host = rp.hostname
|
|
continue
|
|
# Success: read capped body.
|
|
raw_bytes = resp.read(max_bytes)
|
|
break
|
|
else:
|
|
return "Failed to fetch URL: too many redirects."
|
|
|
|
charset = resp.headers.get_content_charset() or "utf-8"
|
|
raw_html = raw_bytes.decode(charset, errors = "replace")
|
|
except _HTTPError as e:
|
|
return f"Failed to fetch URL: HTTP {e.code} {getattr(e, 'reason', '')}"
|
|
except Exception as e:
|
|
return f"Failed to fetch URL: {e}"
|
|
|
|
# Convert HTML to Markdown with the builtin converter (no external deps).
|
|
from ._html_to_md import html_to_markdown
|
|
|
|
text = html_to_markdown(raw_html)
|
|
|
|
if not text:
|
|
return "(page returned no readable text)"
|
|
if len(text) > max_chars:
|
|
text = text[:max_chars] + f"\n\n... (truncated, {len(text)} chars total)"
|
|
return text
|
|
|
|
|
|
def _web_search(
|
|
query: str,
|
|
max_results: int = 5,
|
|
timeout: int = _EXEC_TIMEOUT,
|
|
url: str | None = None,
|
|
) -> str:
|
|
"""Search the web using DuckDuckGo and return formatted results.
|
|
|
|
If ``url`` is provided, fetches that page directly instead of searching.
|
|
"""
|
|
# Direct URL fetch mode.
|
|
if url and url.strip():
|
|
fetch_timeout = 60 if timeout is None else min(timeout, 60)
|
|
return _fetch_page_text(url.strip(), timeout = fetch_timeout)
|
|
|
|
if not query or not query.strip():
|
|
return "No query provided."
|
|
try:
|
|
from ddgs import DDGS
|
|
|
|
results = DDGS(timeout = timeout).text(query, max_results = max_results)
|
|
if not results:
|
|
return "No results found."
|
|
parts = []
|
|
for r in results:
|
|
parts.append(
|
|
f"Title: {r.get('title', '')}\n"
|
|
f"URL: {r.get('href', '')}\n"
|
|
f"Snippet: {r.get('body', '')}"
|
|
)
|
|
text = "\n\n---\n\n".join(parts)
|
|
text += (
|
|
"\n\n---\n\nIMPORTANT: These are only short snippets. "
|
|
"To get the full page content, call web_search with "
|
|
'the url parameter (e.g. {"url": "<URL>"}).'
|
|
)
|
|
return text
|
|
except Exception as e:
|
|
return f"Search failed: {e}"
|
|
|
|
|
|
# ==========================================================================
|
|
# Sandbox static-analysis hardening (feature-flagged; see UNSLOTH_STUDIO_SINK_ANALYZER)
|
|
#
|
|
# A pure, whitelist-only constant folder plus a filesystem-confinement path
|
|
# resolver back the eval/exec payload recursion and the destructive-op gate.
|
|
# Everything here recomputes pure transforms on *literals only* and never runs,
|
|
# imports, or reflects on user code. All limits are bounded so the analyzer can
|
|
# never be slower or crashier than the legacy syntactic checks; on any breach a
|
|
# folder returns None (opaque) and the caller fails safe.
|
|
# ==========================================================================
|
|
|
|
# Folder bounds (Stage 1). Breaching any of these yields None ("un-foldable").
|
|
_FOLD_DEPTH = 24
|
|
_FOLD_MAXLEN = 65536
|
|
_FOLD_OPS = 4000
|
|
_FOLD_MAX_SEQ = 4096
|
|
_FOLD_MAXINT = 1 << 64
|
|
|
|
_UNKNOWN = object() # sentinel: "not statically decidable"
|
|
|
|
|
|
class _FoldState:
|
|
"""Shared op counter + single-assignment const-prop environment."""
|
|
|
|
__slots__ = ("ops", "names")
|
|
|
|
def __init__(self, names = None):
|
|
self.ops = 0
|
|
self.names = names or {}
|
|
|
|
|
|
def _fold_cap(value):
|
|
"""Return value unless a str/bytes exceeds the size cap or an int the magnitude cap."""
|
|
if isinstance(value, (str, bytes, bytearray)) and len(value) > _FOLD_MAXLEN:
|
|
return None
|
|
if isinstance(value, int) and not isinstance(value, bool) and abs(value) > _FOLD_MAXINT:
|
|
return None
|
|
return value
|
|
|
|
|
|
def _too_wide(n):
|
|
"""A format width / precision / size arg large enough to OOM the folder."""
|
|
return isinstance(n, int) and not isinstance(n, bool) and n > _FOLD_MAXLEN
|
|
|
|
|
|
# Format-spec mini-language: reject an oversized width or precision BEFORE format()
|
|
# allocates the padded string. format()/str.format()/f-strings all run in the Studio
|
|
# process during static analysis, ahead of the child-subprocess rlimits.
|
|
_FMT_SPEC_RE = re.compile(r"^(?:.?[<>=^])?[+\- ]?z?#?0?(\d+)?[,_]?(?:\.(\d+))?[a-zA-Z%]?$")
|
|
|
|
|
|
def _format_spec_ok(spec):
|
|
if not isinstance(spec, str) or not spec:
|
|
return True
|
|
m = _FMT_SPEC_RE.match(spec)
|
|
if not m:
|
|
return True # unrecognized spec: let format() itself decide at runtime
|
|
return not any(g and _too_wide(int(g)) for g in m.groups())
|
|
|
|
|
|
def _format_template_ok(template):
|
|
"""Every replacement field of a str.format template has a bounded width."""
|
|
if not isinstance(template, str):
|
|
return True
|
|
try:
|
|
import string as _string
|
|
for _lit, _field, _spec, _conv in _string.Formatter().parse(template):
|
|
if _spec and not _format_spec_ok(_spec):
|
|
return False
|
|
except Exception:
|
|
return True
|
|
return True
|
|
|
|
|
|
def _format_has_nested_spec(template):
|
|
"""A replacement field whose spec itself contains a field ({:{}}): the width is
|
|
supplied by an argument, so a large numeric arg drives the allocation."""
|
|
if not isinstance(template, str):
|
|
return False
|
|
try:
|
|
import string as _string
|
|
for _lit, _field, _spec, _conv in _string.Formatter().parse(template):
|
|
if _spec and "{" in _spec:
|
|
return True
|
|
except Exception:
|
|
return False
|
|
return False
|
|
|
|
|
|
_PRINTF_WIDTH_RE = re.compile(r"%[-+ #0]*(\d+)?(?:\.(\d+))?[hlL]?[diouxXeEfFgGcrsab%]")
|
|
|
|
|
|
def _printf_ok(fmt):
|
|
"""Percent-format string with no oversized (or dynamic '*') width / precision."""
|
|
if isinstance(fmt, (bytes, bytearray)):
|
|
try:
|
|
fmt = fmt.decode("latin-1")
|
|
except Exception:
|
|
return True
|
|
if not isinstance(fmt, str):
|
|
return True
|
|
# A '*' width or precision ('%*s', '%.*f') pulls its size from a runtime argument,
|
|
# so it cannot be bounded statically -- refuse rather than risk a large allocation.
|
|
for m in re.finditer(r"%[-+ #0]*(\*)?(?:\.(\*)?\d*)?", fmt):
|
|
if m.group(1) == "*" or m.group(2) == "*":
|
|
return False
|
|
for m in _PRINTF_WIDTH_RE.finditer(fmt):
|
|
if any(g and _too_wide(int(g)) for g in m.groups()):
|
|
return False
|
|
return True
|
|
|
|
|
|
def _replace_output_ok(recv, call_args):
|
|
"""Bound str.replace/bytes.replace output before it allocates: replacing many
|
|
occurrences with a long replacement can build a multi-gigabyte string."""
|
|
if len(call_args) < 2:
|
|
return True
|
|
old, new = call_args[0], call_args[1]
|
|
if not isinstance(new, (str, bytes, bytearray)):
|
|
return True
|
|
lo = len(old) if isinstance(old, (str, bytes, bytearray)) else 1
|
|
n_repl = (len(recv) + 1) if lo == 0 else (len(recv) // max(lo, 1) + 1)
|
|
if len(call_args) >= 3 and isinstance(call_args[2], int) and call_args[2] >= 0:
|
|
n_repl = min(n_repl, call_args[2])
|
|
return len(recv) + n_repl * len(new) <= _FOLD_MAXLEN
|
|
|
|
|
|
def _join_output_ok(sep, call_args):
|
|
"""Bound str.join/bytes.join output before it allocates."""
|
|
if not call_args or not isinstance(call_args[0], (list, tuple)):
|
|
return True
|
|
items = call_args[0]
|
|
total = len(sep) * max(len(items) - 1, 0)
|
|
for x in items:
|
|
if not isinstance(x, (str, bytes, bytearray)):
|
|
return True # a real join would TypeError; not an allocation concern
|
|
total += len(x)
|
|
if total > _FOLD_MAXLEN:
|
|
return False
|
|
return True
|
|
|
|
|
|
def _fold_apply_codec(name, data):
|
|
"""Pure data transforms only (rot13/hex/base64/zlib/text codecs). Bounded zlib."""
|
|
name = name.lower().replace("-", "_")
|
|
try:
|
|
if name in ("rot_13", "rot13"):
|
|
text = data if isinstance(data, str) else data.decode("latin-1")
|
|
return codecs.decode(text, "rot_13")
|
|
if name == "hex":
|
|
return codecs.decode(data, "hex")
|
|
if name in ("base64", "base_64"):
|
|
return base64.b64decode(data if isinstance(data, (bytes, bytearray)) else data.encode())
|
|
if name == "zlib":
|
|
payload = data if isinstance(data, (bytes, bytearray)) else str(data).encode()
|
|
d = zlib.decompressobj()
|
|
out = d.decompress(payload, _FOLD_MAXLEN)
|
|
if d.unconsumed_tail: # would exceed the cap -> refuse
|
|
return None
|
|
return out
|
|
if name in ("utf_8", "utf8", "latin_1", "latin1", "ascii"):
|
|
if isinstance(data, (bytes, bytearray)):
|
|
return data.decode(name)
|
|
return data.encode(name)
|
|
except Exception:
|
|
return None
|
|
return None # bz2/lzma/gzip and unknowns: bomb-unsafe / opaque -> refuse
|
|
|
|
|
|
_FOLD_PURE_BUILTINS = frozenset(
|
|
{"chr", "ord", "str", "int", "bytes", "bytearray", "hex", "oct", "bin", "bool", "float", "len"}
|
|
)
|
|
_FOLD_STR_METHODS = frozenset(
|
|
{
|
|
"join",
|
|
"replace",
|
|
"upper",
|
|
"lower",
|
|
"strip",
|
|
"lstrip",
|
|
"rstrip",
|
|
"swapcase",
|
|
"title",
|
|
"capitalize",
|
|
"format",
|
|
"zfill",
|
|
"ljust",
|
|
"rjust",
|
|
"center",
|
|
"encode",
|
|
"decode",
|
|
}
|
|
)
|
|
_FOLD_B64_FUNCS = frozenset(
|
|
{
|
|
"b64decode",
|
|
"b64encode",
|
|
"urlsafe_b64decode",
|
|
"standard_b64decode",
|
|
"b32decode",
|
|
"b16decode",
|
|
"a85decode",
|
|
"b85decode",
|
|
}
|
|
)
|
|
|
|
|
|
def _const_fold(
|
|
node,
|
|
env = None,
|
|
_state = None,
|
|
_depth = 0,
|
|
):
|
|
"""Fold an AST expression to a concrete str/bytes/int/list value, else None.
|
|
|
|
Whitelist-only and pure: it never executes user code, never imports, never
|
|
reflects. Only a fixed set of pure transforms over already-folded literals
|
|
(concat/repeat/join/format/slice/reverse, base64/hex/rot13/zlib decode, and
|
|
a handful of pure builtins/str methods) is supported; anything else returns
|
|
None. ``env`` maps single-assignment module-level names to their RHS nodes.
|
|
"""
|
|
if _state is None:
|
|
_state = _FoldState(env)
|
|
_state.ops += 1
|
|
if node is None or _depth > _FOLD_DEPTH or _state.ops > _FOLD_OPS:
|
|
return None
|
|
|
|
if isinstance(node, ast.Constant):
|
|
v = node.value
|
|
if isinstance(v, (str, bytes, bytearray, int, float)) or v is None:
|
|
return _fold_cap(v)
|
|
return None
|
|
|
|
if isinstance(node, ast.Name):
|
|
rhs = _state.names.get(node.id)
|
|
if rhs is None:
|
|
return None
|
|
return _const_fold(rhs, None, _state, _depth + 1)
|
|
|
|
if isinstance(node, (ast.List, ast.Tuple)):
|
|
if len(node.elts) > _FOLD_MAX_SEQ:
|
|
return None
|
|
vals = []
|
|
for e in node.elts:
|
|
v = _const_fold(e, None, _state, _depth + 1)
|
|
if v is None and not (isinstance(e, ast.Constant) and e.value is None):
|
|
return None
|
|
vals.append(v)
|
|
return vals
|
|
|
|
if isinstance(node, ast.JoinedStr):
|
|
out = []
|
|
for part in node.values:
|
|
if isinstance(part, ast.Constant):
|
|
out.append(str(part.value))
|
|
elif isinstance(part, ast.FormattedValue):
|
|
v = _const_fold(part.value, None, _state, _depth + 1)
|
|
if v is None:
|
|
return None
|
|
spec = ""
|
|
if part.format_spec is not None:
|
|
spec = _const_fold(part.format_spec, None, _state, _depth + 1)
|
|
if spec is None:
|
|
return None
|
|
if part.conversion and part.conversion != -1:
|
|
try:
|
|
v = {114: repr, 115: str, 97: ascii}[part.conversion](v)
|
|
except Exception:
|
|
return None
|
|
if not _format_spec_ok(spec if isinstance(spec, str) else ""):
|
|
return None # oversized f-string width/precision: refuse pre-format
|
|
try:
|
|
out.append(format(v, spec if isinstance(spec, str) else ""))
|
|
except Exception:
|
|
return None
|
|
else:
|
|
return None
|
|
return _fold_cap("".join(out))
|
|
|
|
if isinstance(node, ast.BinOp):
|
|
left = _const_fold(node.left, None, _state, _depth + 1)
|
|
right = _const_fold(node.right, None, _state, _depth + 1)
|
|
if left is None or right is None:
|
|
return None
|
|
op = node.op
|
|
try:
|
|
if isinstance(op, ast.Mult):
|
|
if isinstance(left, (str, bytes, bytearray)) and isinstance(right, int):
|
|
if len(left) * max(right, 0) > _FOLD_MAXLEN:
|
|
return None
|
|
if isinstance(right, (str, bytes, bytearray)) and isinstance(left, int):
|
|
if len(right) * max(left, 0) > _FOLD_MAXLEN:
|
|
return None
|
|
# list/tuple repetition allocates len(seq)*n elements before _fold_cap
|
|
# (which only sizes str/bytes) can reject it -- cap it here too.
|
|
if isinstance(left, (list, tuple)) and isinstance(right, int):
|
|
if len(left) * max(right, 0) > _FOLD_MAX_SEQ:
|
|
return None
|
|
if isinstance(right, (list, tuple)) and isinstance(left, int):
|
|
if len(right) * max(left, 0) > _FOLD_MAX_SEQ:
|
|
return None
|
|
return _fold_cap(left * right)
|
|
if isinstance(op, ast.Add):
|
|
# str/bytes concat is sized by _fold_cap, but list/tuple concatenation is
|
|
# not, so a chain (a + a + a + ...) materializes an oversized sequence in the
|
|
# parent process before child rlimits apply. Cap the combined length.
|
|
if isinstance(left, (list, tuple)) and isinstance(right, (list, tuple)):
|
|
if len(left) + len(right) > _FOLD_MAX_SEQ:
|
|
return None
|
|
return _fold_cap(left + right)
|
|
if isinstance(op, ast.Mod):
|
|
if isinstance(left, (str, bytes, bytearray)) and not _printf_ok(left):
|
|
return None # oversized %-format width/precision: refuse pre-format
|
|
return _fold_cap(left % right)
|
|
if isinstance(op, ast.Sub):
|
|
return _fold_cap(left - right)
|
|
if isinstance(op, ast.FloorDiv):
|
|
return _fold_cap(left // right)
|
|
if isinstance(op, ast.Div):
|
|
return _fold_cap(left / right)
|
|
if isinstance(op, ast.BitXor):
|
|
return _fold_cap(left ^ right)
|
|
if isinstance(op, ast.BitOr):
|
|
return _fold_cap(left | right)
|
|
if isinstance(op, ast.BitAnd):
|
|
return _fold_cap(left & right)
|
|
if isinstance(op, ast.LShift) and isinstance(right, int) and 0 <= right < 64:
|
|
return _fold_cap(left << right)
|
|
if isinstance(op, ast.RShift) and isinstance(right, int) and 0 <= right < 64:
|
|
return _fold_cap(left >> right)
|
|
except Exception:
|
|
return None
|
|
return None # Pow and others: refuse (bignum DoS)
|
|
|
|
if isinstance(node, ast.UnaryOp):
|
|
v = _const_fold(node.operand, None, _state, _depth + 1)
|
|
if v is None:
|
|
return None
|
|
try:
|
|
return {
|
|
ast.USub: lambda x: -x,
|
|
ast.UAdd: lambda x: +x,
|
|
ast.Invert: lambda x: ~x,
|
|
ast.Not: lambda x: not x,
|
|
}[type(node.op)](v)
|
|
except Exception:
|
|
return None
|
|
|
|
if isinstance(node, ast.Subscript):
|
|
base = _const_fold(node.value, None, _state, _depth + 1)
|
|
if base is None or not isinstance(base, (str, bytes, bytearray, list, tuple)):
|
|
return None
|
|
sl = node.slice
|
|
try:
|
|
if isinstance(sl, ast.Slice):
|
|
lo = _const_fold(sl.lower, None, _state, _depth + 1) if sl.lower else None
|
|
hi = _const_fold(sl.upper, None, _state, _depth + 1) if sl.upper else None
|
|
st = _const_fold(sl.step, None, _state, _depth + 1) if sl.step else None
|
|
if (
|
|
(sl.lower is not None and lo is None)
|
|
or (sl.upper is not None and hi is None)
|
|
or (sl.step is not None and st is None)
|
|
):
|
|
return None
|
|
return _fold_cap(base[lo:hi:st])
|
|
idx = _const_fold(sl, None, _state, _depth + 1)
|
|
if not isinstance(idx, int):
|
|
return None
|
|
return _fold_cap(base[idx])
|
|
except Exception:
|
|
return None
|
|
|
|
if isinstance(node, ast.Call):
|
|
return _fold_call(node, _state, _depth)
|
|
|
|
return None
|
|
|
|
|
|
def _is_path_join_owner(nv):
|
|
"""AST for ``os.path`` (Attribute) or ``posixpath`` / ``ntpath`` (Name)."""
|
|
if (
|
|
isinstance(nv, ast.Attribute)
|
|
and nv.attr == "path"
|
|
and isinstance(nv.value, ast.Name)
|
|
and nv.value.id == "os"
|
|
):
|
|
return True
|
|
return isinstance(nv, ast.Name) and nv.id in ("posixpath", "ntpath")
|
|
|
|
|
|
def _fold_call(node, _state, _depth):
|
|
"""Fold a whitelisted pure builtin / method / decode call, else None."""
|
|
f = node.func
|
|
args = []
|
|
for a in node.args:
|
|
v = _const_fold(a, None, _state, _depth + 1)
|
|
if v is None and not (isinstance(a, ast.Constant) and a.value is None):
|
|
return None
|
|
args.append(v)
|
|
|
|
if isinstance(f, ast.Name):
|
|
name = f.id
|
|
if name not in _FOLD_PURE_BUILTINS:
|
|
return None
|
|
try:
|
|
if name == "chr":
|
|
if len(args) == 1 and isinstance(args[0], int) and 0 <= args[0] <= 0x10FFFF:
|
|
return chr(args[0])
|
|
return None
|
|
if name == "ord":
|
|
if (
|
|
len(args) == 1
|
|
and isinstance(args[0], (str, bytes, bytearray))
|
|
and len(args[0]) == 1
|
|
):
|
|
return ord(args[0])
|
|
return None
|
|
fn = {
|
|
"str": str,
|
|
"bytes": bytes,
|
|
"bytearray": bytearray,
|
|
"int": int,
|
|
"hex": hex,
|
|
"oct": oct,
|
|
"bin": bin,
|
|
"bool": bool,
|
|
"float": float,
|
|
"len": len,
|
|
}[name]
|
|
if name in ("bytes", "bytearray") and len(args) == 1:
|
|
# bytes(n) / bytearray(n) allocate n zero bytes; refuse an oversized
|
|
# integer size before constructing it so static analysis of e.g.
|
|
# bytes(2_000_000_000) cannot OOM the Studio process (the child
|
|
# sandbox rlimits never get a chance to help during analysis).
|
|
if (
|
|
isinstance(args[0], int)
|
|
and not isinstance(args[0], bool)
|
|
and args[0] > _FOLD_MAXLEN
|
|
):
|
|
return None
|
|
return _fold_cap(fn(*args))
|
|
except Exception:
|
|
return None
|
|
|
|
if isinstance(f, ast.Attribute):
|
|
attr = f.attr
|
|
owner = f.value
|
|
# os.path.join('/etc', 'passwd') / posixpath.join(...) / ntpath.join(...):
|
|
# fold literal path builders so the sensitive-read scanner sees the concrete
|
|
# path (open(os.path.join('/etc','passwd')) must not be treated as opaque).
|
|
if attr == "join" and _is_path_join_owner(owner):
|
|
if args and all(isinstance(x, str) for x in args):
|
|
try:
|
|
return _fold_cap(os.path.join(*args))
|
|
except Exception:
|
|
return None
|
|
return None
|
|
# os.path.normpath / abspath on a literal reveal the same sensitive / traversal
|
|
# path a direct string would (open(os.path.normpath('a/../../etc/passwd')) must
|
|
# not stay opaque). normpath is a pure string transform; abspath is only foldable
|
|
# for an already-absolute arg (a relative abspath depends on the runtime cwd,
|
|
# which is the in-workdir sandbox cwd, so it need not be folded).
|
|
if attr in ("normpath", "abspath") and _is_path_join_owner(owner):
|
|
if len(args) == 1 and isinstance(args[0], str):
|
|
try:
|
|
if attr == "normpath" or os.path.isabs(args[0]):
|
|
return _fold_cap(os.path.normpath(args[0]))
|
|
except Exception:
|
|
return None
|
|
return None
|
|
if isinstance(owner, ast.Name):
|
|
mod = owner.id
|
|
try:
|
|
if mod == "base64" and attr in _FOLD_B64_FUNCS and len(args) >= 1:
|
|
return _fold_cap(getattr(base64, attr)(args[0]))
|
|
if (
|
|
mod == "codecs"
|
|
and attr in ("decode", "encode")
|
|
and len(args) >= 2
|
|
and isinstance(args[1], str)
|
|
):
|
|
return _fold_cap(_fold_apply_codec(args[1], args[0]))
|
|
if mod == "binascii" and attr in ("unhexlify", "a2b_hex") and len(args) >= 1:
|
|
return _fold_cap(binascii.unhexlify(args[0]))
|
|
if (
|
|
mod in ("bytes", "bytearray")
|
|
and attr == "fromhex"
|
|
and len(args) >= 1
|
|
and isinstance(args[0], str)
|
|
):
|
|
return _fold_cap(bytes.fromhex(args[0]))
|
|
except Exception:
|
|
return None
|
|
recv = _const_fold(owner, None, _state, _depth + 1)
|
|
if isinstance(recv, (str, bytes, bytearray)) and attr in _FOLD_STR_METHODS:
|
|
try:
|
|
kwargs = {}
|
|
for kw in node.keywords:
|
|
if kw.arg is None:
|
|
return None
|
|
kv = _const_fold(kw.value, None, _state, _depth + 1)
|
|
if kv is None:
|
|
return None
|
|
kwargs[kw.arg] = kv
|
|
call_args = []
|
|
for a in args:
|
|
call_args.append(
|
|
list(a) if attr == "join" and isinstance(a, (list, tuple)) else a
|
|
)
|
|
# Padding methods take a width as their first arg; str.format takes a
|
|
# template with per-field widths. Reject an oversized width before the
|
|
# method allocates the padded string during folding.
|
|
if attr in ("center", "ljust", "rjust", "zfill") and call_args:
|
|
if _too_wide(call_args[0]):
|
|
return None
|
|
if attr == "replace" and not _replace_output_ok(recv, call_args):
|
|
return None
|
|
if attr == "join" and not _join_output_ok(recv, call_args):
|
|
return None
|
|
if attr == "format":
|
|
if not _format_template_ok(recv):
|
|
return None
|
|
# Nested-width field ({:{}}) whose width comes from a large numeric
|
|
# arg: refuse before format() allocates.
|
|
if _format_has_nested_spec(recv) and any(
|
|
_too_wide(a) for a in list(call_args) + list(kwargs.values())
|
|
):
|
|
return None
|
|
return _fold_cap(getattr(recv, attr)(*call_args, **kwargs))
|
|
except Exception:
|
|
return None
|
|
return None
|
|
|
|
|
|
def _build_const_prop_env(tree):
|
|
"""Names bound exactly once by a module-level ``name = <expr>`` (single Name
|
|
target), never re-assigned / aug-assigned / declared global-nonlocal / used as
|
|
a loop / comprehension / with / except target. Maps name -> RHS node.
|
|
|
|
Conservative: any ambiguity excludes the name. Only module-level statements are
|
|
considered so a name shadowed inside a def / loop is never folded.
|
|
"""
|
|
assigned_once: dict[str, ast.expr] = {}
|
|
disqualified: set[str] = set()
|
|
|
|
def _disqualify_targets(target):
|
|
for n in ast.walk(target):
|
|
if isinstance(n, ast.Name):
|
|
disqualified.add(n.id)
|
|
|
|
# Module-level single assignments.
|
|
for stmt in getattr(tree, "body", []):
|
|
if (
|
|
isinstance(stmt, ast.Assign)
|
|
and len(stmt.targets) == 1
|
|
and isinstance(stmt.targets[0], ast.Name)
|
|
):
|
|
name = stmt.targets[0].id
|
|
if name in assigned_once or name in disqualified:
|
|
disqualified.add(name)
|
|
assigned_once.pop(name, None)
|
|
else:
|
|
assigned_once[name] = stmt.value
|
|
elif isinstance(stmt, ast.Assign):
|
|
for t in stmt.targets:
|
|
_disqualify_targets(t)
|
|
elif isinstance(stmt, (ast.AugAssign, ast.AnnAssign)):
|
|
if getattr(stmt, "target", None) is not None:
|
|
_disqualify_targets(stmt.target)
|
|
|
|
# Any name that is ALSO written anywhere else (loops, defs, walrus, aug, params,
|
|
# comprehension targets, with/except/for) is disqualified.
|
|
for n in ast.walk(tree):
|
|
if isinstance(n, ast.Name) and isinstance(n.ctx, (ast.Store, ast.Del)):
|
|
nm = n.id
|
|
if nm in assigned_once:
|
|
# It is stored somewhere; allow only if that single store is the
|
|
# module-level assign we recorded (identity check below).
|
|
pass
|
|
if isinstance(n, (ast.AugAssign,)):
|
|
_disqualify_targets(n.target)
|
|
elif isinstance(n, ast.NamedExpr):
|
|
_disqualify_targets(n.target)
|
|
elif isinstance(n, (ast.For, ast.AsyncFor)):
|
|
_disqualify_targets(n.target)
|
|
elif isinstance(n, ast.comprehension):
|
|
_disqualify_targets(n.target)
|
|
elif isinstance(n, ast.withitem):
|
|
if n.optional_vars is not None:
|
|
_disqualify_targets(n.optional_vars)
|
|
elif isinstance(n, ast.ExceptHandler):
|
|
if n.name:
|
|
disqualified.add(n.name)
|
|
elif isinstance(n, (ast.Global, ast.Nonlocal)):
|
|
for nm in n.names:
|
|
disqualified.add(nm)
|
|
elif isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
disqualified.add(n.name)
|
|
args = getattr(n, "args", None)
|
|
if args is not None:
|
|
for a in list(args.args) + list(args.posonlyargs) + list(args.kwonlyargs):
|
|
disqualified.add(a.arg)
|
|
for extra in (args.vararg, args.kwarg):
|
|
if extra is not None:
|
|
disqualified.add(extra.arg)
|
|
|
|
# Count how many module-level stores each recorded name really has; if more
|
|
# than one Store target references it anywhere, drop it.
|
|
store_counts: dict[str, int] = {}
|
|
for n in ast.walk(tree):
|
|
if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Store):
|
|
store_counts[n.id] = store_counts.get(n.id, 0) + 1
|
|
|
|
env = {}
|
|
for name, rhs in assigned_once.items():
|
|
if name in disqualified:
|
|
continue
|
|
if store_counts.get(name, 0) != 1:
|
|
continue
|
|
env[name] = rhs
|
|
return env
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Stage 2: eval / exec / compile recursive payload analysis.
|
|
# --------------------------------------------------------------------------
|
|
_MAX_UNWRAP_DEPTH = 5
|
|
_MAX_INNER_SRC = 200 * 1024
|
|
_MAX_INNER_PARSES = 25
|
|
_MAX_TOTAL_INNER_CHARS = 2 * 1024 * 1024
|
|
_MAX_BRACKET_DEPTH = 200
|
|
_MAX_ANALYZER_NODES = 200_000
|
|
|
|
_EXEC_BUILTINS = frozenset({"eval", "exec", "compile"})
|
|
# Deserialization sinks that reconstruct/execute arbitrary objects from bytes.
|
|
_CODE_DESERIALIZE_SINKS = frozenset(
|
|
{
|
|
"pickle.loads",
|
|
"pickle.load",
|
|
"marshal.loads",
|
|
"marshal.load",
|
|
"dill.loads",
|
|
"dill.load",
|
|
"cloudpickle.loads",
|
|
"cloudpickle.load",
|
|
"_pickle.loads",
|
|
"_pickle.load",
|
|
"jsonpickle.decode",
|
|
# PyYAML: unsafe_load / full_load construct arbitrary Python objects from
|
|
# `!!python/object/apply:os.system [...]`. yaml.load is conditional (safe only with a
|
|
# SafeLoader) and is handled separately.
|
|
"yaml.unsafe_load",
|
|
"yaml.unsafe_load_all",
|
|
"yaml.full_load",
|
|
"yaml.full_load_all",
|
|
}
|
|
)
|
|
# Modules whose load/loads/decode entry points run a pickle reduce payload; used to
|
|
# resolve `import pickle as p; p.loads(x)` and `from pickle import loads as l`.
|
|
_DESERIALIZE_MODULES = frozenset(
|
|
{"pickle", "marshal", "dill", "cloudpickle", "_pickle", "jsonpickle", "yaml"}
|
|
)
|
|
# Modules exposing an Unpickler class whose .load() runs the same reduce payload as *.load:
|
|
# pickle.Unpickler(f).load() / dill.Unpickler(f).load() bypass the *.load sink-name check.
|
|
_UNPICKLER_MODULES = frozenset({"pickle", "_pickle", "dill", "cloudpickle"})
|
|
# yaml.load / yaml.load_all construct arbitrary objects UNLESS given a safe loader; flag them
|
|
# when the Loader is absent or is one of the unsafe loader classes.
|
|
_YAML_SAFE_LOADERS = frozenset({"SafeLoader", "CSafeLoader", "BaseLoader"})
|
|
_YAML_LOAD_METHODS = frozenset({"load", "load_all"})
|
|
|
|
|
|
def _yaml_loader_class_name(value):
|
|
"""Terminal attribute/name of a Loader= argument (yaml.SafeLoader -> SafeLoader)."""
|
|
if isinstance(value, ast.Attribute):
|
|
return value.attr
|
|
if isinstance(value, ast.Name):
|
|
return value.id
|
|
return None
|
|
|
|
|
|
def _yaml_call_has_safe_loader(node):
|
|
"""True only when a yaml.load(...) call passes an explicit safe Loader= keyword.
|
|
|
|
A missing Loader (older PyYAML defaults to the full, unsafe loader), an unknown/computed
|
|
loader, or a **kwargs splat all fail closed so the call is treated as an unsafe sink.
|
|
"""
|
|
for kw in node.keywords:
|
|
if kw.arg == "Loader":
|
|
return _yaml_loader_class_name(kw.value) in _YAML_SAFE_LOADERS
|
|
if kw.arg is None:
|
|
# **kwargs unpacking hides the loader; cannot prove it is safe.
|
|
return False
|
|
return False
|
|
|
|
|
|
# Attribute names of pure decode/decompress primitives used to hide a payload.
|
|
_DECODE_ATTRS = frozenset(
|
|
{
|
|
"b64decode",
|
|
"b64encode",
|
|
"urlsafe_b64decode",
|
|
"standard_b64decode",
|
|
"b32decode",
|
|
"b16decode",
|
|
"a85decode",
|
|
"b85decode",
|
|
"decodebytes",
|
|
"fromhex",
|
|
"unhexlify",
|
|
"a2b_hex",
|
|
"a2b_base64",
|
|
"decompress",
|
|
}
|
|
)
|
|
_FETCH_FQ_PREFIXES = (
|
|
"requests.",
|
|
"urllib.",
|
|
"httpx.",
|
|
"socket.",
|
|
"aiohttp.",
|
|
"urllib3.",
|
|
"http.client.",
|
|
)
|
|
# Constant attribute names that, resolved off a sensitive module via getattr, still
|
|
# reach shell / process / delete / dynamic-import / code-exec capabilities. A benign
|
|
# constant attr (getpid, path, sep, getcwd, ...) is allowed; a dynamic attr blocks.
|
|
_DANGEROUS_ATTR_NAMES = frozenset(
|
|
{
|
|
"system",
|
|
"popen",
|
|
"popen2",
|
|
"popen3",
|
|
"popen4",
|
|
"execl",
|
|
"execle",
|
|
"execlp",
|
|
"execlpe",
|
|
"execv",
|
|
"execve",
|
|
"execvp",
|
|
"execvpe",
|
|
"spawnl",
|
|
"spawnle",
|
|
"spawnlp",
|
|
"spawnlpe",
|
|
"spawnv",
|
|
"spawnve",
|
|
"spawnvp",
|
|
"spawnvpe",
|
|
"posix_spawn",
|
|
"posix_spawnp",
|
|
"startfile",
|
|
"fork",
|
|
"forkpty",
|
|
"remove",
|
|
"unlink",
|
|
"rmdir",
|
|
"removedirs",
|
|
"rename",
|
|
"renames",
|
|
"replace",
|
|
"truncate",
|
|
"chmod",
|
|
"lchmod",
|
|
"chown",
|
|
"lchown",
|
|
"chflags",
|
|
"mkdir",
|
|
"makedirs",
|
|
"mknod",
|
|
"symlink",
|
|
"link",
|
|
"chdir",
|
|
"chroot",
|
|
"import_module",
|
|
"__import__",
|
|
"reload",
|
|
"eval",
|
|
"exec",
|
|
"compile",
|
|
"run",
|
|
"call",
|
|
"check_call",
|
|
"check_output",
|
|
"Popen",
|
|
"getoutput",
|
|
"getstatusoutput",
|
|
"load_module",
|
|
"exec_module",
|
|
"loads",
|
|
"load",
|
|
}
|
|
)
|
|
|
|
|
|
class _AnalyzerBudget:
|
|
"""Shared, bounded counters across one classification (incl. exec recursion)."""
|
|
|
|
__slots__ = ("inner_parses", "inner_chars", "nodes")
|
|
|
|
def __init__(self):
|
|
self.inner_parses = 0
|
|
self.inner_chars = 0
|
|
self.nodes = 0
|
|
|
|
|
|
def _fq_attr_name(node):
|
|
"""Return the dotted name for a Name/Attribute chain, else ''."""
|
|
parts = []
|
|
cur = node
|
|
while isinstance(cur, ast.Attribute):
|
|
parts.append(cur.attr)
|
|
cur = cur.value
|
|
if isinstance(cur, ast.Name):
|
|
parts.append(cur.id)
|
|
return ".".join(reversed(parts))
|
|
return ""
|
|
|
|
|
|
def _bracket_depth(s):
|
|
"""Linear max bracket-nesting scan; never invokes the C parser (DoS-safe)."""
|
|
depth = mx = 0
|
|
for ch in s:
|
|
if ch in "([{":
|
|
depth += 1
|
|
if depth > mx:
|
|
mx = depth
|
|
elif ch in ")]}":
|
|
depth = depth - 1 if depth > 0 else 0
|
|
return mx
|
|
|
|
|
|
def _to_text(value):
|
|
if isinstance(value, (bytes, bytearray)):
|
|
try:
|
|
return value.decode("utf-8")
|
|
except Exception:
|
|
return value.decode("latin-1", "replace")
|
|
return value
|
|
|
|
|
|
# PEP 263 source-encoding cookie ("# -*- coding: utf-8 -*-", "# coding: utf_7").
|
|
_CODING_COOKIE_RE = re.compile(rb"coding[:=]\s*([-\w.]+)")
|
|
_CODING_COOKIE_TEXT_RE = re.compile(r"coding[:=]\s*([-\w.]+)")
|
|
|
|
|
|
def _decode_source_bytes(data):
|
|
"""Decode an exec/compile *bytes* payload the way CPython would.
|
|
|
|
exec()/eval()/compile() honor a PEP 263 coding cookie on bytes, so the analyzer
|
|
must decode with that cookie's codec (not a fixed UTF-8 view) or a snippet like
|
|
``exec(b"# coding: utf_7\\n#+AAo-__import__('os').system('id')")`` reads as pure
|
|
comments under UTF-8 while actually running hidden code. Detect the encoding,
|
|
decode, then neutralize the cookie so ast.parse(str) does not reject the decoded
|
|
text (a str carrying a coding declaration raises SyntaxError), preserving line
|
|
numbers so the recursive analysis sees the real source.
|
|
"""
|
|
data = bytes(data)
|
|
enc = "utf-8"
|
|
try:
|
|
import io as _io_mod
|
|
import tokenize as _tok
|
|
enc, _ = _tok.detect_encoding(_io_mod.BytesIO(data).readline)
|
|
except Exception:
|
|
enc = "utf-8"
|
|
for _cand in (enc, "utf-8"):
|
|
try:
|
|
text = data.decode(_cand)
|
|
break
|
|
except Exception:
|
|
text = None
|
|
if text is None:
|
|
text = data.decode("latin-1", "replace")
|
|
lines = text.split("\n")
|
|
for _i in range(min(2, len(lines))):
|
|
if _CODING_COOKIE_TEXT_RE.search(lines[_i]):
|
|
lines[_i] = _CODING_COOKIE_TEXT_RE.sub("coding_neutralized", lines[_i], count = 1)
|
|
return "\n".join(lines)
|
|
|
|
|
|
def _recovered_source(v):
|
|
"""Text an exec/compile sink actually runs: cookie-aware decode for bytes."""
|
|
if isinstance(v, (bytes, bytearray)):
|
|
return _decode_source_bytes(v)
|
|
return _to_text(v)
|
|
|
|
|
|
def _compile_mode(node, const_env):
|
|
"""Recover a compile()'s literal mode= (3rd positional or keyword), else 'exec'."""
|
|
mode_node = None
|
|
if len(node.args) >= 3:
|
|
mode_node = node.args[2]
|
|
for kw in node.keywords or []:
|
|
if kw.arg == "mode":
|
|
mode_node = kw.value
|
|
if mode_node is not None:
|
|
v = _const_fold(mode_node, const_env)
|
|
if v in ("eval", "exec", "single"):
|
|
return "eval" if v == "eval" else "exec"
|
|
return "exec"
|
|
|
|
|
|
def _walk_scope_local(scope):
|
|
"""Yield descendants of ``scope``'s body that share its namespace, WITHOUT
|
|
descending into nested def / lambda / class / comprehension (each of which is a
|
|
new scope). Used so single-assignment alias detection is scope-correct.
|
|
|
|
When ``scope`` is itself a lambda or comprehension, walk its own namespace: a lambda
|
|
body is a single expression, and a comprehension's namespace holds its element
|
|
expression plus the generator iterables / conditions (target bindings are collected
|
|
separately). ``getattr(scope, "body", [])`` only applies to def / class / module."""
|
|
if isinstance(scope, ast.Lambda):
|
|
stack = [scope.body]
|
|
elif isinstance(scope, (ast.ListComp, ast.SetComp, ast.GeneratorExp)):
|
|
stack = [scope.elt] + [g for gen in scope.generators for g in [gen.iter, *gen.ifs]]
|
|
elif isinstance(scope, ast.DictComp):
|
|
stack = [scope.key, scope.value] + [
|
|
g for gen in scope.generators for g in [gen.iter, *gen.ifs]
|
|
]
|
|
else:
|
|
stack = list(getattr(scope, "body", []))
|
|
_NESTED = (
|
|
ast.FunctionDef,
|
|
ast.AsyncFunctionDef,
|
|
ast.Lambda,
|
|
ast.ClassDef,
|
|
ast.ListComp,
|
|
ast.SetComp,
|
|
ast.DictComp,
|
|
ast.GeneratorExp,
|
|
)
|
|
while stack:
|
|
n = stack.pop()
|
|
# A nested def / lambda / class / comprehension opens its OWN scope: its body
|
|
# neither shares this namespace nor should its stores be counted here. Skip it
|
|
# entirely -- do not yield it or descend into it. (Checking the popped node,
|
|
# not just its children, is what keeps a nested `def f(): s = print` from
|
|
# inflating the outer count of an `s = os.system` single-assignment alias.)
|
|
if isinstance(n, _NESTED):
|
|
continue
|
|
yield n
|
|
for child in ast.iter_child_nodes(n):
|
|
stack.append(child)
|
|
|
|
|
|
class _ScopeAliasIndex:
|
|
"""Per-scope single-assignment aliases (shell sink / exec builtin / compiled
|
|
source) resolved with Python lexical scoping. Counting and resolution are per
|
|
function scope, so two functions binding the same local name neither cancel out
|
|
(a real sink would be missed) nor cross-contaminate (a benign call in one function
|
|
would be flagged, or a dynamic exec in another wrongly treated as a safe alias)."""
|
|
|
|
__slots__ = (
|
|
"tree",
|
|
"node_scope",
|
|
"enclosing",
|
|
"shell",
|
|
"execb",
|
|
"compiled",
|
|
"compiledany",
|
|
"impf",
|
|
"deser",
|
|
"strconst",
|
|
"rhsnode",
|
|
"assigned",
|
|
"class_shell",
|
|
"class_execb",
|
|
"class_deser",
|
|
"class_bases",
|
|
"instance_shell",
|
|
"instance_execb",
|
|
"instance_deser",
|
|
)
|
|
|
|
def __init__(self, tree):
|
|
self.tree = tree
|
|
self.node_scope: dict = {tree: tree}
|
|
self.enclosing: dict = {tree: None}
|
|
self.shell: dict = {}
|
|
self.execb: dict = {}
|
|
self.compiled: dict = {}
|
|
# name -> True: bound to a compile() result (foldable OR dynamic). Used to catch
|
|
# a code object executed through types.FunctionType(c, ...) after `c = compile(...)`.
|
|
self.compiledany: dict = {}
|
|
self.impf: dict = {} # name -> True: alias of __import__ / importlib.import_module
|
|
self.deser: dict = {} # name -> fq deserializer sink (pickle.loads, ...)
|
|
self.strconst: dict = {} # name -> folded str/bytes constant (for read scanning)
|
|
self.rhsnode: dict = {} # name -> single-assignment RHS node (for pathlib reads)
|
|
self.assigned: dict = {}
|
|
# class NAME -> {attr: sink}: a class-body alias (class C: f = os.system) accessed
|
|
# as C.f from outside the class, which lexical scope resolution does not cover.
|
|
self.class_shell: dict = {}
|
|
self.class_execb: dict = {}
|
|
self.class_deser: dict = {}
|
|
# class NAME -> [base class names]: literal Name bases, so a subclass access
|
|
# (class D(C): pass; D.s) can follow inheritance to a base's class-body sink alias.
|
|
self.class_bases: dict = {}
|
|
# (receiver_name, attr) -> sink: a simple instance-attribute alias assigned a
|
|
# dangerous callable (c.e = exec; c.e(payload) / obj.s = os.system; obj.s('rm -rf /')).
|
|
# Tracked tree-wide as a fail-closed over-approximation (attribute values are not
|
|
# lexically scoped), so a call through the same receiver name + attr is analyzed.
|
|
self.instance_shell: dict = {}
|
|
self.instance_execb: dict = {}
|
|
self.instance_deser: dict = {}
|
|
|
|
def resolve_class_attr(
|
|
self,
|
|
cname,
|
|
attr,
|
|
kind,
|
|
_seen = None,
|
|
):
|
|
table = getattr(self, "class_" + kind)
|
|
m = table.get(cname)
|
|
if m and attr in m:
|
|
return m[attr]
|
|
# Follow literal base classes so an inherited alias (class C: s = os.system;
|
|
# class D(C): pass; D.s(...)) resolves through C. Cycle-guarded.
|
|
if _seen is None:
|
|
_seen = set()
|
|
if cname in _seen:
|
|
return None
|
|
_seen.add(cname)
|
|
for _base in self.class_bases.get(cname, ()):
|
|
hit = self.resolve_class_attr(_base, attr, kind, _seen)
|
|
if hit is not None:
|
|
return hit
|
|
return None
|
|
|
|
def resolve_instance_attr(self, recv, attr, kind):
|
|
return getattr(self, "instance_" + kind).get((recv, attr))
|
|
|
|
def _chain(self, node):
|
|
s = self.node_scope.get(node, self.tree)
|
|
while s is not None:
|
|
yield s
|
|
s = self.enclosing.get(s)
|
|
|
|
def resolve(self, name, node, kind):
|
|
maps = getattr(self, kind)
|
|
for s in self._chain(node):
|
|
m = maps.get(s)
|
|
if m and name in m:
|
|
return m[name]
|
|
if name in self.assigned.get(s, ()): # locally shadowed by a non-alias
|
|
return None
|
|
return None
|
|
|
|
def effective(self, node, kind):
|
|
maps = getattr(self, kind)
|
|
result: dict = {}
|
|
shadowed: set = set()
|
|
for s in self._chain(node):
|
|
for k, v in maps.get(s, {}).items():
|
|
if k not in shadowed and k not in result:
|
|
result[k] = v
|
|
shadowed |= self.assigned.get(s, set())
|
|
return result
|
|
|
|
|
|
def _build_scope_alias_index(tree, const_env):
|
|
idx = _ScopeAliasIndex(tree)
|
|
|
|
def _rec(node, scope, func_enclose):
|
|
# scope: namespace the direct children belong to (for node_scope + counting).
|
|
# func_enclose: the scope a nested FUNCTION / class body encloses to. Python skips
|
|
# class scope for nested functions, so inside a class body this stays the class's
|
|
# own lexical function/module parent rather than the class.
|
|
for child in ast.iter_child_nodes(node):
|
|
idx.node_scope[child] = scope
|
|
if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)):
|
|
# A lambda, like a def, opens its own scope: a default-bound param alias
|
|
# ((lambda e=exec: e(payload))()) lives in the lambda body's namespace, and
|
|
# anything nested inside encloses to the lambda itself.
|
|
idx.enclosing[child] = func_enclose
|
|
_rec(child, child, child)
|
|
elif isinstance(child, (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp)):
|
|
# A comprehension opens its own scope in Python 3; its target bindings
|
|
# ([e(p) for e in [exec]]) belong to that scope, enclosing to func_enclose.
|
|
idx.enclosing[child] = func_enclose
|
|
_rec(child, child, func_enclose)
|
|
elif isinstance(child, ast.ClassDef):
|
|
# A class body executes immediately with its OWN namespace, so it is a
|
|
# real alias scope (class C: e = eval; e(...) runs eval), but its names
|
|
# are not visible to methods defined inside it -- those enclose to
|
|
# func_enclose, skipping the class.
|
|
idx.enclosing[child] = func_enclose
|
|
_rec(child, child, func_enclose)
|
|
else:
|
|
_rec(child, scope, func_enclose)
|
|
|
|
_rec(tree, tree, tree)
|
|
|
|
# os / subprocess import + from-import aliases are collected tree-wide (imports
|
|
# are lexically visible module-wide in practice) and shared across scopes.
|
|
os_aliases = {"os"}
|
|
subprocess_aliases = {"subprocess"}
|
|
from_aliases: dict[str, str] = {}
|
|
builtins_aliases = {"builtins", "__builtins__"}
|
|
importlib_aliases = {"importlib"}
|
|
# `compile` bound by name (bare builtin or `from builtins import compile as comp`).
|
|
compile_aliases = {"compile"}
|
|
deser_module_aliases: dict[str, str] = {m: m for m in _DESERIALIZE_MODULES}
|
|
for n in ast.walk(tree):
|
|
if isinstance(n, ast.Import):
|
|
for a in n.names:
|
|
if a.name == "os":
|
|
os_aliases.add(a.asname or "os")
|
|
elif a.name in ("posix", "nt"):
|
|
# posix / nt are the C backend os wraps (posix.system == os.system), so a
|
|
# single-assignment alias s = posix.system resolves to an os shell sink.
|
|
os_aliases.add(a.asname or a.name)
|
|
elif a.name == "subprocess":
|
|
subprocess_aliases.add(a.asname or "subprocess")
|
|
elif a.name == "builtins":
|
|
builtins_aliases.add(a.asname or "builtins")
|
|
elif a.name == "importlib":
|
|
importlib_aliases.add(a.asname or "importlib")
|
|
if a.name in _DESERIALIZE_MODULES:
|
|
deser_module_aliases[a.asname or a.name] = a.name
|
|
elif isinstance(n, ast.ImportFrom) and n.module in ("os", "subprocess", "posix", "nt"):
|
|
_eff = "subprocess" if n.module == "subprocess" else "os"
|
|
for a in n.names:
|
|
fq = f"{_eff}.{a.name}"
|
|
if fq in _SHELL_SINK_FUNCS:
|
|
from_aliases[a.asname or a.name] = fq
|
|
elif isinstance(n, ast.ImportFrom) and n.module == "builtins":
|
|
for a in n.names:
|
|
if a.name == "compile":
|
|
compile_aliases.add(a.asname or "compile")
|
|
|
|
def _rhs_is_compile_call(rhs):
|
|
# `compile(...)` reached as the bare builtin, `builtins.compile(...)`, or a
|
|
# `from builtins import compile as comp` alias -- the callee forms that produce a
|
|
# code object bound to a name (for the types.FunctionType(c) execution gadget).
|
|
if not isinstance(rhs, ast.Call):
|
|
return False
|
|
f = rhs.func
|
|
if isinstance(f, ast.Name):
|
|
return f.id in compile_aliases
|
|
if (
|
|
isinstance(f, ast.Attribute)
|
|
and f.attr == "compile"
|
|
and isinstance(f.value, ast.Name)
|
|
and f.value.id in builtins_aliases
|
|
):
|
|
return True
|
|
return False
|
|
|
|
def _rhs_exec_builtin(rhs):
|
|
# bare `exec` / `eval` / `compile`, or `builtins.eval` (attribute form).
|
|
if isinstance(rhs, ast.Name) and rhs.id in _EXEC_BUILTINS:
|
|
return rhs.id
|
|
if (
|
|
isinstance(rhs, ast.Attribute)
|
|
and rhs.attr in _EXEC_BUILTINS
|
|
and isinstance(rhs.value, ast.Name)
|
|
and rhs.value.id in builtins_aliases
|
|
):
|
|
return rhs.attr
|
|
return None
|
|
|
|
def _rhs_import_func(rhs):
|
|
# `__import__` / `importlib.import_module` (+ reload) bound to a name.
|
|
if isinstance(rhs, ast.Name) and rhs.id in ("__import__", "import_module"):
|
|
return True
|
|
if (
|
|
isinstance(rhs, ast.Attribute)
|
|
and rhs.attr in ("import_module", "reload", "__import__")
|
|
and isinstance(rhs.value, ast.Name)
|
|
and rhs.value.id in importlib_aliases
|
|
):
|
|
return True
|
|
return False
|
|
|
|
def _rhs_deserializer(rhs):
|
|
# `pickle.loads` (+ aliased module) bound to a name.
|
|
if isinstance(rhs, ast.Attribute) and isinstance(rhs.value, ast.Name):
|
|
canon = deser_module_aliases.get(rhs.value.id)
|
|
if canon is not None:
|
|
fq = f"{canon}.{rhs.attr}"
|
|
if fq in _CODE_DESERIALIZE_SINKS:
|
|
return fq
|
|
return None
|
|
|
|
def _unwrap_container_index(rhs):
|
|
# `s = [os.system][0]` / `e = {'e': exec}['e']` / `l = (pickle.loads,)[0]`: the
|
|
# unwrapped callable is assigned first, then called. Resolve the inline
|
|
# literal-container index to the element node so the sink resolvers below see the
|
|
# real callable instead of an opaque Subscript.
|
|
if not isinstance(rhs, ast.Subscript):
|
|
return rhs
|
|
container = rhs.value
|
|
ci = _const_fold(rhs.slice, const_env)
|
|
if isinstance(container, (ast.List, ast.Tuple)) and isinstance(ci, int):
|
|
if -len(container.elts) <= ci < len(container.elts):
|
|
return container.elts[ci]
|
|
if isinstance(container, ast.Dict) and ci is not None:
|
|
for k, v in zip(container.keys, container.values):
|
|
if k is not None and _const_fold(k, const_env) == ci:
|
|
return v
|
|
return rhs
|
|
|
|
scopes = [tree] + [
|
|
n
|
|
for n in ast.walk(tree)
|
|
if isinstance(
|
|
n,
|
|
(
|
|
ast.FunctionDef,
|
|
ast.AsyncFunctionDef,
|
|
ast.ClassDef,
|
|
ast.Lambda,
|
|
ast.ListComp,
|
|
ast.SetComp,
|
|
ast.DictComp,
|
|
ast.GeneratorExp,
|
|
),
|
|
)
|
|
]
|
|
for scope in scopes:
|
|
counts: dict[str, int] = {}
|
|
rebound: set[str] = set()
|
|
global_names: set[str] = set()
|
|
nonlocal_names: set[str] = set()
|
|
assigns: list[tuple[str, ast.expr]] = []
|
|
allnames: set[str] = set()
|
|
# Function parameters bind local names that lexically shadow an outer alias of
|
|
# the same name, so count them as local assignments for the shadowing rules.
|
|
_sargs = getattr(scope, "args", None)
|
|
if _sargs is not None:
|
|
for _a in list(_sargs.posonlyargs) + list(_sargs.args) + list(_sargs.kwonlyargs):
|
|
allnames.add(_a.arg)
|
|
for _extra in (_sargs.vararg, _sargs.kwarg):
|
|
if _extra is not None:
|
|
allnames.add(_extra.arg)
|
|
for n in _walk_scope_local(scope):
|
|
if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Store):
|
|
counts[n.id] = counts.get(n.id, 0) + 1
|
|
allnames.add(n.id)
|
|
elif isinstance(n, ast.Global):
|
|
rebound.update(n.names)
|
|
global_names.update(n.names)
|
|
elif isinstance(n, ast.Nonlocal):
|
|
rebound.update(n.names)
|
|
nonlocal_names.update(n.names)
|
|
elif (
|
|
isinstance(n, ast.Assign)
|
|
and len(n.targets) == 1
|
|
and isinstance(n.targets[0], ast.Name)
|
|
):
|
|
assigns.append((n.targets[0].id, n.value))
|
|
elif (
|
|
# An annotated single-assignment (e: object = exec) is a real binding whose
|
|
# RHS must be recorded like a plain Assign, else e(payload) skips analysis.
|
|
isinstance(n, ast.AnnAssign)
|
|
and n.value is not None
|
|
and isinstance(n.target, ast.Name)
|
|
):
|
|
assigns.append((n.target.id, n.value))
|
|
elif (
|
|
# A parallel unpacking assignment binds each name to the matching RHS element
|
|
# ((s,) = (os.system,); [e] = [exec]; a, b = os.system, 1), which then reaches
|
|
# a sink call the same way a plain alias does. Pair a literal tuple/list target
|
|
# with a literal tuple/list RHS of equal length element-wise so those aliases
|
|
# are recorded; a starred / mismatched / non-literal RHS is left alone.
|
|
isinstance(n, ast.Assign)
|
|
and len(n.targets) == 1
|
|
and isinstance(n.targets[0], (ast.Tuple, ast.List))
|
|
and isinstance(n.value, (ast.Tuple, ast.List))
|
|
and len(n.targets[0].elts) == len(n.value.elts)
|
|
and not any(isinstance(_t, ast.Starred) for _t in n.targets[0].elts)
|
|
):
|
|
for _tgt, _val in zip(n.targets[0].elts, n.value.elts):
|
|
if isinstance(_tgt, ast.Name):
|
|
assigns.append((_tgt.id, _val))
|
|
# A comprehension generator binds its target like a single-assignment alias when the
|
|
# iterable is a one-element literal: [e(p) for e in [exec]] binds e to exec, so the
|
|
# payload passed through e must still get eval/exec recursion.
|
|
for _gen in getattr(scope, "generators", []):
|
|
if (
|
|
isinstance(_gen.target, ast.Name)
|
|
and isinstance(_gen.iter, (ast.List, ast.Tuple, ast.Set))
|
|
and len(_gen.iter.elts) == 1
|
|
):
|
|
_tn = _gen.target.id
|
|
counts[_tn] = counts.get(_tn, 0) + 1
|
|
allnames.add(_tn)
|
|
assigns.append((_tn, _gen.iter.elts[0]))
|
|
# A `global`/`nonlocal`-declared name is NOT a local binding, so it must not shadow an
|
|
# outer alias here (its assignment rebinds the target scope instead).
|
|
allnames -= rebound
|
|
idx.assigned[scope] = allnames
|
|
smap: dict[str, str] = {}
|
|
emap: dict[str, str] = {}
|
|
cmap: dict[str, tuple] = {}
|
|
camap: dict[str, bool] = {}
|
|
imap: dict[str, bool] = {}
|
|
dmap: dict[str, str] = {}
|
|
scmap: dict[str, object] = {}
|
|
rnmap: dict[str, ast.expr] = {}
|
|
# Process assignments in SOURCE order so a chained single-assignment alias resolves
|
|
# against the earlier binding it copies (s = os.system; t = s -> t is os.system). The
|
|
# scope walk yields assignments out of order, so sort by the RHS position; Python
|
|
# binds top-to-bottom, so the aliased name is always defined on an earlier line.
|
|
assigns.sort(key = lambda _p: (getattr(_p[1], "lineno", 0), getattr(_p[1], "col_offset", 0)))
|
|
for name, rhs in assigns:
|
|
if counts.get(name) != 1 or name in rebound:
|
|
continue
|
|
# Single-assignment RHS node, used by the read scanner to resolve a pathlib
|
|
# expression bound to a name (p = Path('..') / 'etc' / 'passwd'; p.read_text()).
|
|
rnmap[name] = rhs
|
|
# An inline-container index RHS (s = [os.system][0]) hides the callable; resolve
|
|
# it to the element so the sink resolvers below see the real sink.
|
|
rhs_eff = _unwrap_container_index(rhs)
|
|
fq = _resolve_static_shell_sink(rhs_eff, os_aliases, subprocess_aliases, from_aliases)
|
|
# A chained single-assignment alias (s = os.system; t = s; t('rm -rf /')): the RHS
|
|
# is a bare Name already resolved to a sink earlier in this scope (assigns are in
|
|
# source order), so propagate its sink identity instead of dropping it.
|
|
if fq is None and isinstance(rhs_eff, ast.Name) and rhs_eff.id in smap:
|
|
fq = smap[rhs_eff.id]
|
|
if fq:
|
|
smap[name] = fq
|
|
eb = _rhs_exec_builtin(rhs_eff)
|
|
if eb is None and isinstance(rhs_eff, ast.Name) and rhs_eff.id in emap:
|
|
eb = emap[rhs_eff.id] # chained alias e = exec; f = e; f(payload)
|
|
if eb is not None:
|
|
emap[name] = eb
|
|
elif (
|
|
_rhs_is_compile_call(rhs_eff)
|
|
or (
|
|
# A local alias of compile (cfn = compile; co = cfn(src, ...)) is not in
|
|
# compile_aliases, so resolve the callee through this scope's exec-builtin
|
|
# map (built in source order, cfn precedes co) before giving up.
|
|
isinstance(rhs_eff, ast.Call)
|
|
and isinstance(rhs_eff.func, ast.Name)
|
|
and emap.get(rhs_eff.func.id) == "compile"
|
|
)
|
|
) and rhs_eff.args:
|
|
# Any `c = compile(...)` (bare / builtins.compile / from-import alias)
|
|
# binds a code object, tracked for the types.FunctionType(c) execution
|
|
# gadget below (dynamic or foldable payload).
|
|
camap[name] = True
|
|
v = _const_fold(rhs_eff.args[0], const_env)
|
|
if isinstance(v, (str, bytes, bytearray)):
|
|
cmap[name] = (
|
|
_recovered_source(v),
|
|
_compile_mode(rhs_eff, const_env),
|
|
isinstance(v, (bytes, bytearray)),
|
|
)
|
|
if _rhs_import_func(rhs_eff):
|
|
imap[name] = True
|
|
dfq = _rhs_deserializer(rhs_eff)
|
|
if dfq is None and isinstance(rhs_eff, ast.Name) and rhs_eff.id in dmap:
|
|
dfq = dmap[rhs_eff.id] # chained alias d = pickle.loads; e = d; e(payload)
|
|
if dfq is not None:
|
|
dmap[name] = dfq
|
|
# Single-assignment string/bytes path constant (p = '/etc/passwd'), used by
|
|
# the sensitive-read scanner to fold function-local read paths.
|
|
cv = _const_fold(rhs, const_env)
|
|
if isinstance(cv, (str, bytes, bytearray)):
|
|
scmap[name] = cv
|
|
# A parameter DEFAULT that is a dangerous callable acts as an alias inside the body:
|
|
# def f(e=exec): e(payload) / def f(s=os.system): s('rm -rf /'). Bind it like a
|
|
# single-assignment alias unless the parameter is reassigned in the body.
|
|
if _sargs is not None:
|
|
_pos = list(_sargs.posonlyargs) + list(_sargs.args)
|
|
_paired = list(zip(_pos[len(_pos) - len(_sargs.defaults) :], _sargs.defaults))
|
|
_paired += [
|
|
(a, d) for a, d in zip(_sargs.kwonlyargs, _sargs.kw_defaults) if d is not None
|
|
]
|
|
for _p, _d in _paired:
|
|
_pn = _p.arg
|
|
if counts.get(_pn, 0) != 0 or _pn in rebound:
|
|
continue
|
|
_de = _unwrap_container_index(_d)
|
|
_dfq_sh = _resolve_static_shell_sink(
|
|
_de, os_aliases, subprocess_aliases, from_aliases
|
|
)
|
|
if _dfq_sh and _pn not in smap:
|
|
smap[_pn] = _dfq_sh
|
|
_deb = _rhs_exec_builtin(_de)
|
|
if _deb is not None and _pn not in emap:
|
|
emap[_pn] = _deb
|
|
_ddfq = _rhs_deserializer(_de)
|
|
if _ddfq is not None and _pn not in dmap:
|
|
dmap[_pn] = _ddfq
|
|
if _rhs_import_func(_de) and _pn not in imap:
|
|
imap[_pn] = True
|
|
# A `global name = <sink>` (or `nonlocal name = <sink>`) rebinds the name in the
|
|
# TARGET scope (module for global, nearest enclosing scope for nonlocal), NOT locally,
|
|
# so `global s; s = os.system; s('rm -rf /')` must record the alias in that target
|
|
# scope -- otherwise the local pass skips it (name in rebound) and the call resolves to
|
|
# nothing. Target scopes are processed before nested scopes, so setdefault preserves
|
|
# any alias they already hold.
|
|
if global_names or nonlocal_names:
|
|
for name, rhs in assigns:
|
|
if name in global_names:
|
|
_target = tree
|
|
elif name in nonlocal_names:
|
|
_target = idx.enclosing.get(scope)
|
|
else:
|
|
continue
|
|
if _target is None:
|
|
continue
|
|
_rhs_eff = _unwrap_container_index(rhs)
|
|
_gfq = _resolve_static_shell_sink(
|
|
_rhs_eff, os_aliases, subprocess_aliases, from_aliases
|
|
)
|
|
if _gfq:
|
|
idx.shell.setdefault(_target, {}).setdefault(name, _gfq)
|
|
_geb = _rhs_exec_builtin(_rhs_eff)
|
|
if _geb is not None:
|
|
idx.execb.setdefault(_target, {}).setdefault(name, _geb)
|
|
_gdfq = _rhs_deserializer(_rhs_eff)
|
|
if _gdfq is not None:
|
|
idx.deser.setdefault(_target, {}).setdefault(name, _gdfq)
|
|
if smap:
|
|
idx.shell[scope] = smap
|
|
if emap:
|
|
idx.execb[scope] = emap
|
|
if cmap:
|
|
idx.compiled[scope] = cmap
|
|
if camap:
|
|
idx.compiledany[scope] = camap
|
|
if imap:
|
|
idx.impf[scope] = imap
|
|
if dmap:
|
|
idx.deser[scope] = dmap
|
|
if scmap:
|
|
idx.strconst[scope] = scmap
|
|
if rnmap:
|
|
idx.rhsnode[scope] = rnmap
|
|
# Class-body aliases are also reachable as ClassName.attr from OUTSIDE the class
|
|
# (class C: f = os.system; C.f('rm -rf /')), which lexical scope resolution does not
|
|
# cover, so index them by the class name too.
|
|
if isinstance(scope, ast.ClassDef):
|
|
if smap:
|
|
idx.class_shell[scope.name] = dict(smap)
|
|
if emap:
|
|
idx.class_execb[scope.name] = dict(emap)
|
|
if dmap:
|
|
idx.class_deser[scope.name] = dict(dmap)
|
|
_bases = [b.id for b in scope.bases if isinstance(b, ast.Name)]
|
|
if _bases:
|
|
idx.class_bases[scope.name] = _bases
|
|
# Instance-attribute sink aliases (c.e = exec; c.e(payload) / obj.s = os.system;
|
|
# obj.s('rm -rf /')): a simple `Name.attr = <sink>` store binds the attribute to a
|
|
# dangerous callable. Tracked tree-wide by (receiver_name, attr) as a fail-closed
|
|
# over-approximation, so a later call through the same receiver name gets analyzed.
|
|
for n in ast.walk(tree):
|
|
if not (
|
|
isinstance(n, ast.Assign)
|
|
and len(n.targets) == 1
|
|
and isinstance(n.targets[0], ast.Attribute)
|
|
and isinstance(n.targets[0].value, ast.Name)
|
|
):
|
|
continue
|
|
key = (n.targets[0].value.id, n.targets[0].attr)
|
|
rhs_eff = _unwrap_container_index(n.value)
|
|
_fq = _resolve_static_shell_sink(rhs_eff, os_aliases, subprocess_aliases, from_aliases)
|
|
if _fq:
|
|
idx.instance_shell[key] = _fq
|
|
_eb = _rhs_exec_builtin(rhs_eff)
|
|
if _eb is not None:
|
|
idx.instance_execb[key] = _eb
|
|
_dfq = _rhs_deserializer(rhs_eff)
|
|
if _dfq is not None:
|
|
idx.instance_deser[key] = _dfq
|
|
return idx
|
|
|
|
|
|
def _payload_has_obfuscation_primitive(node):
|
|
"""True when a (non-plain) exec/eval payload is assembled from decode / fetch /
|
|
runtime-assembly primitives -- the canonical loader shapes that are essentially
|
|
never benign inside a sandbox."""
|
|
if node is None or isinstance(node, (ast.Name, ast.Constant)):
|
|
return False
|
|
for sub in ast.walk(node):
|
|
if isinstance(sub, ast.Call):
|
|
fn = sub.func
|
|
fq = _fq_attr_name(fn)
|
|
attr = (
|
|
fn.attr
|
|
if isinstance(fn, ast.Attribute)
|
|
else (fn.id if isinstance(fn, ast.Name) else "")
|
|
)
|
|
if fq in _CODE_DESERIALIZE_SINKS:
|
|
return True
|
|
# A dynamic exec payload produced by another eval/exec/compile is a
|
|
# nested-dynamic-exec obfuscation (also fails closed on eval(eval(...))).
|
|
if isinstance(fn, ast.Name) and fn.id in _EXEC_BUILTINS:
|
|
return True
|
|
if attr in _DECODE_ATTRS or attr in ("decode", "translate"):
|
|
return True
|
|
if fq and any(fq.startswith(p) for p in _FETCH_FQ_PREFIXES):
|
|
return True
|
|
if attr == "join" and sub.args:
|
|
a0 = sub.args[0]
|
|
if isinstance(a0, (ast.GeneratorExp, ast.ListComp, ast.SetComp)):
|
|
return True
|
|
if (
|
|
isinstance(a0, ast.Call)
|
|
and isinstance(a0.func, ast.Name)
|
|
and a0.func.id in ("map", "filter")
|
|
):
|
|
return True
|
|
if isinstance(fn, ast.Name) and fn.id in ("bytes", "bytearray") and sub.args:
|
|
a0 = sub.args[0]
|
|
if isinstance(a0, (ast.GeneratorExp, ast.ListComp, ast.SetComp)):
|
|
return True
|
|
elif isinstance(sub, ast.Attribute) and sub.attr in ("text", "content"):
|
|
if isinstance(sub.value, ast.Call):
|
|
return True
|
|
elif isinstance(sub, ast.Subscript) and isinstance(sub.slice, ast.Slice):
|
|
if sub.slice.step is not None and _const_fold(sub.slice.step) == -1:
|
|
return True
|
|
elif isinstance(sub, ast.BinOp) and isinstance(sub.op, ast.Mult):
|
|
# Large string/bytes repetition assembles an oversized payload (parse
|
|
# bomb) that the folder refuses on size; fail closed.
|
|
for a, b in ((sub.left, sub.right), (sub.right, sub.left)):
|
|
sv = _const_fold(a)
|
|
nv = _const_fold(b)
|
|
if isinstance(sv, (str, bytes, bytearray)) and isinstance(nv, int) and nv >= 1024:
|
|
return True
|
|
return False
|
|
|
|
|
|
def _safe_parse_inner(src, mode, depth, budget):
|
|
"""DoS-safe gateway to ast.parse on an attacker-influenced payload string.
|
|
|
|
Returns one of ("PARSED", tree|None), ("SYNTAX_BAD", None), ("BOUND_HIT", None).
|
|
A linear bracket-depth pre-scan rejects pathological nesting *before* the C
|
|
parser runs (defends the CPython C-stack overflow on deeply-nested input)."""
|
|
if depth >= _MAX_UNWRAP_DEPTH:
|
|
return ("BOUND_HIT", None)
|
|
if len(src) > _MAX_INNER_SRC:
|
|
return ("BOUND_HIT", None)
|
|
if budget.inner_parses >= _MAX_INNER_PARSES:
|
|
return ("BOUND_HIT", None)
|
|
if budget.inner_chars + len(src) > _MAX_TOTAL_INNER_CHARS:
|
|
return ("BOUND_HIT", None)
|
|
if _bracket_depth(src) > _MAX_BRACKET_DEPTH:
|
|
return ("BOUND_HIT", None)
|
|
budget.inner_parses += 1
|
|
budget.inner_chars += len(src)
|
|
try:
|
|
return ("PARSED", ast.parse(src, mode = mode))
|
|
except SyntaxError:
|
|
try:
|
|
ast.parse(src, mode = ("exec" if mode == "eval" else "eval"))
|
|
return ("PARSED", None)
|
|
except SyntaxError:
|
|
return ("SYNTAX_BAD", None)
|
|
except (RecursionError, MemoryError, ValueError):
|
|
return ("BOUND_HIT", None)
|
|
|
|
|
|
def _first_unsafe_reason(info):
|
|
for key in (
|
|
"shell_escapes",
|
|
"dynamic_exec",
|
|
"network_calls",
|
|
"sensitive_file_reads",
|
|
"filesystem_violations",
|
|
"signal_tampering",
|
|
"exception_catching",
|
|
):
|
|
for item in info.get(key, []) or []:
|
|
desc = item.get("description")
|
|
if desc:
|
|
return desc
|
|
return "unsafe operation"
|
|
|
|
|
|
def _recover_exec_payload(node, func_id, const_env, compiled_env):
|
|
"""Recover a statically foldable source string for eval/exec/compile.
|
|
|
|
Returns ("RECOVERED", src, mode, is_bytes) / ("DYNAMIC", None, None, False) /
|
|
("NO_PAYLOAD", None, None, False). ``is_bytes`` records that the payload folded to
|
|
a bytes/bytearray literal -- ``exec``/``compile`` honor PEP 263 coding cookies on
|
|
bytes, so a bytes payload that fails to parse as UTF-8 Python is treated as an
|
|
obfuscation vector by the caller rather than a harmless SyntaxError.
|
|
"""
|
|
if not node.args:
|
|
return ("NO_PAYLOAD", None, None, False)
|
|
arg0 = node.args[0]
|
|
base_mode = "eval" if func_id == "eval" else "exec"
|
|
|
|
# exec(compile("...", ...)) / eval(compile("...", "<s>", "eval"))
|
|
if (
|
|
isinstance(arg0, ast.Call)
|
|
and isinstance(arg0.func, ast.Name)
|
|
and arg0.func.id == "compile"
|
|
and arg0.args
|
|
):
|
|
v = _const_fold(arg0.args[0], const_env)
|
|
if isinstance(v, (str, bytes, bytearray)):
|
|
return (
|
|
"RECOVERED",
|
|
_recovered_source(v),
|
|
_compile_mode(arg0, const_env),
|
|
isinstance(v, (bytes, bytearray)),
|
|
)
|
|
return ("DYNAMIC", None, None, False)
|
|
|
|
# c = compile("..."); exec(c)
|
|
if isinstance(arg0, ast.Name) and arg0.id in compiled_env:
|
|
csrc, cmode, cbytes = compiled_env[arg0.id]
|
|
return ("RECOVERED", csrc, cmode, cbytes)
|
|
|
|
v = _const_fold(arg0, const_env)
|
|
if isinstance(v, (str, bytes, bytearray)):
|
|
mode = _compile_mode(node, const_env) if func_id == "compile" else base_mode
|
|
return ("RECOVERED", _recovered_source(v), mode, isinstance(v, (bytes, bytearray)))
|
|
return ("DYNAMIC", None, None, False)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Stage 3: static sensitive-read scanner.
|
|
#
|
|
# Filesystem WRITE confinement is enforced at runtime by the realpath backstop
|
|
# (Stage 5), which is strictly more robust than static path proving. Reads are not
|
|
# confined there, so this small static pass blocks sandboxed code from reading host
|
|
# secrets: any call arg that folds to a sensitive host path (covers open()/os.open
|
|
# and library loaders like pandas.read_csv('/etc/shadow')), plus `..`/`~` traversal
|
|
# on the dedicated open()/read callees. Dynamic paths are left to the backstop.
|
|
# --------------------------------------------------------------------------
|
|
|
|
# Sensitive read targets: exact host-identity / credential files, credential dirs,
|
|
# and the classic /proc self-inspection paths. Substring tokens are only consulted
|
|
# for absolute or ~-rooted paths with no whitespace (avoids sentence false positives).
|
|
_SANDBOX_SENSITIVE_EXACT = frozenset(
|
|
{"/etc/passwd", "/etc/shadow", "/etc/sudoers", "/etc/gshadow", "/etc/master.passwd"}
|
|
)
|
|
_SANDBOX_SENSITIVE_DIR_PARTS = (
|
|
"/etc/ssh/",
|
|
"/root/",
|
|
"/.ssh/",
|
|
"/.aws/",
|
|
"/.config/gcloud",
|
|
"/.kube/",
|
|
"/.docker/",
|
|
# In-cluster Kubernetes service-account credentials (token / ca.crt / namespace)
|
|
# mounted into every pod; reading the token impersonates the pod to the API server.
|
|
"/var/run/secrets/kubernetes.io/",
|
|
"/run/secrets/kubernetes.io/",
|
|
)
|
|
_SANDBOX_SENSITIVE_TOKENS = (
|
|
"id_rsa",
|
|
"id_ed25519",
|
|
".pem",
|
|
".netrc",
|
|
"credentials",
|
|
".git-credentials",
|
|
"/.huggingface/token",
|
|
".kube/config",
|
|
)
|
|
_SANDBOX_SENSITIVE_RE = re.compile(
|
|
r"^/proc/(?:self|\d+)/(?:environ|cmdline|maps|mem|task/\d+/environ)$"
|
|
)
|
|
|
|
|
|
def _is_sensitive_abs_path(s):
|
|
"""Provably-sensitive absolute (or ~-rooted) path, whitespace-free."""
|
|
if not isinstance(s, str) or not s:
|
|
return False
|
|
norm = s.replace("\\", "/")
|
|
if any(ch.isspace() for ch in norm):
|
|
return False
|
|
if not (norm.startswith("/") or norm.startswith("~")):
|
|
return False
|
|
if norm in _SANDBOX_SENSITIVE_EXACT:
|
|
return True
|
|
# The directory markers carry a trailing slash to match descendants (/root/id_rsa);
|
|
# append one to the candidate so the directory ITSELF (an unguarded `ls /root` /
|
|
# `find /etc/ssh`) matches too, without loosening the component boundary.
|
|
if any(part in (norm + "/") for part in _SANDBOX_SENSITIVE_DIR_PARTS):
|
|
return True
|
|
if _SANDBOX_SENSITIVE_RE.match(norm):
|
|
return True
|
|
low = norm.lower()
|
|
return any(tok in low for tok in _SANDBOX_SENSITIVE_TOKENS)
|
|
|
|
|
|
# Punctuation separators plus the bash compound-statement keywords (if/while/until/then/do/
|
|
# else/elif), so a reader in a CONDITION body (`if cat $SECRET; then :; fi`) is scanned at
|
|
# command position rather than treated as an argument of the keyword.
|
|
_READ_SCAN_SEPARATORS = (
|
|
";",
|
|
"&&",
|
|
"||",
|
|
"|",
|
|
"&",
|
|
"(",
|
|
")",
|
|
"`",
|
|
"{",
|
|
"}",
|
|
"\n",
|
|
) + tuple(_SHELL_KEYWORDS_AS_SEP)
|
|
|
|
|
|
def _join_chdir(base, newdir):
|
|
"""Resolve an ``env -C DIR`` / ``--chdir=DIR`` operand against the current child cwd.
|
|
|
|
A relative DIR chdirs relative to wherever the child already is (an ambient
|
|
subprocess ``cwd=`` or a prior ``env -C``), so ``env -C . cat passwd`` under
|
|
``cwd=/etc`` still reads ``/etc/passwd``; join it onto the current base rather
|
|
than replacing the base with the bare relative fragment. An absolute / ``~``
|
|
DIR overrides the base outright."""
|
|
if newdir.startswith("/") or newdir.startswith("~"):
|
|
return newdir
|
|
return os.path.join(base, newdir) if base else newdir
|
|
|
|
|
|
def _extract_command_subs(s):
|
|
"""Extract the inner payloads of ``$(...)`` and backtick command substitutions from a
|
|
shell string, INCLUDING those inside double quotes (bash runs a substitution regardless
|
|
of surrounding quotes: ``echo "$(head /etc/passwd)"``). Returns a list of inner command
|
|
strings for recursive read scanning. ``$((arith))`` yields a harmless ``(arith)`` payload
|
|
that scans clean."""
|
|
subs = []
|
|
i, n = 0, len(s)
|
|
while i < n:
|
|
c = s[i]
|
|
if c == "`":
|
|
j = s.find("`", i + 1)
|
|
if j == -1:
|
|
break
|
|
subs.append(s[i + 1 : j])
|
|
i = j + 1
|
|
elif c == "$" and i + 1 < n and s[i + 1] == "(":
|
|
depth = 1
|
|
k = i + 2
|
|
while k < n and depth:
|
|
if s[k] == "(":
|
|
depth += 1
|
|
elif s[k] == ")":
|
|
depth -= 1
|
|
k += 1
|
|
subs.append(s[i + 2 : k - 1] if depth == 0 else s[i + 2 : k])
|
|
i = k
|
|
else:
|
|
i += 1
|
|
return subs
|
|
|
|
|
|
def _argv_env_chdir(str_elts):
|
|
"""Extract an ``env -C DIR`` / ``--chdir[=DIR]`` target from a folded argv vector.
|
|
|
|
A wrapper-prefixed child that chdirs before the reader (``['env', '-C', '/etc', 'cat',
|
|
'passwd']``) reads ``/etc/passwd`` in the unguarded child, so the relative reader arg must
|
|
be resolved against DIR. Returns the DIR string (or None). Only the ``env`` wrapper honors
|
|
``-C``; other flags / wrappers are skipped until the real command word is reached."""
|
|
i, n = 0, len(str_elts)
|
|
wrapper = None
|
|
while i < n:
|
|
tok = str_elts[i]
|
|
if tok is None:
|
|
return None
|
|
if _ASSIGNMENT_RE.match(tok):
|
|
i += 1
|
|
continue
|
|
if tok.startswith("-"):
|
|
if wrapper == "env":
|
|
if tok in ("-C", "--chdir"):
|
|
return str_elts[i + 1] if i + 1 < n else None
|
|
if tok.startswith("--chdir="):
|
|
return tok.split("=", 1)[1]
|
|
if _wrapper_flag_takes_operand("env", tok):
|
|
i += 2
|
|
continue
|
|
i += 1
|
|
continue
|
|
base = os.path.basename(tok).lower()
|
|
if base in _COMMAND_PREFIXES:
|
|
wrapper = base
|
|
i += 1
|
|
continue
|
|
return None # reached the executed command word before any env -C
|
|
return None
|
|
|
|
|
|
def _scan_command_string_for_reads(
|
|
command,
|
|
*,
|
|
strict_traversal,
|
|
cwd = None,
|
|
cwd_dynamic = False,
|
|
_depth = 0,
|
|
):
|
|
"""Scan a shell command STRING for an embedded host-secret read; return a short reason or
|
|
None. Covers literal sensitive / traversal paths, input redirects, and $ / backtick /
|
|
escaping-glob expansions on file-reading commands. Reads from a shell child are not
|
|
runtime-confined, so these are refused statically.
|
|
|
|
strict_traversal=True blocks ANY ``..`` / ``~`` read path (the os.system() shell-string
|
|
policy in the Python tool); False blocks only a traversal that resolves onto a sensitive
|
|
path, so ordinary terminal relative navigation (``ls ../src``) is not flagged.
|
|
|
|
The reader / command word is resolved past leading ``VAR=value`` assignments and command
|
|
wrappers (env / sudo / nice / timeout / ...), a ``VAR=value`` assignment whose value is a
|
|
sensitive path is flagged, and a nested ``bash -c '<payload>'`` shell has its payload
|
|
recursively scanned, so a read hidden behind a normal command-prefix form is still caught."""
|
|
if not command or _depth > 6:
|
|
return None
|
|
# Model bash brace expansion so a brace-hidden reader / path (`{cat,/etc/passwd}`) is seen.
|
|
cmd = _expand_braces(_expand_ifs(_normalize_ansi_c_quotes(command)))
|
|
|
|
def _traversal_hits_sensitive(norm):
|
|
# A relative path that climbs out of the workdir with '..' can name a host secret
|
|
# (../../../../etc/passwd). Resolve the climb and test whether the descent lands on a
|
|
# sensitive path; a plain in-tree relative path (../sibling/file.txt) is left alone.
|
|
if ".." not in norm.split("/"):
|
|
return False
|
|
try:
|
|
canon = os.path.normpath(norm)
|
|
except Exception:
|
|
canon = norm
|
|
parts = [p for p in canon.split("/") if p not in ("", ".", "..")]
|
|
return bool(parts) and _is_sensitive_abs_path("/" + "/".join(parts))
|
|
|
|
def _flag(s):
|
|
norm = s.replace("\\", "/")
|
|
try:
|
|
canon = os.path.normpath(norm)
|
|
except Exception:
|
|
canon = norm
|
|
if _is_sensitive_abs_path(norm) or _is_sensitive_abs_path(canon):
|
|
return f"{s!r} is a sensitive host identity / credential file"
|
|
if strict_traversal:
|
|
if s[:1] == "~" or ".." in norm.split("/"):
|
|
return f"{s!r} escapes the session workdir via path traversal"
|
|
elif _traversal_hits_sensitive(norm):
|
|
return f"{s!r} reads a sensitive host path via directory traversal"
|
|
return None
|
|
|
|
def _escaping_glob(tok):
|
|
if not any(g in tok for g in "*?["):
|
|
return False
|
|
tn = tok.replace("\\", "/")
|
|
return tok[:1] == "~" or tn.startswith("/")
|
|
|
|
def _check_assignment_rhs(tok):
|
|
# FOO=/etc/passwd binds a sensitive path into a variable that a later reader dereferences
|
|
# (P=/etc/passwd cat ${P}); flag the assigned value directly.
|
|
_rhs = tok.split("=", 1)[1] if "=" in tok else ""
|
|
for _pc in re.split(r"[;|&<>()`{}]+", _rhs):
|
|
if _pc and not _pc.startswith("-"):
|
|
_r = _flag(_pc)
|
|
if _r is not None:
|
|
return _r
|
|
return None
|
|
|
|
# Literal-path token scan (absolute-sensitive + traversal), splitting shell punctuation
|
|
# glued to an adjacent word (cat /etc/passwd|wc) so the path piece is still checked.
|
|
try:
|
|
toks = shlex.split(cmd, posix = True)
|
|
except ValueError:
|
|
toks = cmd.split()
|
|
for _t in toks:
|
|
for _piece in re.split(r"[;|&<>()`{}]+", _t):
|
|
if _piece and not _piece.startswith("-"):
|
|
_r = _flag(_piece)
|
|
if _r is not None:
|
|
return _r
|
|
if _ASSIGNMENT_RE.match(_t):
|
|
_r = _check_assignment_rhs(_t)
|
|
if _r is not None:
|
|
return _r
|
|
|
|
# Re-tokenize keeping redirects / separators for the input-redirect + expansion scan.
|
|
try:
|
|
_lx = shlex.shlex(cmd, posix = True, punctuation_chars = ";&|()`<>")
|
|
_lx.whitespace_split = True
|
|
ptoks = list(_lx)
|
|
except ValueError:
|
|
ptoks = cmd.split()
|
|
|
|
# find ... -exec CMD ... ; runs CMD directly on each match; CMD may be a nested shell
|
|
# (sh -c 'cat /etc/passwd') or a reader, so scan each -exec segment through this scanner
|
|
# (mirrors the blocked-command find -exec handling). The main command-word loop below only
|
|
# recurses into a shell that IS the command word, so the quoted -c payload would otherwise
|
|
# be treated as one inert argument.
|
|
for _fi, _ft in enumerate(ptoks):
|
|
if _ft in _FIND_EXEC_FLAGS:
|
|
_seg = []
|
|
_fj = _fi + 1
|
|
while _fj < len(ptoks) and ptoks[_fj] not in _FIND_EXEC_TERMINATORS:
|
|
_seg.append(ptoks[_fj])
|
|
_fj += 1
|
|
if _seg:
|
|
_r = _scan_command_string_for_reads(
|
|
shlex.join(_seg),
|
|
strict_traversal = strict_traversal,
|
|
cwd = cwd,
|
|
cwd_dynamic = cwd_dynamic,
|
|
_depth = _depth + 1,
|
|
)
|
|
if _r is not None:
|
|
return _r
|
|
# trap 'CMD' SIG: the quoted handler runs as shell code on EXIT / a signal; scan it.
|
|
if _ft == "trap" and _fi + 1 < len(ptoks):
|
|
_th = ptoks[_fi + 1]
|
|
if _th and _th != "-" and _th not in _READ_SCAN_SEPARATORS:
|
|
_r = _scan_command_string_for_reads(
|
|
_th,
|
|
strict_traversal = strict_traversal,
|
|
cwd = cwd,
|
|
cwd_dynamic = cwd_dynamic,
|
|
_depth = _depth + 1,
|
|
)
|
|
if _r is not None:
|
|
return _r
|
|
|
|
# A command substitution ($(...) / `...`) runs its payload as a shell command regardless of
|
|
# surrounding quotes, so `echo "$(head /etc/passwd)"` reads the file even though the outer
|
|
# command is not a reader and the tokenizer keeps the quoted substitution as one argument.
|
|
# Scan each substitution payload recursively, independent of the outer command word.
|
|
for _cs in _extract_command_subs(cmd):
|
|
if _cs.strip():
|
|
_r = _scan_command_string_for_reads(
|
|
_cs,
|
|
strict_traversal = strict_traversal,
|
|
cwd = cwd,
|
|
cwd_dynamic = cwd_dynamic,
|
|
_depth = _depth + 1,
|
|
)
|
|
if _r is not None:
|
|
return _r
|
|
|
|
def _risky_read_target(tgt):
|
|
if not tgt:
|
|
return False
|
|
if "$" in tgt or "`" in tgt or _escaping_glob(tgt):
|
|
return True
|
|
tn = tgt.replace("\\", "/")
|
|
if _is_sensitive_abs_path(tgt):
|
|
return True
|
|
return ".." in tn.split("/") if strict_traversal else _traversal_hits_sensitive(tn)
|
|
|
|
_at_cmd = True
|
|
_cur_reader = False
|
|
_wrapper = None
|
|
_skip_operand = False
|
|
# The child's cwd for a relative read: seeded from an ambient cwd (a subprocess cwd=), and
|
|
# overridable per-command by env -C DIR. Resets to the ambient cwd at each separator.
|
|
_chdir = cwd
|
|
_pending_chdir = False
|
|
for _pi, _pt in enumerate(ptoks):
|
|
if _pt in _READ_SCAN_SEPARATORS:
|
|
_at_cmd = True
|
|
_cur_reader = False
|
|
_wrapper = None
|
|
_skip_operand = False
|
|
_chdir = cwd
|
|
_pending_chdir = False
|
|
continue
|
|
if _pt.startswith("<"):
|
|
_rt = _pt.lstrip("<") or (ptoks[_pi + 1] if _pi + 1 < len(ptoks) else "")
|
|
if _risky_read_target(_rt):
|
|
return f"shell input redirect from a non-literal / sensitive path {_rt!r}"
|
|
continue
|
|
if _pt.startswith(">"):
|
|
continue # output redirects are handled by _find_blocked_commands
|
|
if _at_cmd:
|
|
if _skip_operand: # a wrapper flag's separated operand (env -u NAME)
|
|
if _pending_chdir: # ...but env -C DIR's operand is the child cwd
|
|
_chdir = _join_chdir(_chdir, _pt)
|
|
_pending_chdir = False
|
|
_skip_operand = False
|
|
continue
|
|
if _ASSIGNMENT_RE.match(_pt):
|
|
_r = _check_assignment_rhs(_pt)
|
|
if _r is not None:
|
|
return _r
|
|
continue # assignment prefix; the command word is still ahead
|
|
if _pt.startswith("-"):
|
|
# env -C DIR / --chdir DIR changes the child's cwd before the command runs, so
|
|
# a later relative reader arg (env -C /etc cat passwd -> /etc/passwd) resolves
|
|
# against DIR, not the workdir. Capture DIR instead of just skipping it.
|
|
if _wrapper == "env" and _pt in ("-C", "--chdir"):
|
|
_pending_chdir = True
|
|
_skip_operand = True
|
|
elif _wrapper == "env" and _pt.startswith("--chdir="):
|
|
_chdir = _join_chdir(_chdir, _pt.split("=", 1)[1])
|
|
elif _wrapper and _wrapper_flag_takes_operand(_wrapper, _pt):
|
|
_skip_operand = True
|
|
continue # wrapper flag; still before the command word
|
|
# A wrapper's numeric operand (`timeout 1 bash -c ...`, `nice 5 cat ...`) is not
|
|
# the command word; skip it so the real command (bash / cat) after it is scanned.
|
|
if _wrapper and _is_wrapper_numeric_arg(_pt):
|
|
continue
|
|
_base = os.path.basename(_pt).lower()
|
|
if _base in _COMMAND_PREFIXES:
|
|
_wrapper = _base
|
|
continue # env / sudo / nice / timeout ...; command word is still ahead
|
|
if _base in _SHELL_BINARIES:
|
|
# A nested shell runs its -c payload as fresh shell code; scan it recursively.
|
|
for _k in range(_pi + 1, len(ptoks)):
|
|
_ft = ptoks[_k]
|
|
if _ft in _READ_SCAN_SEPARATORS:
|
|
break
|
|
_fl = _ft.lower()
|
|
if _fl == "-c" or (
|
|
_fl.startswith("-") and not _fl.startswith("--") and _fl.endswith("c")
|
|
):
|
|
if _k + 1 < len(ptoks):
|
|
_r = _scan_command_string_for_reads(
|
|
ptoks[_k + 1],
|
|
strict_traversal = strict_traversal,
|
|
cwd = _chdir,
|
|
cwd_dynamic = cwd_dynamic,
|
|
_depth = _depth + 1,
|
|
)
|
|
if _r is not None:
|
|
return _r
|
|
break
|
|
_at_cmd = False
|
|
_cur_reader = False
|
|
_wrapper = None
|
|
continue
|
|
_cur_reader = _base in _SHELL_READ_COMMANDS
|
|
_at_cmd = False
|
|
_wrapper = None
|
|
continue
|
|
if _cur_reader and not _pt.startswith("-"):
|
|
if "$" in _pt or "`" in _pt or _escaping_glob(_pt):
|
|
return f"shell read command reads an expanded path {_pt!r}"
|
|
_rel = not _pt.startswith("/") and not _pt.startswith("~")
|
|
# Under a known chdir (env -C DIR or an ambient subprocess cwd=), a relative reader
|
|
# arg resolves against DIR (cat passwd + cwd=/etc -> /etc/passwd).
|
|
if _chdir and _rel:
|
|
_r = _flag(os.path.join(_chdir, _pt))
|
|
if _r is not None:
|
|
return _r
|
|
# A relative reader arg under a NON-literal cwd cannot be proven sandbox-local, so
|
|
# fail closed (subprocess.run('cat passwd', shell=True, cwd=P)).
|
|
if cwd_dynamic and _chdir is None and _rel:
|
|
return f"shell read command reads {_pt!r} under a non-literal cwd"
|
|
return None
|
|
|
|
|
|
def _command_reads_sensitive(command: str) -> str | None:
|
|
"""Scan a raw terminal-tool shell command STRING for an embedded host-secret read; return a
|
|
short reason or None. The terminal tool runs the command in an unguarded shell child (the
|
|
runtime open() backstop only wraps the Python tool), so a read of an identity / credential
|
|
file, a sensitive-target ``..`` traversal, or an escaping glob / ``$()`` / backtick
|
|
expansion on a file-reading command must be refused statically. Benign in-tree relative
|
|
navigation is left alone (strict_traversal disabled)."""
|
|
return _scan_command_string_for_reads(command, strict_traversal = False)
|
|
|
|
|
|
# Stage 4: pragmatic aliasing (single-assignment alias + inline literal container).
|
|
# Catches `s = os.system; s('rm -rf /')` and `[os.system][0](...)` feeding the
|
|
# existing shell-command denylist. Deliberately low-FP: only unambiguous single
|
|
# assignments (a name stored exactly once) and inline literal containers, never a
|
|
# flow-insensitive union (so `s = os.system; s = print; s('hi')` is NOT aliased).
|
|
# --------------------------------------------------------------------------
|
|
_SHELL_SINK_FUNCS = frozenset(
|
|
{
|
|
"os.system",
|
|
"os.popen",
|
|
"os.popen2",
|
|
"os.popen3",
|
|
"os.popen4",
|
|
"os.execl",
|
|
"os.execle",
|
|
"os.execlp",
|
|
"os.execlpe",
|
|
"os.execv",
|
|
"os.execve",
|
|
"os.execvp",
|
|
"os.execvpe",
|
|
"os.spawnl",
|
|
"os.spawnle",
|
|
"os.spawnlp",
|
|
"os.spawnlpe",
|
|
"os.spawnv",
|
|
"os.spawnve",
|
|
"os.spawnvp",
|
|
"os.spawnvpe",
|
|
"os.posix_spawn",
|
|
"os.posix_spawnp",
|
|
"subprocess.run",
|
|
"subprocess.call",
|
|
"subprocess.check_call",
|
|
"subprocess.check_output",
|
|
"subprocess.Popen",
|
|
"subprocess.getoutput",
|
|
"subprocess.getstatusoutput",
|
|
}
|
|
)
|
|
|
|
|
|
def _resolve_static_shell_sink(node, os_aliases, subprocess_aliases, from_aliases):
|
|
"""Resolve an expression to a shell-sink fully-qualified name, else None."""
|
|
if isinstance(node, ast.Attribute):
|
|
v = node.value
|
|
# os / posix / nt receiver as a simple name (os.system, posix.system) or a re-exported
|
|
# module reached as an attribute (pathlib.os.system, tempfile.os.system).
|
|
is_os = (isinstance(v, ast.Name) and v.id in os_aliases) or (
|
|
isinstance(v, ast.Attribute) and v.attr in ("os", "posix", "nt")
|
|
)
|
|
is_sp = (isinstance(v, ast.Name) and v.id in subprocess_aliases) or (
|
|
isinstance(v, ast.Attribute) and v.attr == "subprocess"
|
|
)
|
|
if is_os:
|
|
fq = f"os.{node.attr}"
|
|
if fq in _SHELL_SINK_FUNCS:
|
|
return fq
|
|
if is_sp:
|
|
fq = f"subprocess.{node.attr}"
|
|
if fq in _SHELL_SINK_FUNCS:
|
|
return fq
|
|
if isinstance(node, ast.Name):
|
|
return from_aliases.get(node.id)
|
|
return None
|
|
|
|
|
|
def _check_signal_escape_patterns(
|
|
code: str,
|
|
_depth: int = 0,
|
|
_budget = None,
|
|
):
|
|
"""Check for patterns that could escape signal-based timeouts. Returns
|
|
(safe: bool, details: dict). Vendored from unsloth_zoo.rl_environments to
|
|
avoid importing unsloth_zoo (needs GPU drivers; fails on Apple Silicon)."""
|
|
try:
|
|
tree = ast.parse(code)
|
|
except SyntaxError as e:
|
|
return False, {
|
|
"error": f"SyntaxError: {e}",
|
|
"signal_tampering": [],
|
|
"exception_catching": [],
|
|
"warnings": [],
|
|
}
|
|
|
|
# Charge this tree's node count against the shared analyzer budget BEFORE the several
|
|
# unbounded ast.walk / visitor passes below. A syntactically valid file (or a huge
|
|
# recovered exec/eval payload, since the budget is shared across the recursion) with
|
|
# hundreds of thousands of nodes would otherwise tie up the Studio parent process
|
|
# before the child rlimits apply. Fail closed (block) when the budget is exceeded.
|
|
if _budget is None:
|
|
_budget = _AnalyzerBudget()
|
|
try:
|
|
_budget.nodes += sum(1 for _ in ast.walk(tree))
|
|
except Exception: # pragma: no cover - defensive
|
|
pass
|
|
if _budget.nodes > _MAX_ANALYZER_NODES:
|
|
return False, {
|
|
"error": None,
|
|
"signal_tampering": [],
|
|
"exception_catching": [],
|
|
"shell_escapes": [],
|
|
"dynamic_exec": [
|
|
{
|
|
"type": "analyzer_budget",
|
|
"line": -1,
|
|
"description": (
|
|
"code exceeds the static-analysis node budget (too large to verify safely)"
|
|
),
|
|
}
|
|
],
|
|
"network_calls": [],
|
|
"sensitive_file_reads": [],
|
|
"filesystem_violations": [],
|
|
"warnings": [],
|
|
}
|
|
|
|
signal_tampering = []
|
|
exception_catching = []
|
|
shell_escapes = []
|
|
dynamic_exec = []
|
|
filesystem_violations = []
|
|
warnings = []
|
|
|
|
# Feature flag + shared budget for the recursive sink analyzer (Stages 2-4).
|
|
# Default on; UNSLOTH_STUDIO_SINK_ANALYZER=0 reverts to the legacy blanket
|
|
# eval/exec ban and disables filesystem-confinement + aliasing analysis.
|
|
_analyzer_on = os.environ.get("UNSLOTH_STUDIO_SINK_ANALYZER", "1") != "0"
|
|
if _analyzer_on:
|
|
try:
|
|
_const_env = _build_const_prop_env(tree)
|
|
_scope_idx = _build_scope_alias_index(tree, _const_env)
|
|
except Exception: # pragma: no cover - defensive: never crashier than legacy
|
|
logger.warning("sandbox analyzer context build failed; legacy fallback", exc_info = True)
|
|
_analyzer_on = False
|
|
_const_env = {}
|
|
_scope_idx = _ScopeAliasIndex(tree)
|
|
else:
|
|
_const_env = {}
|
|
_scope_idx = _ScopeAliasIndex(tree)
|
|
|
|
def _analyze_exec_call(node, func_id):
|
|
"""Stage 2 driver: recover + recurse a foldable payload, else dynamic policy."""
|
|
try:
|
|
# Resolve compiled-code aliases (c = compile(...)) in the CALL's scope so a
|
|
# safe alias in one function cannot shadow a dynamic exec(c) in another.
|
|
_compiled_here = _scope_idx.effective(node, "compiled")
|
|
kind, src, mode, is_bytes = _recover_exec_payload(
|
|
node, func_id, _const_env, _compiled_here
|
|
)
|
|
if kind == "NO_PAYLOAD":
|
|
return
|
|
if kind == "RECOVERED":
|
|
parsed_kind, _ = _safe_parse_inner(src, mode, _depth, _budget)
|
|
if parsed_kind == "PARSED":
|
|
inner_safe, inner_info = _check_signal_escape_patterns(src, _depth + 1, _budget)
|
|
if not inner_safe and not inner_info.get("error"):
|
|
dynamic_exec.append(
|
|
{
|
|
"type": "dynamic_exec",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": (
|
|
f"{func_id}() payload reaches unsafe operation: "
|
|
f"{_first_unsafe_reason(inner_info)}"
|
|
),
|
|
}
|
|
)
|
|
return
|
|
if parsed_kind == "BOUND_HIT":
|
|
dynamic_exec.append(
|
|
{
|
|
"type": "dynamic_exec",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": (
|
|
f"{func_id}() payload exceeds static-analysis bounds "
|
|
"(oversized / too-deeply-nested / too-many-layers)"
|
|
),
|
|
}
|
|
)
|
|
return
|
|
# SYNTAX_BAD on a fully RECOVERED literal: we hold the exact
|
|
# source and it simply is not valid Python for this sink's mode,
|
|
# so it raises SyntaxError at runtime (same mode as the static
|
|
# parse) -- harmless, not an ACE vector. Allow it; only truly
|
|
# opaque (non-recoverable) payloads fall to the dynamic policy.
|
|
#
|
|
# Exception: a *bytes* payload for an executing sink. exec()/eval()/
|
|
# compile() honor PEP 263 coding cookies (e.g. "# coding: utf-7") on
|
|
# bytes, decoding them through a codec this static pass does not
|
|
# replicate -- the UTF-8 view we parsed is SYNTAX_BAD precisely
|
|
# because the real (cookie-decoded) source is hidden. A legitimate
|
|
# exec(b"...") uses plain ASCII/UTF-8 that parses cleanly, so blocking
|
|
# the unparseable-bytes case closes the codec-smuggling vector with
|
|
# negligible false positives.
|
|
if is_bytes and func_id != "compile":
|
|
dynamic_exec.append(
|
|
{
|
|
"type": "dynamic_exec",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": (
|
|
f"{func_id}() of a bytes payload that is not valid UTF-8 "
|
|
"Python (may smuggle code via a PEP 263 coding cookie)"
|
|
),
|
|
}
|
|
)
|
|
return
|
|
payload = node.args[0] if node.args else None
|
|
if _payload_has_obfuscation_primitive(payload):
|
|
dynamic_exec.append(
|
|
{
|
|
"type": "dynamic_exec",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": (
|
|
f"{func_id}() of a runtime-decoded / fetched / assembled payload"
|
|
),
|
|
}
|
|
)
|
|
else:
|
|
# An opaque, non-recoverable payload for an executing sink (eval/exec/
|
|
# runpy) is a universal ACE bypass: it can synthesize any shell/network/
|
|
# filesystem escape at runtime, invisibly to every static check. compile()
|
|
# does not itself run, but its CODE OBJECT can be executed without exec/eval
|
|
# (fn.__code__ = compile(src, '<p>', 'exec'); fn()), so an opaque compile
|
|
# source is equally unverifiable and is blocked too. A literal source is
|
|
# analyzed recursively above; ast.literal_eval / json.loads cover data.
|
|
dynamic_exec.append(
|
|
{
|
|
"type": "dynamic_exec",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": (
|
|
f"{func_id}() of a non-literal payload cannot be statically "
|
|
"verified (use ast.literal_eval / json.loads for data)"
|
|
),
|
|
}
|
|
)
|
|
except Exception: # pragma: no cover - fail closed, never crashier than legacy
|
|
dynamic_exec.append(
|
|
{
|
|
"type": "dynamic_exec",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": f"dynamic code execution via {func_id}()",
|
|
}
|
|
)
|
|
|
|
def _ast_name_matches(node, names):
|
|
if isinstance(node, ast.Name):
|
|
return node.id in names
|
|
elif isinstance(node, ast.Attribute):
|
|
full_name = []
|
|
current = node
|
|
while isinstance(current, ast.Attribute):
|
|
full_name.append(current.attr)
|
|
current = current.value
|
|
if isinstance(current, ast.Name):
|
|
full_name.append(current.id)
|
|
full_name = ".".join(reversed(full_name))
|
|
return full_name in names
|
|
return False
|
|
|
|
# Dangerous os/subprocess functions that can execute shell commands
|
|
# (defined at module scope as _SHELL_SINK_FUNCS so Stage 4 alias resolution
|
|
# can share it).
|
|
_SHELL_EXEC_FUNCS = _SHELL_SINK_FUNCS
|
|
|
|
# Dynamic-execution / obfuscation primitives that defeat the static (name-based) checks
|
|
# above: they build or reach a dangerous callable at runtime, so a bare name match cannot
|
|
# see the payload. eval/exec/compile are direct code-execution builtins; __import__ /
|
|
# importlib load a module by (possibly computed) name; getattr/setattr on a sensitive
|
|
# module implement `getattr(os, 'sys'+'tem')(...)`.
|
|
_DYNAMIC_EXEC_BUILTINS = frozenset({"eval", "exec", "compile"})
|
|
_DYNAMIC_IMPORT_FUNCS = frozenset(
|
|
{"importlib.import_module", "importlib.reload", "importlib.__import__"}
|
|
)
|
|
# Dynamic import is a real workflow (e.g. importing huggingface_hub), so it is flagged only
|
|
# when the target is computed (non-literal name = obfuscation) or names a module that can
|
|
# reach code execution / shell / builtins. A benign literal (json, numpy, huggingface_hub)
|
|
# passes; the HF upload gate below still validates its call args separately.
|
|
_DANGEROUS_IMPORT_NAMES = frozenset(
|
|
{
|
|
"os",
|
|
"posix", # the C module os wraps; __import__('posix').system(...) == os.system
|
|
"nt", # Windows analogue of posix
|
|
"subprocess",
|
|
"sys",
|
|
"builtins",
|
|
"importlib",
|
|
"ctypes",
|
|
"pty",
|
|
"socket",
|
|
"signal",
|
|
"resource",
|
|
"shutil",
|
|
"multiprocessing",
|
|
"runpy",
|
|
"code",
|
|
"codeop",
|
|
"pdb",
|
|
"mmap",
|
|
"fcntl",
|
|
}
|
|
)
|
|
# Attribute-name obfuscation via getattr/setattr is only flagged when aimed at a module
|
|
# that can execute code or reach builtins (keeps ordinary getattr(obj, "field") benign).
|
|
_DYNAMIC_ATTR_TARGETS = frozenset({"os", "subprocess", "sys", "builtins", "importlib"})
|
|
# Introspection "gadget" dunders used to walk from a harmless object to os/builtins
|
|
# (``().__class__.__bases__[0].__subclasses__()``). ``__class__`` / ``__dict__`` are
|
|
# intentionally excluded (too common); the chain still trips on the others.
|
|
# __mro__ and __code__ are deliberately EXCLUDED here: on their own they do not reach
|
|
# an execution primitive, and they are read by ordinary ML/debugging code
|
|
# (trainer_class.__mro__, fn.__code__), so flagging them over-blocks legitimate
|
|
# snippets. The terminal escape primitives below still trip on the real gadget
|
|
# chains (().__class__.__bases__[0].__subclasses__(), f.__globals__['os']). A
|
|
# SUBSCRIPTED __mro__ (cls.__mro__[1], the base-class extraction shape) is flagged
|
|
# separately in visit_Subscript so plain iteration stays allowed.
|
|
# cell_contents is the ONLY way to read a closure cell's value, so it is the terminal
|
|
# step of recovering a guarded wrapper's original callable via __closure__; flagging
|
|
# it closes that recovery even when the __closure__ name was built dynamically.
|
|
_GADGET_DUNDERS = frozenset(
|
|
{
|
|
"__subclasses__",
|
|
"__bases__",
|
|
"__base__",
|
|
"__globals__",
|
|
"__builtins__",
|
|
"__closure__",
|
|
"cell_contents",
|
|
# Frame / traceback introspection recovers a runtime-guard wrapper's ORIGINAL
|
|
# unguarded callable: it is a free variable (`real`) in the wrapper's frame, so a
|
|
# snippet that triggers a denied open() can read it back via a trace hook or the
|
|
# caught exception's traceback (frame.f_locals['real'], tb.tb_frame.f_locals) and
|
|
# call it directly, escaping the filesystem boundary. __closure__ / cell_contents
|
|
# are already blocked, so the frame path is the remaining channel; close it by
|
|
# flagging frame acquisition (settrace / _getframe / currentframe / tb_frame) and
|
|
# the f_locals / f_globals value read for ANY receiver.
|
|
"f_locals",
|
|
"f_globals",
|
|
"f_back",
|
|
"f_builtins",
|
|
"tb_frame",
|
|
"tb_next",
|
|
"gi_frame",
|
|
"cr_frame",
|
|
"ag_frame",
|
|
"settrace",
|
|
"setprofile",
|
|
"_getframe",
|
|
"_current_frames",
|
|
"currentframe",
|
|
}
|
|
)
|
|
|
|
def _extract_string_from_node(node):
|
|
"""Extract a plain string value from an AST node, if it is a constant."""
|
|
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
|
return node.value
|
|
return None
|
|
|
|
def _extract_strings_from_list(node):
|
|
"""Extract string elements from an AST List or Tuple node."""
|
|
if isinstance(node, (ast.List, ast.Tuple)):
|
|
parts = []
|
|
for elt in node.elts:
|
|
s = _extract_string_from_node(elt)
|
|
if s is not None:
|
|
parts.append(s)
|
|
return parts
|
|
return []
|
|
|
|
# Kwarg names that carry command content (not control flags like
|
|
# check=True, text=True, capture_output=True).
|
|
_CMD_KWARGS = frozenset({"args", "command", "executable", "path", "file"})
|
|
|
|
def _check_shell_argv(elts):
|
|
"""Analyze a subprocess argv VECTOR (['bash', '-c', '...'], ['sh', 's.sh']) as a
|
|
whole. A shell argv is only safe when it carries an inline -c payload that scans
|
|
clean; a script-file / -s / bare-shell / dynamic-payload form runs unscanned code
|
|
and is denied. Returns a set of blocked markers (empty if not a shell argv or the
|
|
scanned -c payload is benign)."""
|
|
if not elts:
|
|
return set()
|
|
first = _extract_string_from_node(elts[0])
|
|
if first is None or os.path.basename(first).lower() not in _SHELL_BINARIES:
|
|
return set()
|
|
found = set()
|
|
i = 1
|
|
while i < len(elts):
|
|
f = _extract_string_from_node(elts[i])
|
|
if f is not None and (
|
|
f == "-c" or (f.startswith("-") and not f.startswith("--") and f.endswith("c"))
|
|
):
|
|
if i + 1 < len(elts):
|
|
payload = _extract_string_from_node(elts[i + 1])
|
|
if payload is None:
|
|
found.add("shell-dynamic-c") # unanalyzable inline payload
|
|
else:
|
|
found |= _find_blocked_commands(payload)
|
|
else:
|
|
found.add("shell-script:" + first) # -c with no payload
|
|
return found
|
|
i += 1
|
|
# No -c: a script file, -s (stdin), or a bare shell that reads stdin.
|
|
found.add("shell-script:" + first)
|
|
return found
|
|
|
|
def _check_args_for_blocked(args_nodes, shell_maybe_true = False):
|
|
"""Check if any call arguments contain blocked commands."""
|
|
found = set()
|
|
for arg in args_nodes:
|
|
s = _extract_string_from_node(arg)
|
|
if s is not None:
|
|
found |= _find_blocked_commands(s)
|
|
continue
|
|
if isinstance(arg, (ast.List, ast.Tuple)):
|
|
str_elts = [_extract_string_from_node(e) for e in arg.elts]
|
|
first = str_elts[0] if str_elts else None
|
|
# With shell=True, POSIX subprocess passes the FIRST sequence element to
|
|
# /bin/sh -c as the command string (the rest become $0, $1, ...); so
|
|
# subprocess.run(['echo x > /tmp/p'], shell=True) runs a full shell command,
|
|
# not an argv vector. Scan elts[0] with the shell parser in that case.
|
|
if shell_maybe_true and first is not None:
|
|
found |= _find_blocked_commands(first)
|
|
# A shell argv vector is analyzed as a whole so `['bash', '-c', 'echo hi']`
|
|
# scans the payload instead of tripping the bare-shell block on the 'bash'
|
|
# element. A non-shell argv only executes its command word (argv[0] plus any
|
|
# wrapper prefix), so scan just that -- scanning every element would misread a
|
|
# benign argument (subprocess.run(['echo', 'python'])) as a blocked command.
|
|
elif first is not None and os.path.basename(first).lower() in _SHELL_BINARIES:
|
|
found |= _check_shell_argv(arg.elts)
|
|
else:
|
|
_argv_blocked, _cmd_idx = _blocked_in_argv(str_elts)
|
|
found |= _argv_blocked
|
|
if _cmd_idx is not None and str_elts[_cmd_idx] is not None:
|
|
_cmd_base = os.path.basename(str_elts[_cmd_idx]).lower()
|
|
# A wrapper-hidden shell binary (env bash s.sh, nice sh -c '...')
|
|
# resolves to a shell as its command word; analyze the shell + its args
|
|
# (script file or -c payload) so the bare-shell / unscanned-script forms
|
|
# are caught.
|
|
if _cmd_base in _SHELL_BINARIES:
|
|
found |= _check_shell_argv(arg.elts[_cmd_idx:])
|
|
# find -exec/-delete, sed -i, sort -o interpret LATER argv elements as
|
|
# actions / write flags, so reconstruct a command line from the argv
|
|
# tail and reuse the full scanner (which handles those forms).
|
|
elif _cmd_base in _ARGV_TAIL_SCAN_COMMANDS:
|
|
found |= _find_blocked_commands(
|
|
" ".join(
|
|
shlex.quote(s) for s in str_elts[_cmd_idx:] if s is not None
|
|
)
|
|
)
|
|
continue
|
|
for s in _extract_strings_from_list(arg):
|
|
found |= _find_blocked_commands(s)
|
|
return found
|
|
|
|
class SignalEscapeVisitor(ast.NodeVisitor):
|
|
def __init__(self):
|
|
self.imports_signal = False
|
|
self.signal_aliases = {"signal"}
|
|
self.os_aliases = {"os"}
|
|
self.subprocess_aliases = {"subprocess"}
|
|
self.importlib_aliases = {"importlib"}
|
|
self.sys_aliases = {"sys"}
|
|
# __builtins__ is the builtins *module* in __main__ (how the sandbox runs
|
|
# user code as `python <file>.py`), so builtins.eval / __builtins__.eval work.
|
|
self.builtins_aliases = {"builtins", "__builtins__"}
|
|
# Bare name -> fully-qualified form for from-import tracking
|
|
# (e.g. "system" -> "os.system").
|
|
self.shell_exec_aliases: dict[str, str] = {}
|
|
# from importlib import import_module as im -> {"im"}
|
|
self.import_func_aliases: set[str] = set()
|
|
# from builtins import exec as e -> {"e": "exec"}
|
|
self.exec_from_aliases: dict[str, str] = {}
|
|
# import pickle as p -> {"p": "pickle"}; from pickle import loads as l -> {"l": "pickle.loads"}
|
|
self.deserialize_module_aliases: dict[str, str] = {}
|
|
self.deserialize_aliases: dict[str, str] = {}
|
|
# `from yaml import load [as X]` / load_all: yaml.load is conditional (safe only
|
|
# with a SafeLoader) so it is not a static sink; track the bare-name alias so the
|
|
# safe-loader check can be applied to the direct-call form.
|
|
self.yaml_load_aliases: dict[str, str] = {}
|
|
# `from pickle import Unpickler [as X]`: Unpickler(f).load() reaches the same reduce
|
|
# path as pickle.load; track the ctor alias so the .load() method call is flagged.
|
|
self.unpickler_aliases: set[str] = set()
|
|
# import types as t -> {"types", "t"}; from types import FunctionType as F -> {"F"}.
|
|
# FunctionType(code, globals)() runs a code object WITHOUT eval/exec, so a
|
|
# dynamic compile() result reaches execution through it (see visit_Call).
|
|
self.types_aliases = {"types"}
|
|
self.functiontype_aliases: set[str] = set()
|
|
# import runpy as r -> {"runpy", "r"}. runpy.run_path/run_module execute a
|
|
# file/module in the guarded interpreter without the recursive source
|
|
# analysis exec/eval receive, so treat those calls as execution sinks.
|
|
self.runpy_aliases = {"runpy"}
|
|
# from runpy import run_path as X / run_module as Y -> {"X", "Y"}.
|
|
self.runpy_func_aliases: set[str] = set()
|
|
# import inspect as i -> {"inspect", "i"}. inspect.getclosurevars(fn) hands back
|
|
# the cells a guard wrapper closes over (the original unguarded callable), so
|
|
# treat it as a closure-recovery gadget like __closure__ / cell_contents.
|
|
self.inspect_aliases = {"inspect"}
|
|
# from inspect import getclosurevars as g -> {"g"}.
|
|
self.getclosurevars_aliases: set[str] = set()
|
|
# import operator as op -> {"operator", "op"}. operator.attrgetter('name')(obj)
|
|
# is the same attribute-fetch obfuscation as getattr(obj, 'name').
|
|
self.operator_aliases = {"operator"}
|
|
# from operator import attrgetter as ag -> {"ag"}.
|
|
self.attrgetter_aliases: set[str] = set()
|
|
# from operator import methodcaller as mc -> {"mc"}. methodcaller('__getattribute__',
|
|
# 'system')(os) fetches os.system, the same obfuscation as attrgetter.
|
|
self.methodcaller_aliases: set[str] = set()
|
|
# import gc as g -> {"gc", "g"}. gc.get_referents / get_referrers / get_objects
|
|
# walk the object graph to a guard wrapper's closure cell (the original unguarded
|
|
# callable) without spelling __closure__, so treat them as recovery gadgets.
|
|
self.gc_aliases = {"gc"}
|
|
# from gc import get_referents as gr -> {"gr"}.
|
|
self.gc_walk_aliases: set[str] = set()
|
|
# import pty as p -> {"pty", "p"}. pty.spawn([...]) / pty.fork() run an unguarded
|
|
# child process (a shell) outside the sandbox.
|
|
self.pty_aliases: set[str] = set()
|
|
# from pty import spawn as s -> {"s"}: bare-name aliases of the pty child sinks.
|
|
self.pty_func_aliases: set[str] = set()
|
|
self.loop_depth = 0
|
|
|
|
def visit_Import(self, node):
|
|
for alias in node.names:
|
|
if alias.name == "signal":
|
|
self.imports_signal = True
|
|
if alias.asname:
|
|
self.signal_aliases.add(alias.asname)
|
|
elif alias.name == "os":
|
|
self.os_aliases.add(alias.asname or "os")
|
|
elif alias.name in ("posix", "nt"):
|
|
# posix / nt are the C backend os wraps: posix.system(...) == os.system,
|
|
# and posix.exec*/spawn*/popen mirror os. Model them as os aliases so a
|
|
# direct `import posix; posix.system('...')` resolves to an os shell sink.
|
|
self.os_aliases.add(alias.asname or alias.name)
|
|
elif alias.name == "subprocess":
|
|
self.subprocess_aliases.add(alias.asname or "subprocess")
|
|
elif alias.name == "pty":
|
|
# pty.spawn([...]) / pty.fork() run an unguarded child process.
|
|
self.pty_aliases.add(alias.asname or "pty")
|
|
elif alias.name == "importlib":
|
|
self.importlib_aliases.add(alias.asname or "importlib")
|
|
elif alias.name == "sys":
|
|
self.sys_aliases.add(alias.asname or "sys")
|
|
elif alias.name == "builtins":
|
|
self.builtins_aliases.add(alias.asname or "builtins")
|
|
elif alias.name == "types":
|
|
self.types_aliases.add(alias.asname or "types")
|
|
elif alias.name == "runpy":
|
|
self.runpy_aliases.add(alias.asname or "runpy")
|
|
elif alias.name == "inspect":
|
|
self.inspect_aliases.add(alias.asname or "inspect")
|
|
elif alias.name == "operator":
|
|
self.operator_aliases.add(alias.asname or "operator")
|
|
elif alias.name == "gc":
|
|
self.gc_aliases.add(alias.asname or "gc")
|
|
if alias.name in _DESERIALIZE_MODULES:
|
|
self.deserialize_module_aliases[alias.asname or alias.name] = alias.name
|
|
self.generic_visit(node)
|
|
|
|
def visit_ImportFrom(self, node):
|
|
if node.module == "signal":
|
|
self.imports_signal = True
|
|
for alias in node.names:
|
|
if alias.name in (
|
|
"signal",
|
|
"SIGALRM",
|
|
"SIG_IGN",
|
|
"setitimer",
|
|
"ITIMER_REAL",
|
|
"pthread_sigmask",
|
|
"SIG_BLOCK",
|
|
"alarm",
|
|
):
|
|
self.signal_aliases.add(alias.asname or alias.name)
|
|
elif node.module in ("os", "subprocess", "posix", "nt"):
|
|
if node.module == "subprocess":
|
|
self.subprocess_aliases.add("subprocess")
|
|
_eff_mod = "subprocess"
|
|
else:
|
|
# posix / nt are the C backend os wraps: `from posix import system` is
|
|
# os.system, so model the sink under os so it is caught the same way.
|
|
self.os_aliases.add("os")
|
|
_eff_mod = "os"
|
|
# Track from-imports of dangerous functions.
|
|
for alias in node.names:
|
|
fq = f"{_eff_mod}.{alias.name}"
|
|
if fq in _SHELL_EXEC_FUNCS:
|
|
self.shell_exec_aliases[alias.asname or alias.name] = fq
|
|
elif node.module == "importlib":
|
|
for alias in node.names:
|
|
if alias.name in ("import_module", "reload", "__import__"):
|
|
self.import_func_aliases.add(alias.asname or alias.name)
|
|
elif node.module == "builtins":
|
|
for alias in node.names:
|
|
if alias.name in _DYNAMIC_EXEC_BUILTINS:
|
|
self.exec_from_aliases[alias.asname or alias.name] = alias.name
|
|
elif alias.name == "__import__":
|
|
# `from builtins import __import__ as imp; imp('os').system(...)`
|
|
# is a dynamic import exactly like a bare __import__ call.
|
|
self.import_func_aliases.add(alias.asname or alias.name)
|
|
elif node.module in _DESERIALIZE_MODULES:
|
|
for alias in node.names:
|
|
fq = f"{node.module}.{alias.name}"
|
|
if fq in _CODE_DESERIALIZE_SINKS:
|
|
self.deserialize_aliases[alias.asname or alias.name] = fq
|
|
elif node.module == "yaml" and alias.name in _YAML_LOAD_METHODS:
|
|
self.yaml_load_aliases[alias.asname or alias.name] = alias.name
|
|
elif alias.name == "Unpickler" and node.module in _UNPICKLER_MODULES:
|
|
self.unpickler_aliases.add(alias.asname or alias.name)
|
|
elif node.module == "types":
|
|
for alias in node.names:
|
|
if alias.name == "FunctionType":
|
|
self.functiontype_aliases.add(alias.asname or alias.name)
|
|
elif node.module == "runpy":
|
|
for alias in node.names:
|
|
if alias.name in ("run_path", "run_module"):
|
|
self.runpy_func_aliases.add(alias.asname or alias.name)
|
|
elif node.module == "pty":
|
|
# from pty import spawn / fork: bare-name aliases of the pty child sinks.
|
|
for alias in node.names:
|
|
if alias.name in ("spawn", "fork"):
|
|
self.pty_func_aliases.add(alias.asname or alias.name)
|
|
elif node.module == "inspect":
|
|
for alias in node.names:
|
|
if alias.name == "getclosurevars":
|
|
self.getclosurevars_aliases.add(alias.asname or alias.name)
|
|
elif node.module == "operator":
|
|
for alias in node.names:
|
|
if alias.name == "attrgetter":
|
|
self.attrgetter_aliases.add(alias.asname or alias.name)
|
|
elif alias.name == "methodcaller":
|
|
self.methodcaller_aliases.add(alias.asname or alias.name)
|
|
elif node.module == "gc":
|
|
for alias in node.names:
|
|
if alias.name in ("get_referents", "get_referrers", "get_objects"):
|
|
self.gc_walk_aliases.add(alias.asname or alias.name)
|
|
self.generic_visit(node)
|
|
|
|
def visit_While(self, node):
|
|
self.loop_depth += 1
|
|
self.generic_visit(node)
|
|
self.loop_depth -= 1
|
|
|
|
def visit_For(self, node):
|
|
self.loop_depth += 1
|
|
self.generic_visit(node)
|
|
self.loop_depth -= 1
|
|
|
|
def _resolve_container_sink(self, sub):
|
|
"""Resolve an inline literal-container index callee to a shell sink fq.
|
|
|
|
Covers ``[os.system][0]``, ``(os.system,)[0]`` and ``{'k': os.system}['k']``.
|
|
"""
|
|
|
|
def _elt(elt):
|
|
fq = _resolve_static_shell_sink(
|
|
elt, self.os_aliases, self.subprocess_aliases, self.shell_exec_aliases
|
|
)
|
|
if fq:
|
|
return fq
|
|
if isinstance(elt, ast.Name):
|
|
return _scope_idx.resolve(elt.id, elt, "shell")
|
|
return None
|
|
|
|
container = sub.value
|
|
ci = _const_fold(sub.slice, _const_env)
|
|
if isinstance(container, (ast.List, ast.Tuple)) and isinstance(ci, int):
|
|
if -len(container.elts) <= ci < len(container.elts):
|
|
return _elt(container.elts[ci])
|
|
if isinstance(container, ast.Dict) and ci is not None:
|
|
for k, v in zip(container.keys, container.values):
|
|
if k is not None and _const_fold(k, _const_env) == ci:
|
|
return _elt(v)
|
|
return None
|
|
|
|
def _resolve_container_exec(self, sub):
|
|
"""Resolve an inline literal-container index callee to a dynamic-exec builtin.
|
|
|
|
Covers ({'e': exec}['e'])(...), [exec][0](...), (eval,)[0](...): an inline
|
|
container hiding an eval/exec/compile sink from the bare-name recursion."""
|
|
|
|
def _elt(elt):
|
|
if isinstance(elt, ast.Name):
|
|
if elt.id in _DYNAMIC_EXEC_BUILTINS:
|
|
return elt.id
|
|
if elt.id in self.exec_from_aliases:
|
|
return self.exec_from_aliases[elt.id]
|
|
if _analyzer_on:
|
|
return _scope_idx.resolve(elt.id, elt, "execb")
|
|
if (
|
|
isinstance(elt, ast.Attribute)
|
|
and elt.attr in _DYNAMIC_EXEC_BUILTINS
|
|
and _ast_name_matches(elt.value, self.builtins_aliases)
|
|
):
|
|
return elt.attr
|
|
return None
|
|
|
|
container = sub.value
|
|
ci = _const_fold(sub.slice, _const_env)
|
|
if isinstance(container, (ast.List, ast.Tuple)) and isinstance(ci, int):
|
|
if -len(container.elts) <= ci < len(container.elts):
|
|
return _elt(container.elts[ci])
|
|
if isinstance(container, ast.Dict) and ci is not None:
|
|
for k, v in zip(container.keys, container.values):
|
|
if k is not None and _const_fold(k, _const_env) == ci:
|
|
return _elt(v)
|
|
return None
|
|
|
|
def _resolve_container_deser(self, sub):
|
|
"""Resolve an inline literal-container index callee to a deserializer sink fq.
|
|
|
|
Covers ([pickle.loads][0])(payload), (pickle.loads,)[0](...) and
|
|
{'k': pickle.loads}['k'](...): an inline container hiding a pickle/marshal
|
|
reduce sink from the attribute/name deserializer checks."""
|
|
|
|
def _elt(elt):
|
|
if isinstance(elt, ast.Attribute):
|
|
if isinstance(elt.value, ast.Name):
|
|
canon = self.deserialize_module_aliases.get(elt.value.id)
|
|
if canon is not None:
|
|
cand = f"{canon}.{elt.attr}"
|
|
if cand in _CODE_DESERIALIZE_SINKS:
|
|
return cand
|
|
fq = _fq_attr_name(elt)
|
|
if fq in _CODE_DESERIALIZE_SINKS:
|
|
return fq
|
|
elif isinstance(elt, ast.Name):
|
|
fq = self.deserialize_aliases.get(elt.id)
|
|
if fq is not None:
|
|
return fq
|
|
if _analyzer_on:
|
|
return _scope_idx.resolve(elt.id, elt, "deser")
|
|
return None
|
|
|
|
container = sub.value
|
|
ci = _const_fold(sub.slice, _const_env)
|
|
if isinstance(container, (ast.List, ast.Tuple)) and isinstance(ci, int):
|
|
if -len(container.elts) <= ci < len(container.elts):
|
|
return _elt(container.elts[ci])
|
|
if isinstance(container, ast.Dict) and ci is not None:
|
|
for k, v in zip(container.keys, container.values):
|
|
if k is not None and _const_fold(k, _const_env) == ci:
|
|
return _elt(v)
|
|
return None
|
|
|
|
def _is_unpickler_ctor(self, recv):
|
|
"""True when ``recv`` constructs a pickle/dill Unpickler instance.
|
|
|
|
Covers pickle.Unpickler(f) (incl. an aliased module: import pickle as p;
|
|
p.Unpickler(f)) and a from-imported ctor (from pickle import Unpickler;
|
|
Unpickler(f)); its .load() runs the same reduce payload as pickle.load."""
|
|
if not isinstance(recv, ast.Call):
|
|
return False
|
|
cf = recv.func
|
|
if isinstance(cf, ast.Attribute) and cf.attr == "Unpickler":
|
|
if isinstance(cf.value, ast.Name):
|
|
return self.deserialize_module_aliases.get(cf.value.id) in _UNPICKLER_MODULES
|
|
return _fq_attr_name(cf) in {m + ".Unpickler" for m in _UNPICKLER_MODULES}
|
|
if isinstance(cf, ast.Name):
|
|
return cf.id in self.unpickler_aliases
|
|
return False
|
|
|
|
def _attrgetter_name(self, n):
|
|
"""Return the single attribute name for an ``operator.attrgetter('name')``
|
|
call (or a ``from operator import attrgetter`` alias), else None. A dotted or
|
|
multi-attr getter (attrgetter('a.b'), attrgetter('a', 'b')) returns None."""
|
|
if not isinstance(n, ast.Call) or len(n.args) != 1 or n.keywords:
|
|
return None
|
|
af = n.func
|
|
is_attrgetter = (
|
|
isinstance(af, ast.Attribute)
|
|
and af.attr == "attrgetter"
|
|
and _ast_name_matches(af.value, self.operator_aliases)
|
|
) or (isinstance(af, ast.Name) and af.id in self.attrgetter_aliases)
|
|
if not is_attrgetter:
|
|
return None
|
|
name = _const_fold(n.args[0], _const_env)
|
|
if isinstance(name, str) and "." not in name:
|
|
return name
|
|
return None
|
|
|
|
def _methodcaller_getattr_name(self, n):
|
|
"""Return the attribute name for an ``operator.methodcaller('__getattribute__',
|
|
'name')`` / ``__getattr__`` call (or a from-import alias), else None. This form
|
|
fetches ``obj.name`` exactly like attrgetter, so it needs the same normalization."""
|
|
if not isinstance(n, ast.Call) or len(n.args) != 2 or n.keywords:
|
|
return None
|
|
af = n.func
|
|
is_mc = (
|
|
isinstance(af, ast.Attribute)
|
|
and af.attr == "methodcaller"
|
|
and _ast_name_matches(af.value, self.operator_aliases)
|
|
) or (isinstance(af, ast.Name) and af.id in self.methodcaller_aliases)
|
|
if not is_mc:
|
|
return None
|
|
meth = _const_fold(n.args[0], _const_env)
|
|
if meth in ("__getattribute__", "__getattr__"):
|
|
name = _const_fold(n.args[1], _const_env)
|
|
if isinstance(name, str) and "." not in name:
|
|
return name
|
|
return None
|
|
|
|
def _methodcaller_module_call(self, node):
|
|
"""Rewrite ``operator.methodcaller('meth', *args)(receiver)`` into the equivalent
|
|
``receiver.meth(*args)`` Call when the receiver is an os/subprocess module
|
|
reference, so a methodcaller-hidden sink -- methodcaller('system', 'rm -rf /')(os)
|
|
-- is analyzed exactly like the direct os.system('rm -rf /') call. Returns the
|
|
synthetic Call node (with the original location) or None when the pattern does not
|
|
apply. Only os/subprocess receivers are rewritten; a methodcaller aimed at some
|
|
other object is left untouched so benign method calls are not misread."""
|
|
if len(node.args) != 1 or node.keywords:
|
|
return None
|
|
mc = node.func
|
|
while isinstance(mc, ast.Attribute) and mc.attr == "__call__":
|
|
mc = mc.value
|
|
if not isinstance(mc, ast.Call) or not mc.args:
|
|
return None
|
|
mf = mc.func
|
|
is_mc = (
|
|
isinstance(mf, ast.Attribute)
|
|
and mf.attr == "methodcaller"
|
|
and _ast_name_matches(mf.value, self.operator_aliases)
|
|
) or (isinstance(mf, ast.Name) and mf.id in self.methodcaller_aliases)
|
|
if not is_mc:
|
|
return None
|
|
meth = _const_fold(mc.args[0], _const_env)
|
|
if not isinstance(meth, str) or not meth.isidentifier():
|
|
return None
|
|
receiver = node.args[0]
|
|
if not (
|
|
isinstance(receiver, ast.Name)
|
|
and (receiver.id in self.os_aliases or receiver.id in self.subprocess_aliases)
|
|
):
|
|
return None
|
|
synth = ast.Call(
|
|
func = ast.Attribute(value = receiver, attr = meth, ctx = ast.Load()),
|
|
args = list(mc.args[1:]),
|
|
keywords = list(mc.keywords),
|
|
)
|
|
ast.copy_location(synth, node)
|
|
ast.fix_missing_locations(synth)
|
|
return synth
|
|
|
|
def _sink_ref_desc(self, n):
|
|
"""Describe ``n`` when it is a bare reference to a dangerous callable used as a
|
|
first-class VALUE (map/reduce/partial argument): a dynamic-exec builtin, a shell
|
|
sink (os.system / subprocess.*), a dynamic-import function, or a code
|
|
deserializer. Returns a short description or None. The payloads such a sink runs
|
|
never reach the recursive analyzer, so passing one by reference is unsafe."""
|
|
if isinstance(n, ast.Subscript):
|
|
# An inline literal-container index hides the sink from the name/attribute
|
|
# checks: map([eval][0], [...]) / partial({'e': exec}['e'], ...). Resolve the
|
|
# element node and describe it, the same unwrap direct calls already apply.
|
|
container = n.value
|
|
ci = _const_fold(n.slice, _const_env)
|
|
elt = None
|
|
if isinstance(container, (ast.List, ast.Tuple)) and isinstance(ci, int):
|
|
if -len(container.elts) <= ci < len(container.elts):
|
|
elt = container.elts[ci]
|
|
elif isinstance(container, ast.Dict) and ci is not None:
|
|
for _k, _v in zip(container.keys, container.values):
|
|
if _k is not None and _const_fold(_k, _const_env) == ci:
|
|
elt = _v
|
|
break
|
|
return self._sink_ref_desc(elt) if elt is not None else None
|
|
if isinstance(n, ast.Name):
|
|
if n.id in _DYNAMIC_EXEC_BUILTINS:
|
|
return f"{n.id} (dynamic exec)"
|
|
if n.id in self.exec_from_aliases:
|
|
return f"{self.exec_from_aliases[n.id]} (dynamic exec)"
|
|
if n.id in ("__import__", "import_module") or n.id in self.import_func_aliases:
|
|
return "dynamic import"
|
|
_sh = self.shell_exec_aliases.get(n.id)
|
|
if _sh in _SHELL_EXEC_FUNCS:
|
|
return f"{_sh} (shell)"
|
|
_ds = self.deserialize_aliases.get(n.id)
|
|
if _ds is not None:
|
|
return f"{_ds} (deserialize)"
|
|
if _analyzer_on:
|
|
_r = _scope_idx.resolve(n.id, n, "shell")
|
|
if _r in _SHELL_EXEC_FUNCS:
|
|
return f"{_r} (shell)"
|
|
if _scope_idx.resolve(n.id, n, "execb") in _DYNAMIC_EXEC_BUILTINS:
|
|
return f"{_scope_idx.resolve(n.id, n, 'execb')} (dynamic exec)"
|
|
_rd = _scope_idx.resolve(n.id, n, "deser")
|
|
if _rd:
|
|
return f"{_rd} (deserialize)"
|
|
return None
|
|
if isinstance(n, ast.Attribute):
|
|
if n.attr in _DYNAMIC_EXEC_BUILTINS and _ast_name_matches(
|
|
n.value, self.builtins_aliases
|
|
):
|
|
return f"{n.attr} (dynamic exec)"
|
|
_sh = _resolve_static_shell_sink(
|
|
n, self.os_aliases, self.subprocess_aliases, self.shell_exec_aliases
|
|
)
|
|
if _sh in _SHELL_EXEC_FUNCS:
|
|
return f"{_sh} (shell)"
|
|
if isinstance(n.value, ast.Name):
|
|
_c = self.deserialize_module_aliases.get(n.value.id)
|
|
if _c is not None and f"{_c}.{n.attr}" in _CODE_DESERIALIZE_SINKS:
|
|
return f"{_c}.{n.attr} (deserialize)"
|
|
_fq = _fq_attr_name(n)
|
|
if _fq in _CODE_DESERIALIZE_SINKS:
|
|
return f"{_fq} (deserialize)"
|
|
if _fq in _SHELL_EXEC_FUNCS:
|
|
return f"{_fq} (shell)"
|
|
return None
|
|
return None
|
|
|
|
def _rhs_module_attr(self, func, attrs, mod_aliases):
|
|
"""A single-assignment alias (x = mod.attr; x(...)) resolving to an attribute in
|
|
``attrs`` on a module in ``mod_aliases``. Returns the attribute name or None, so a
|
|
re-bound execution sink (r = runpy.run_path, s = pty.spawn) is caught the same as
|
|
the direct call."""
|
|
if not (_analyzer_on and isinstance(func, ast.Name)):
|
|
return None
|
|
rhs = _scope_idx.resolve(func.id, func, "rhsnode")
|
|
if (
|
|
isinstance(rhs, ast.Attribute)
|
|
and rhs.attr in attrs
|
|
and _ast_name_matches(rhs.value, mod_aliases)
|
|
):
|
|
return rhs.attr
|
|
return None
|
|
|
|
def _is_sys_modules_expr(self, n):
|
|
# `sys.modules` (attribute form) or getattr(sys, 'modules') (the getattr-
|
|
# obfuscated form) -- both denote the loader table itself.
|
|
if (
|
|
isinstance(n, ast.Attribute)
|
|
and n.attr == "modules"
|
|
and _ast_name_matches(n.value, self.sys_aliases)
|
|
):
|
|
return True
|
|
if (
|
|
isinstance(n, ast.Call)
|
|
and isinstance(n.func, ast.Name)
|
|
and n.func.id == "getattr"
|
|
and len(n.args) >= 2
|
|
and _ast_name_matches(n.args[0], self.sys_aliases)
|
|
and _extract_string_from_node(n.args[1]) == "modules"
|
|
):
|
|
return True
|
|
# object.__getattribute__(sys, 'modules') / type(sys).__getattribute__(sys,
|
|
# 'modules'): the unbound dunder accessor reaches the loader table exactly like
|
|
# getattr, so treat it the same.
|
|
if (
|
|
isinstance(n, ast.Call)
|
|
and isinstance(n.func, ast.Attribute)
|
|
and n.func.attr in ("__getattribute__", "__getattr__")
|
|
and len(n.args) >= 2
|
|
and _ast_name_matches(n.args[0], self.sys_aliases)
|
|
and _extract_string_from_node(n.args[1]) == "modules"
|
|
):
|
|
return True
|
|
return False
|
|
|
|
def _is_sys_modules(self, n):
|
|
# `sys.modules` / getattr(sys, 'modules'), or a single-assignment alias of
|
|
# either (m = sys.modules; m.pop('_io')). Used by the loader-table mutation
|
|
# checks (sys.modules.pop('posix'); import posix drops the guard-patched module).
|
|
if self._is_sys_modules_expr(n):
|
|
return True
|
|
if _analyzer_on and isinstance(n, ast.Name):
|
|
rhs = _scope_idx.resolve(n.id, n, "rhsnode")
|
|
if rhs is not None and self._is_sys_modules_expr(rhs):
|
|
return True
|
|
return False
|
|
|
|
def _is_namespace_dict_expr(self, n):
|
|
# globals() / locals() / vars() with no args, or a single-assignment alias of one
|
|
# (g = globals(); g['__builtins__']). Used by the namespace-dict subscript check.
|
|
def _direct(x):
|
|
return (
|
|
isinstance(x, ast.Call)
|
|
and isinstance(x.func, ast.Name)
|
|
and x.func.id in ("globals", "locals", "vars")
|
|
and not x.args
|
|
)
|
|
|
|
if _direct(n):
|
|
return True
|
|
if _analyzer_on and isinstance(n, ast.Name):
|
|
rhs = _scope_idx.resolve(n.id, n, "rhsnode")
|
|
if rhs is not None and _direct(rhs):
|
|
return True
|
|
return False
|
|
|
|
def _is_builtins_ref(self, n):
|
|
# The builtins module (builtins / __builtins__ / an import alias), or a
|
|
# single-assignment alias of one (b = __builtins__; b.__import__('os')).
|
|
if _ast_name_matches(n, self.builtins_aliases):
|
|
return True
|
|
if _analyzer_on and isinstance(n, ast.Name):
|
|
rhs = _scope_idx.resolve(n.id, n, "rhsnode")
|
|
if rhs is not None and _ast_name_matches(rhs, self.builtins_aliases):
|
|
return True
|
|
return False
|
|
|
|
def _is_compile_result(self, arg):
|
|
"""True when ``arg`` is a ``compile(...)`` code object (bare / builtins /
|
|
single-assignment alias / inline-container unwrap). Used to catch code objects
|
|
executed through ``types.FunctionType`` instead of eval/exec."""
|
|
# (compile(src, ...),)[0] / [compile(...)][0] / {'k': compile(...)}['k']: a
|
|
# trivial container unwrap hiding the compile() code object from the direct-call
|
|
# check. Resolve the indexed element and recurse.
|
|
if isinstance(arg, ast.Subscript):
|
|
container = arg.value
|
|
ci = _const_fold(arg.slice, _const_env)
|
|
if isinstance(container, (ast.List, ast.Tuple)) and isinstance(ci, int):
|
|
if -len(container.elts) <= ci < len(container.elts):
|
|
return self._is_compile_result(container.elts[ci])
|
|
if isinstance(container, ast.Dict) and ci is not None:
|
|
for k, v in zip(container.keys, container.values):
|
|
if k is not None and _const_fold(k, _const_env) == ci:
|
|
return self._is_compile_result(v)
|
|
return False
|
|
if isinstance(arg, ast.Call):
|
|
af = arg.func
|
|
if isinstance(af, ast.Name):
|
|
if af.id == "compile" or self.exec_from_aliases.get(af.id) == "compile":
|
|
return True
|
|
if _analyzer_on and _scope_idx.resolve(af.id, arg, "execb") == "compile":
|
|
return True
|
|
elif (
|
|
isinstance(af, ast.Attribute)
|
|
and af.attr == "compile"
|
|
and _ast_name_matches(af.value, self.builtins_aliases)
|
|
):
|
|
return True
|
|
if _analyzer_on and isinstance(arg, ast.Name):
|
|
if _scope_idx.resolve(arg.id, arg, "compiledany"):
|
|
return True
|
|
return False
|
|
|
|
def _attr_obfuscation_targets(self):
|
|
# Modules whose DYNAMIC attribute / dict access (getattr, vars, __dict__) is
|
|
# obfuscation that reaches code execution: the exec / import / shell modules
|
|
# PLUS the deserializer modules -- getattr(pickle, 'loads')(x) and
|
|
# vars(pickle)['loads'](x) are just pickle.loads(x) with the name hidden.
|
|
return (
|
|
_DYNAMIC_ATTR_TARGETS
|
|
| self.os_aliases
|
|
| self.subprocess_aliases
|
|
| self.importlib_aliases
|
|
| self.sys_aliases
|
|
| self.builtins_aliases
|
|
| set(self.deserialize_module_aliases)
|
|
)
|
|
|
|
def visit_Call(self, node):
|
|
# operator.methodcaller('system', 'rm -rf /')(os) applies a deferred method to a
|
|
# module receiver; rewrite it to the direct os.system('rm -rf /') call and analyze
|
|
# that instead so the hidden shell/exec sink is not missed.
|
|
_mc_rewrite = self._methodcaller_module_call(node)
|
|
if _mc_rewrite is not None:
|
|
self.visit_Call(_mc_rewrite)
|
|
return
|
|
if self._is_unbound_mro_gadget(node):
|
|
# type.mro(io.FileIO) / type.__getattribute__(io.FileIO, '__mro__') /
|
|
# getattr(io.FileIO, 'mro'): reaches the unguarded MRO without a .mro / .__mro__
|
|
# attribute for visit_Attribute to see.
|
|
dynamic_exec.append(
|
|
{
|
|
"type": "dynamic_exec",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": "unbound MRO access on a file class recovers an unguarded base (gadget)",
|
|
}
|
|
)
|
|
func = node.func
|
|
# A trailing `.__call__` invokes the underlying callable through its bound
|
|
# method: os.system.__call__(cmd), __import__.__call__('os'),
|
|
# pickle.loads.__call__(blob). Strip it so the shell / import / deserializer /
|
|
# attribute resolvers below see the real sink instead of a plain attribute.
|
|
_ecf = func
|
|
while isinstance(_ecf, ast.Attribute) and _ecf.attr == "__call__":
|
|
_ecf = _ecf.value
|
|
func_name = None
|
|
if isinstance(func, ast.Attribute):
|
|
if isinstance(func.value, ast.Name):
|
|
if func.value.id in self.signal_aliases:
|
|
func_name = f"signal.{func.attr}"
|
|
elif isinstance(func, ast.Name):
|
|
if func.id in ("signal", "setitimer", "alarm", "pthread_sigmask"):
|
|
func_name = func.id
|
|
|
|
if func_name:
|
|
if func_name in ("signal.signal", "signal"):
|
|
if len(node.args) >= 1:
|
|
if _ast_name_matches(node.args[0], ("SIGALRM", "signal.SIGALRM")):
|
|
signal_tampering.append(
|
|
{
|
|
"type": "signal_handler_override",
|
|
"line": node.lineno,
|
|
"description": "Overrides SIGALRM handler",
|
|
}
|
|
)
|
|
elif func_name in ("signal.setitimer", "setitimer"):
|
|
if len(node.args) >= 1:
|
|
if _ast_name_matches(node.args[0], ("ITIMER_REAL", "signal.ITIMER_REAL")):
|
|
signal_tampering.append(
|
|
{
|
|
"type": "timer_manipulation",
|
|
"line": node.lineno,
|
|
"description": "Manipulates ITIMER_REAL timer",
|
|
}
|
|
)
|
|
elif func_name in ("signal.alarm", "alarm"):
|
|
signal_tampering.append(
|
|
{
|
|
"type": "alarm_manipulation",
|
|
"line": node.lineno,
|
|
"description": "Manipulates alarm timer",
|
|
}
|
|
)
|
|
elif func_name in ("signal.pthread_sigmask", "pthread_sigmask"):
|
|
signal_tampering.append(
|
|
{
|
|
"type": "signal_mask",
|
|
"line": node.lineno,
|
|
"description": "Modifies signal mask (may block SIGALRM)",
|
|
}
|
|
)
|
|
|
|
# --- Shell escape detection ---
|
|
# Resolve the FQ function name for os.*/subprocess.* (via the __call__-stripped
|
|
# effective callee so os.system.__call__(cmd) resolves to os.system).
|
|
shell_func = None
|
|
if isinstance(_ecf, ast.Attribute):
|
|
if isinstance(_ecf.value, ast.Name):
|
|
if _ecf.value.id in self.os_aliases:
|
|
shell_func = f"os.{_ecf.attr}"
|
|
elif _ecf.value.id in self.subprocess_aliases:
|
|
shell_func = f"subprocess.{_ecf.attr}"
|
|
# class-body alias reached as ClassName.attr (class C: f = os.system;
|
|
# C.f('rm -rf /')), or an instance-attribute alias (obj.s = os.system;
|
|
# obj.s('rm -rf /')).
|
|
elif _analyzer_on:
|
|
shell_func = _scope_idx.resolve_class_attr(
|
|
_ecf.value.id, _ecf.attr, "shell"
|
|
) or _scope_idx.resolve_instance_attr(_ecf.value.id, _ecf.attr, "shell")
|
|
elif (
|
|
_analyzer_on
|
|
and isinstance(_ecf.value, ast.Call)
|
|
and isinstance(_ecf.value.func, ast.Name)
|
|
):
|
|
# An instance built inline, ClassName().attr: instance lookup still returns
|
|
# the class-body sink alias. class C: s = os.system; C().s('rm -rf /').
|
|
shell_func = _scope_idx.resolve_class_attr(
|
|
_ecf.value.func.id, _ecf.attr, "shell"
|
|
)
|
|
elif isinstance(_ecf.value, ast.Attribute) and _ecf.value.attr in ("os", "posix"):
|
|
# A stdlib module that re-exports os as an attribute (pathlib.os.system,
|
|
# tempfile.os.system, subprocess.os.system): the `.os` attribute IS the os
|
|
# module, so treat the chain as an os.* sink.
|
|
shell_func = f"os.{_ecf.attr}"
|
|
elif isinstance(_ecf.value, ast.Attribute) and _ecf.value.attr == "subprocess":
|
|
# ...and *.subprocess.run (a module re-exporting subprocess).
|
|
shell_func = f"subprocess.{_ecf.attr}"
|
|
elif isinstance(_ecf, ast.Name):
|
|
# from-import aliases: from os import system; system(...)
|
|
shell_func = self.shell_exec_aliases.get(_ecf.id)
|
|
# Stage 4: single-assignment alias `s = os.system; s('rm -rf /')`,
|
|
# resolved in the call's own scope (per-function).
|
|
if shell_func is None and _analyzer_on:
|
|
shell_func = _scope_idx.resolve(_ecf.id, _ecf, "shell")
|
|
elif _analyzer_on and isinstance(_ecf, ast.Subscript):
|
|
# Stage 4: inline literal container index `[os.system][0](...)`.
|
|
shell_func = self._resolve_container_sink(_ecf)
|
|
|
|
if shell_func and shell_func in _SHELL_EXEC_FUNCS:
|
|
# Expand **kwargs dicts to inspect their keys.
|
|
expanded_kwargs: dict[str, ast.AST] = {}
|
|
has_opaque_kwargs = False
|
|
for kw in node.keywords:
|
|
if kw.arg is not None:
|
|
expanded_kwargs[kw.arg] = kw.value
|
|
elif isinstance(kw.value, ast.Dict):
|
|
for k, v in zip(kw.value.keys, kw.value.values):
|
|
key = _extract_string_from_node(k) if k else None
|
|
if key is not None:
|
|
expanded_kwargs[key] = v
|
|
else:
|
|
has_opaque_kwargs = True
|
|
|
|
cmd_kw_values = [v for k, v in expanded_kwargs.items() if k in _CMD_KWARGS]
|
|
all_call_args = list(node.args) + cmd_kw_values
|
|
# A non-literal-False shell= is treated as potentially True (conservative), so a
|
|
# sequence's first element is the shell -c command string, not an argv vector.
|
|
_shell_node = expanded_kwargs.get("shell")
|
|
_shell_maybe_true = not (
|
|
_shell_node is None
|
|
or (isinstance(_shell_node, ast.Constant) and _shell_node.value is False)
|
|
)
|
|
blocked_in_args = _check_args_for_blocked(all_call_args, _shell_maybe_true)
|
|
|
|
# A shell startup variable (BASH_ENV / ENV) in the env= mapping names a script
|
|
# bash / sh SOURCES before the -c payload runs, executing unscanned code
|
|
# (subprocess.run(['bash','-c','echo OK'], env={'BASH_ENV':'env.sh'})). Flag a
|
|
# non-empty value; an empty string is inert. Cover a literal dict, a dict(...)
|
|
# call, and -- for a shell child -- a non-literal mapping we cannot prove free
|
|
# of BASH_ENV / ENV (fail closed). Whether the child is a shell: shell=True, or
|
|
# the argv command word resolves to bash / sh.
|
|
_env_node = expanded_kwargs.get("env")
|
|
if _env_node is not None:
|
|
_is_shell_child = _shell_maybe_true
|
|
if (
|
|
not _is_shell_child
|
|
and node.args
|
|
and isinstance(node.args[0], (ast.List, ast.Tuple))
|
|
):
|
|
_elts0 = [_extract_string_from_node(_e) for _e in node.args[0].elts]
|
|
_ci0 = _blocked_in_argv(_elts0)[1]
|
|
if _ci0 is not None and _ci0 < len(_elts0) and _elts0[_ci0]:
|
|
_is_shell_child = (
|
|
os.path.basename(_elts0[_ci0]).lower() in _SHELL_BINARIES
|
|
)
|
|
if isinstance(_env_node, ast.Dict):
|
|
_opaque_key = False
|
|
for _ek, _ev in zip(_env_node.keys, _env_node.values):
|
|
_ekey = _extract_string_from_node(_ek) if _ek is not None else None
|
|
if _ekey in ("BASH_ENV", "ENV") and (
|
|
_extract_string_from_node(_ev) != ""
|
|
):
|
|
blocked_in_args = blocked_in_args | {"shell-startup-env:" + _ekey}
|
|
elif _ek is not None and _ekey is None:
|
|
_opaque_key = True # a computed key could be BASH_ENV / ENV
|
|
if _opaque_key and _is_shell_child:
|
|
blocked_in_args = blocked_in_args | {"shell-startup-env:opaque"}
|
|
elif (
|
|
isinstance(_env_node, ast.Call)
|
|
and isinstance(_env_node.func, ast.Name)
|
|
and _env_node.func.id == "dict"
|
|
):
|
|
for _kw2 in _env_node.keywords:
|
|
if _kw2.arg in ("BASH_ENV", "ENV") and (
|
|
_extract_string_from_node(_kw2.value) != ""
|
|
):
|
|
blocked_in_args = blocked_in_args | {
|
|
"shell-startup-env:" + _kw2.arg
|
|
}
|
|
elif _kw2.arg is None and _is_shell_child:
|
|
blocked_in_args = blocked_in_args | {"shell-startup-env:opaque"}
|
|
elif _is_shell_child:
|
|
# A non-literal env mapping (env=e, a comprehension) for a shell child
|
|
# cannot be proven free of BASH_ENV / ENV, so fail closed.
|
|
blocked_in_args = blocked_in_args | {"shell-startup-env:non-literal"}
|
|
|
|
# os.execl(path, a0, a1, ...) / os.execv(path, [a0, ...]) / os.spawnl(mode,
|
|
# path, a0, ...) spread the child's argv across separate positional args (or a
|
|
# single list), so scanning each string alone misses a mutating tail like
|
|
# `sed -i ...`. Reconstruct the executed command line (program path + argv[1:],
|
|
# since argv[0] is the cosmetic name) and run the full scanner over it.
|
|
if shell_func.startswith("os.exec") or shell_func.startswith("os.spawn"):
|
|
_name = shell_func.split(".", 1)[1]
|
|
if _name.startswith("spawn"): # spawn*(mode, path, ...)
|
|
_path_node = node.args[1] if len(node.args) > 1 else None
|
|
_tail = node.args[2:]
|
|
else: # exec*(path, ...) / posix_spawn(path, argv, env)
|
|
_path_node = node.args[0] if node.args else None
|
|
_tail = node.args[1:]
|
|
_is_v = "execv" in _name or "spawnv" in _name or _name.startswith("posix_spawn")
|
|
if _is_v:
|
|
_argv = (
|
|
[_extract_string_from_node(e) for e in _tail[0].elts]
|
|
if _tail and isinstance(_tail[0], (ast.List, ast.Tuple))
|
|
else []
|
|
)
|
|
else:
|
|
_argv = [_extract_string_from_node(a) for a in _tail]
|
|
_parts = [p for p in ([_extract_string_from_node(_path_node)] + _argv[1:]) if p]
|
|
if _parts:
|
|
blocked_in_args = blocked_in_args | _find_blocked_commands(
|
|
" ".join(shlex.quote(p) for p in _parts)
|
|
)
|
|
|
|
if has_opaque_kwargs:
|
|
# Can't inspect dynamic **kwargs; flag as unsafe.
|
|
shell_escapes.append(
|
|
{
|
|
"type": "shell_escape_dynamic",
|
|
"line": node.lineno,
|
|
"description": (f"{shell_func}() called with dynamic **kwargs"),
|
|
}
|
|
)
|
|
elif blocked_in_args:
|
|
shell_escapes.append(
|
|
{
|
|
"type": "shell_escape",
|
|
"line": node.lineno,
|
|
"description": (
|
|
f"{shell_func}() invokes blocked command(s): "
|
|
f"{', '.join(sorted(blocked_in_args))}"
|
|
),
|
|
}
|
|
)
|
|
else:
|
|
# Only flag dynamic args for funcs that interpret strings as
|
|
# shell commands, or when shell= might be on. Any non-literal-
|
|
# False shell= is treated as potentially True (conservative).
|
|
_STRING_SHELL_FUNCS = frozenset(
|
|
{
|
|
"os.system",
|
|
"os.popen",
|
|
"os.popen2",
|
|
"os.popen3",
|
|
"os.popen4",
|
|
"subprocess.getoutput",
|
|
"subprocess.getstatusoutput",
|
|
}
|
|
)
|
|
shell_node = expanded_kwargs.get("shell")
|
|
shell_safe = shell_node is None or (
|
|
isinstance(shell_node, ast.Constant) and shell_node.value is False
|
|
)
|
|
# Dynamic shell-exec args (chr/format/concat bypasses).
|
|
if (
|
|
shell_func in _STRING_SHELL_FUNCS
|
|
or shell_func in _SHELL_EXEC_FUNCS
|
|
or not shell_safe
|
|
):
|
|
|
|
def _is_safe_literal(n):
|
|
if _extract_string_from_node(n) is not None:
|
|
return True
|
|
if isinstance(n, (ast.List, ast.Tuple)):
|
|
return all(_extract_string_from_node(e) is not None for e in n.elts)
|
|
return False
|
|
|
|
has_non_literal = any(not _is_safe_literal(a) for a in all_call_args)
|
|
if has_non_literal:
|
|
shell_escapes.append(
|
|
{
|
|
"type": "shell_escape_dynamic",
|
|
"line": node.lineno,
|
|
"description": (
|
|
f"{shell_func}() called with non-literal "
|
|
f"shell command (potential shell escape)"
|
|
),
|
|
}
|
|
)
|
|
|
|
# --- Dynamic execution / obfuscation primitives ---
|
|
# eval / exec / compile (bare builtin or a single-assignment alias).
|
|
exec_func_id = None
|
|
if isinstance(func, ast.Name):
|
|
if func.id in _DYNAMIC_EXEC_BUILTINS:
|
|
exec_func_id = func.id
|
|
elif func.id in self.exec_from_aliases:
|
|
exec_func_id = self.exec_from_aliases[func.id] # from builtins import exec as e
|
|
elif _analyzer_on:
|
|
# single-assignment `e = exec` alias, resolved in the call's scope.
|
|
exec_func_id = _scope_idx.resolve(func.id, func, "execb")
|
|
elif (
|
|
isinstance(func, ast.Attribute)
|
|
and func.attr in _DYNAMIC_EXEC_BUILTINS
|
|
and _ast_name_matches(func.value, self.builtins_aliases)
|
|
):
|
|
exec_func_id = func.attr # builtins.eval(...) / __builtins__.exec(...)
|
|
elif isinstance(func, ast.Attribute) and func.attr == "__call__":
|
|
# eval.__call__("...") / exec.__call__(...) / builtins.eval.__call__(...)
|
|
# invoke the builtin indirectly through its bound method; the payload is
|
|
# still node.args[0], so recover + recurse it exactly like a direct call.
|
|
_base = func.value
|
|
if isinstance(_base, ast.Name):
|
|
if _base.id in _DYNAMIC_EXEC_BUILTINS:
|
|
exec_func_id = _base.id
|
|
elif _base.id in self.exec_from_aliases:
|
|
exec_func_id = self.exec_from_aliases[_base.id]
|
|
elif _analyzer_on:
|
|
exec_func_id = _scope_idx.resolve(_base.id, _base, "execb")
|
|
elif (
|
|
isinstance(_base, ast.Attribute)
|
|
and _base.attr in _DYNAMIC_EXEC_BUILTINS
|
|
and _ast_name_matches(_base.value, self.builtins_aliases)
|
|
):
|
|
exec_func_id = _base.attr
|
|
elif (
|
|
_analyzer_on
|
|
and isinstance(func, ast.Attribute)
|
|
and isinstance(func.value, ast.Name)
|
|
):
|
|
# class-body alias reached as ClassName.attr (class C: e = eval; C.e('...')),
|
|
# or an instance-attribute alias (c.e = exec; c.e('...')).
|
|
exec_func_id = _scope_idx.resolve_class_attr(func.value.id, func.attr, "execb")
|
|
if exec_func_id is None:
|
|
exec_func_id = _scope_idx.resolve_instance_attr(
|
|
func.value.id, func.attr, "execb"
|
|
)
|
|
elif isinstance(func, ast.Subscript):
|
|
# ({'e': exec}['e'])(...) / [exec][0](...): an inline container hides the
|
|
# sink from the bare-name / attribute checks above.
|
|
exec_func_id = self._resolve_container_exec(func)
|
|
|
|
if exec_func_id is not None:
|
|
if _analyzer_on:
|
|
# Stage 2: recover + recurse the payload instead of a blanket ban,
|
|
# so eval("2+2") passes while obfuscated escapes still block.
|
|
_analyze_exec_call(node, exec_func_id)
|
|
else:
|
|
dynamic_exec.append(
|
|
{
|
|
"type": "dynamic_exec",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": f"dynamic code execution via {exec_func_id}()",
|
|
}
|
|
)
|
|
else:
|
|
dynamic_desc = None
|
|
# A dangerous sink passed as a first-class VALUE (not called here) runs its
|
|
# payloads through a higher-order applier the recursive analyzer never sees:
|
|
# list(map(eval, ["..."])), functools.reduce(exec, ...),
|
|
# list(map(os.system, ['rm -rf /'])), functools.partial(subprocess.getoutput,
|
|
# 'wget ...')(), list(map(pickle.loads, [blob])). Flag any bare reference to a
|
|
# dynamic-exec / shell / import / deserializer sink appearing as a call
|
|
# argument, unpacking a literal *[...] / *(...) starred arg too.
|
|
_cand_args = []
|
|
for _a in list(node.args) + [k.value for k in node.keywords]:
|
|
if isinstance(_a, ast.Starred) and isinstance(_a.value, (ast.List, ast.Tuple)):
|
|
_cand_args.extend(_a.value.elts)
|
|
elif isinstance(_a, ast.Starred):
|
|
_cand_args.append(_a.value)
|
|
else:
|
|
_cand_args.append(_a)
|
|
_indirect_sink = None
|
|
for _t in _cand_args:
|
|
_indirect_sink = self._sink_ref_desc(_t)
|
|
if _indirect_sink is not None:
|
|
break
|
|
if _indirect_sink is not None:
|
|
dynamic_exec.append(
|
|
{
|
|
"type": "dynamic_exec",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": (
|
|
f"{_indirect_sink} passed as a value to a higher-order call "
|
|
"(indirect execution of an un-analyzed payload)"
|
|
),
|
|
}
|
|
)
|
|
# An attribute-access call whose (receiver, attr-name) pair is the same
|
|
# obfuscation as getattr(): the builtin getattr/setattr, or the dunder
|
|
# forms object.__getattribute__(obj, 'name') / type.__getattribute__(...)
|
|
# / obj.__getattr__('name') that fetch an attribute without matching the
|
|
# bare getattr name. Normalized here so the gadget + sensitive-module
|
|
# checks below cover all of them.
|
|
_attr_call = None
|
|
_attr_dunder = False # True when reached via __getattribute__/__getattr__
|
|
if (
|
|
isinstance(func, ast.Name)
|
|
and func.id in ("getattr", "setattr")
|
|
and len(node.args) >= 2
|
|
):
|
|
_attr_call = (node.args[0], node.args[1])
|
|
elif isinstance(func, ast.Attribute) and func.attr in (
|
|
"__getattribute__",
|
|
"__getattr__",
|
|
):
|
|
# Unbound form object.__getattribute__(obj, 'name') carries the receiver
|
|
# as arg0; the BOUND form obj.__getattribute__('name') carries it as the
|
|
# attribute's own value (builtins.open.__getattribute__('__closure__')).
|
|
_attr_dunder = True
|
|
if len(node.args) >= 2:
|
|
_attr_call = (node.args[0], node.args[1])
|
|
elif len(node.args) == 1:
|
|
_attr_call = (func.value, node.args[0])
|
|
elif self._attrgetter_name(func) is not None and len(node.args) == 1:
|
|
# operator.attrgetter('name')(obj) evaluates to obj.name -- the same
|
|
# attribute-fetch obfuscation as getattr(obj, 'name'). Detect the
|
|
# attrgetter APPLICATION call itself (node.func is the attrgetter,
|
|
# node.args[0] is the object) so it is caught whether or not the result
|
|
# is immediately invoked: attrgetter('__closure__')(open)[0] and the
|
|
# chained attrgetter('system')(os)('rm -rf /') both normalize here.
|
|
_attr_call = (node.args[0], ast.Constant(value = self._attrgetter_name(func)))
|
|
elif self._methodcaller_getattr_name(func) is not None and len(node.args) == 1:
|
|
# operator.methodcaller('__getattribute__', 'name')(obj) fetches obj.name,
|
|
# the same attribute obfuscation as attrgetter/getattr.
|
|
_attr_call = (
|
|
node.args[0],
|
|
ast.Constant(value = self._methodcaller_getattr_name(func)),
|
|
)
|
|
is_dynamic_import = (
|
|
_ast_name_matches(_ecf, _DYNAMIC_IMPORT_FUNCS)
|
|
or (
|
|
isinstance(_ecf, ast.Name)
|
|
and (
|
|
_ecf.id in ("__import__", "import_module")
|
|
or _ecf.id in self.import_func_aliases
|
|
)
|
|
)
|
|
or (
|
|
isinstance(_ecf, ast.Attribute)
|
|
and _ecf.attr in ("import_module", "reload", "__import__")
|
|
and _ast_name_matches(_ecf.value, self.importlib_aliases)
|
|
)
|
|
or (
|
|
# builtins.__import__('os') / __builtins__.__import__(...), incl. a
|
|
# single-assignment alias (b = __builtins__; b.__import__('os')).
|
|
isinstance(_ecf, ast.Attribute)
|
|
and _ecf.attr == "__import__"
|
|
and self._is_builtins_ref(_ecf.value)
|
|
)
|
|
or (
|
|
# single-assignment `im = importlib.import_module` in scope.
|
|
_analyzer_on
|
|
and isinstance(_ecf, ast.Name)
|
|
and bool(_scope_idx.resolve(_ecf.id, _ecf, "impf"))
|
|
)
|
|
)
|
|
# Deserialization sinks reconstruct arbitrary objects/code from bytes.
|
|
# Resolve aliased imports (from pickle import loads as l), module aliases
|
|
# (import pickle as p; p.loads) and the file-based *.load variants -- not
|
|
# just the exact pickle.loads name. Uses the __call__-stripped effective
|
|
# callee so pickle.loads.__call__(blob) resolves like pickle.loads(blob).
|
|
_deser_fq = None
|
|
if isinstance(_ecf, ast.Attribute) and isinstance(_ecf.value, ast.Name):
|
|
_canon = self.deserialize_module_aliases.get(_ecf.value.id)
|
|
if _canon is not None:
|
|
_cand = f"{_canon}.{_ecf.attr}"
|
|
if _cand in _CODE_DESERIALIZE_SINKS:
|
|
_deser_fq = _cand
|
|
if _deser_fq is None and _analyzer_on:
|
|
# class-body alias reached as ClassName.attr (class C: l = pickle.loads).
|
|
_deser_fq = _scope_idx.resolve_class_attr(_ecf.value.id, _ecf.attr, "deser")
|
|
elif isinstance(_ecf, ast.Name):
|
|
_deser_fq = self.deserialize_aliases.get(_ecf.id)
|
|
if _deser_fq is None and _analyzer_on:
|
|
# single-assignment `l = pickle.loads` in the call's scope.
|
|
_deser_fq = _scope_idx.resolve(_ecf.id, _ecf, "deser")
|
|
elif _analyzer_on and isinstance(_ecf, ast.Subscript):
|
|
# ([pickle.loads][0])(payload) / {'k': pickle.loads}['k'](payload):
|
|
# an inline container hides the sink from the attribute / name checks.
|
|
_deser_fq = self._resolve_container_deser(_ecf)
|
|
if _deser_fq is None:
|
|
_fq_func = _fq_attr_name(_ecf)
|
|
if _fq_func in _CODE_DESERIALIZE_SINKS:
|
|
_deser_fq = _fq_func
|
|
if _deser_fq is None and isinstance(_ecf, ast.Attribute):
|
|
# yaml.load(...) / yaml.load_all(...) reconstruct arbitrary objects unless
|
|
# handed a safe loader. Resolve the (possibly module-aliased) yaml receiver
|
|
# and only flag when no explicit SafeLoader is passed, so yaml.load(data,
|
|
# Loader=yaml.SafeLoader) and yaml.safe_load(data) stay allowed.
|
|
if _ecf.attr in _YAML_LOAD_METHODS and isinstance(_ecf.value, ast.Name):
|
|
if self.deserialize_module_aliases.get(_ecf.value.id) == "yaml":
|
|
if not _yaml_call_has_safe_loader(node):
|
|
_deser_fq = "yaml." + _ecf.attr
|
|
# pickle.Unpickler(f).load() / dill.Unpickler(f).load(): the reduce payload
|
|
# runs on .load(); the sink-name check misses it because the callee is a
|
|
# method on an Unpickler instance, not a *.load module function.
|
|
if (
|
|
_deser_fq is None
|
|
and _ecf.attr in ("load", "load_all")
|
|
and self._is_unpickler_ctor(_ecf.value)
|
|
):
|
|
_deser_fq = "pickle.Unpickler.load"
|
|
if _deser_fq is None and isinstance(_ecf, ast.Name):
|
|
# from yaml import load; load(data): apply the same safe-loader check to the
|
|
# bare-name alias so importing the function directly is not a bypass.
|
|
_ym = self.yaml_load_aliases.get(_ecf.id)
|
|
if _ym is not None and not _yaml_call_has_safe_loader(node):
|
|
_deser_fq = "yaml." + _ym
|
|
if _analyzer_on and _deser_fq is not None:
|
|
dynamic_desc = f"{_deser_fq}() deserializes an unverifiable code payload"
|
|
elif is_dynamic_import:
|
|
# Computed module name (obfuscation) or a dangerous target is unsafe; a
|
|
# benign literal import (huggingface_hub, json, ...) passes. With the
|
|
# analyzer on, the name is constant-folded first so `__import__(
|
|
# "hugging"+"face_hub")` resolves to a real module instead of blocking.
|
|
if node.args:
|
|
if _analyzer_on:
|
|
folded = _const_fold(node.args[0], _const_env)
|
|
mod = folded if isinstance(folded, str) else None
|
|
else:
|
|
mod = _extract_string_from_node(node.args[0])
|
|
else:
|
|
mod = None
|
|
_mod_top = mod.split(".")[0] if mod else None
|
|
if (
|
|
mod is None
|
|
or _mod_top in _DANGEROUS_IMPORT_NAMES
|
|
or _mod_top in _DESERIALIZE_MODULES
|
|
):
|
|
# Deserializer modules (pickle/marshal/...) are dangerous import
|
|
# targets too: __import__('pickle').loads(blob) executes a reduce
|
|
# payload even though a plain `import pickle` is benign.
|
|
dynamic_desc = "dynamic import of a computed or sensitive module name"
|
|
elif _attr_call is not None and (
|
|
(
|
|
isinstance(_const_fold(_attr_call[1], _const_env), str)
|
|
and _const_fold(_attr_call[1], _const_env) in _GADGET_DUNDERS
|
|
)
|
|
or (
|
|
# A __getattribute__/__getattr__ dunder call whose attribute name is
|
|
# not constant-foldable hides a gadget dunder behind a runtime
|
|
# expression (open.__getattribute__(''.join(map(chr, ...)))), which
|
|
# recovers a guarded wrapper's __closure__/cell_contents. No program
|
|
# legitimately spells __getattribute__ with a computed name, so fail
|
|
# closed on the dynamic form.
|
|
_attr_dunder and not isinstance(_const_fold(_attr_call[1], _const_env), str)
|
|
)
|
|
):
|
|
# getattr(anything, '__globals__' / '__subclasses__' / ...) or the
|
|
# object.__getattribute__ equivalent reaches an introspection gadget
|
|
# with no ast.Attribute for visit_Attribute to catch. Direct
|
|
# x.__globals__ is already flagged for ANY receiver, so flag the
|
|
# dynamic-attr-name form regardless of receiver too. (Also closes the
|
|
# __closure__ recovery of a guarded wrapper's original callable.)
|
|
_gv = _const_fold(_attr_call[1], _const_env)
|
|
if isinstance(_gv, str):
|
|
dynamic_desc = (
|
|
f"dynamic attribute access of an introspection gadget dunder ({_gv})"
|
|
)
|
|
else:
|
|
dynamic_desc = (
|
|
"computed attribute name via __getattribute__/__getattr__ "
|
|
"(obfuscated introspection gadget)"
|
|
)
|
|
elif (
|
|
isinstance(func, ast.Name)
|
|
and func.id == "vars"
|
|
and node.args
|
|
and _ast_name_matches(node.args[0], self._attr_obfuscation_targets())
|
|
):
|
|
# vars(os) / vars(__builtins__) returns the module __dict__, the same
|
|
# obfuscation as os.__dict__['system'] but without the attribute access.
|
|
dynamic_desc = "vars() on a sensitive module (dict obfuscation)"
|
|
elif _attr_call is not None and _ast_name_matches(
|
|
_attr_call[0], self._attr_obfuscation_targets()
|
|
):
|
|
# Stage 2 refinement: a benign constant attr (getattr(os, "getpid"))
|
|
# is allowed; only a dynamic attr or a dangerous constant attr blocks.
|
|
# Covers getattr/setattr and object.__getattribute__(builtins, 'eval').
|
|
if _analyzer_on:
|
|
attr_val = _const_fold(_attr_call[1], _const_env)
|
|
if isinstance(attr_val, str):
|
|
if attr_val in _DANGEROUS_ATTR_NAMES or attr_val == "__dict__":
|
|
# getattr(__builtins__, '__dict__')['__import__'] exposes the
|
|
# module namespace the same way vars()/direct .__dict__ do, so
|
|
# a constant '__dict__' on a sensitive module is dangerous too.
|
|
dynamic_desc = (
|
|
"dynamic attribute access on a sensitive module "
|
|
"(attribute-name obfuscation)"
|
|
)
|
|
else:
|
|
dynamic_desc = (
|
|
"dynamic attribute access on a sensitive module "
|
|
"(attribute-name obfuscation)"
|
|
)
|
|
else:
|
|
dynamic_desc = (
|
|
"dynamic attribute access on a sensitive module "
|
|
"(attribute-name obfuscation)"
|
|
)
|
|
elif (
|
|
# sys.modules.get('os') -- the .get() twin of sys.modules['os'], incl. a
|
|
# single-assignment alias (m = sys.modules; m.get('os')).
|
|
isinstance(func, ast.Attribute)
|
|
and func.attr == "get"
|
|
and self._is_sys_modules(func.value)
|
|
and node.args
|
|
):
|
|
# Constant-fold the key so sys.modules.get('o' + 's') is caught, not
|
|
# just a bare literal (the module is already loaded by the prelude).
|
|
_key = _const_fold(node.args[0], _const_env)
|
|
if isinstance(_key, str) and _key.split(".")[0] in _DANGEROUS_IMPORT_NAMES:
|
|
dynamic_desc = "sys.modules.get(...) access to a sensitive module"
|
|
elif (
|
|
# sys.modules.pop('_io', None) / .clear() / .update(...) / .setdefault(...)
|
|
# mutate the loader table just like `del sys.modules[...]`: dropping a
|
|
# guarded module entry lets `import _io` / `import posix` reload a fresh,
|
|
# UNWRAPPED C module (the prelude patched only the old object), bypassing
|
|
# filesystem confinement. The subscript-Store/Del check misses method calls.
|
|
isinstance(func, ast.Attribute)
|
|
and func.attr
|
|
in (
|
|
"pop",
|
|
"popitem",
|
|
"clear",
|
|
"setdefault",
|
|
"update",
|
|
"__setitem__",
|
|
"__delitem__",
|
|
)
|
|
and self._is_sys_modules(func.value)
|
|
):
|
|
dynamic_desc = (
|
|
f"sys.modules.{func.attr}(...) mutates the loader table "
|
|
"(can drop a guarded module for reimport)"
|
|
)
|
|
elif (
|
|
# The same loader-table mutation through an UNBOUND dict method:
|
|
# dict.pop(sys.modules, '_io') / type(sys.modules).__delitem__(sys.modules,
|
|
# ...). The receiver is `dict` / `type(sys.modules)`, not `sys.modules`
|
|
# itself, so the check above misses it; here sys.modules is the first arg.
|
|
isinstance(func, ast.Attribute)
|
|
and func.attr
|
|
in (
|
|
"pop",
|
|
"popitem",
|
|
"clear",
|
|
"setdefault",
|
|
"update",
|
|
"__setitem__",
|
|
"__delitem__",
|
|
)
|
|
and node.args
|
|
and self._is_sys_modules(node.args[0])
|
|
and (
|
|
(isinstance(func.value, ast.Name) and func.value.id == "dict")
|
|
or (
|
|
isinstance(func.value, ast.Call)
|
|
and isinstance(func.value.func, ast.Name)
|
|
and func.value.func.id == "type"
|
|
)
|
|
)
|
|
):
|
|
dynamic_desc = (
|
|
f"unbound dict.{func.attr}(sys.modules, ...) mutates the loader table "
|
|
"(can drop a guarded module for reimport)"
|
|
)
|
|
elif (
|
|
# globals().get('__builtins__') / locals().get(...) / vars().get(...)
|
|
# -- the .get() twin of the globals()['__builtins__'] subscript form.
|
|
isinstance(func, ast.Attribute)
|
|
and func.attr == "get"
|
|
and isinstance(func.value, ast.Call)
|
|
and isinstance(func.value.func, ast.Name)
|
|
and func.value.func.id in ("globals", "locals", "vars")
|
|
and not func.value.args
|
|
and node.args
|
|
):
|
|
_key = _const_fold(node.args[0], _const_env)
|
|
if isinstance(_key, str) and (
|
|
_key in ("__builtins__", "__builtin__")
|
|
or _key.split(".")[0] in _DANGEROUS_IMPORT_NAMES
|
|
):
|
|
dynamic_desc = (
|
|
"namespace-dict .get() access to builtins / a sensitive module"
|
|
)
|
|
elif (
|
|
# dict.__getitem__(globals(), '__builtins__') / dict.get(locals(), ...): the
|
|
# unbound dict-method twin of globals()['__builtins__'], pulling the builtins
|
|
# namespace (or a dangerous module) out of the namespace dict without a
|
|
# subscript node. The receiver is `dict`, not the namespace dict itself, so
|
|
# the subscript scan misses it; here the namespace dict is the first argument.
|
|
isinstance(func, ast.Attribute)
|
|
and func.attr in ("__getitem__", "get")
|
|
and isinstance(func.value, ast.Name)
|
|
and func.value.id == "dict"
|
|
and len(node.args) >= 2
|
|
and isinstance(node.args[0], ast.Call)
|
|
and isinstance(node.args[0].func, ast.Name)
|
|
and node.args[0].func.id in ("globals", "locals", "vars")
|
|
and not node.args[0].args
|
|
):
|
|
_key = _const_fold(node.args[1], _const_env)
|
|
if isinstance(_key, str) and (
|
|
_key in ("__builtins__", "__builtin__")
|
|
or _key.split(".")[0] in _DANGEROUS_IMPORT_NAMES
|
|
):
|
|
dynamic_desc = (
|
|
"unbound dict access to builtins / a sensitive module namespace dict"
|
|
)
|
|
elif (
|
|
(
|
|
# types.FunctionType(compile(src, ...), {})() runs a code object WITHOUT
|
|
# eval/exec, so a dynamic compile() payload reaches execution here even
|
|
# though compile() alone is allowed. Flag when a FunctionType call takes
|
|
# a compile()-derived code object as its first argument.
|
|
(
|
|
isinstance(func, ast.Attribute)
|
|
and func.attr == "FunctionType"
|
|
and _ast_name_matches(func.value, self.types_aliases)
|
|
)
|
|
or (isinstance(func, ast.Name) and func.id in self.functiontype_aliases)
|
|
or (
|
|
# The same constructor is reachable as type(lambda: None): the
|
|
# type of any function IS types.FunctionType, so
|
|
# type(lambda: None)(code, {})() executes a code object too.
|
|
isinstance(func, ast.Call)
|
|
and not func.keywords
|
|
and len(func.args) == 1
|
|
and isinstance(func.args[0], ast.Lambda)
|
|
and (
|
|
(isinstance(func.func, ast.Name) and func.func.id == "type")
|
|
or (
|
|
isinstance(func.func, ast.Attribute)
|
|
and func.func.attr == "type"
|
|
and _ast_name_matches(func.func.value, self.builtins_aliases)
|
|
)
|
|
)
|
|
)
|
|
)
|
|
and node.args
|
|
and self._is_compile_result(node.args[0])
|
|
):
|
|
dynamic_desc = (
|
|
"types.FunctionType() executes a compile() code object "
|
|
"(bypasses the eval/exec gate)"
|
|
)
|
|
elif (
|
|
# runpy.run_path('evil.py') / runpy.run_module('evil') execute a
|
|
# file/module in the guarded interpreter WITHOUT the recursive source
|
|
# analysis exec/eval receive, so a sandboxed snippet can write a local
|
|
# evil.py and run it. Treat these as direct execution sinks. Covers the
|
|
# attribute form, a `from runpy import run_path` bare-name alias, and a
|
|
# single-assignment alias (r = runpy.run_path; r('evil.py')).
|
|
(
|
|
isinstance(func, ast.Attribute)
|
|
and func.attr in ("run_path", "run_module")
|
|
and _ast_name_matches(func.value, self.runpy_aliases)
|
|
)
|
|
or (isinstance(func, ast.Name) and func.id in self.runpy_func_aliases)
|
|
or self._rhs_module_attr(func, ("run_path", "run_module"), self.runpy_aliases)
|
|
):
|
|
if isinstance(func, ast.Attribute):
|
|
_rn = func.attr
|
|
elif func.id in self.runpy_func_aliases:
|
|
_rn = func.id
|
|
else:
|
|
_rn = self._rhs_module_attr(
|
|
func, ("run_path", "run_module"), self.runpy_aliases
|
|
)
|
|
dynamic_desc = f"runpy.{_rn}() executes a file/module without static analysis"
|
|
elif (
|
|
# pty.spawn([...]) / pty.fork() run an unguarded child process (typically a
|
|
# shell) outside the sandbox, the same escape as subprocess / os.system.
|
|
# Covers the attribute form, a `from pty import spawn` bare-name alias, and a
|
|
# single-assignment alias (s = pty.spawn; s([...])).
|
|
(
|
|
isinstance(func, ast.Attribute)
|
|
and func.attr in ("spawn", "fork")
|
|
and _ast_name_matches(func.value, self.pty_aliases)
|
|
)
|
|
or (isinstance(func, ast.Name) and func.id in self.pty_func_aliases)
|
|
or self._rhs_module_attr(func, ("spawn", "fork"), self.pty_aliases)
|
|
):
|
|
if isinstance(func, ast.Attribute):
|
|
_pn = func.attr
|
|
elif func.id in self.pty_func_aliases:
|
|
_pn = func.id
|
|
else:
|
|
_pn = self._rhs_module_attr(func, ("spawn", "fork"), self.pty_aliases)
|
|
dynamic_desc = f"pty.{_pn}() spawns an unguarded child process"
|
|
elif (
|
|
# inspect.getclosurevars(open).nonlocals['real'] recovers the original
|
|
# unguarded callable a guard wrapper closes over, without spelling
|
|
# __closure__ / cell_contents. Block the introspection primitive
|
|
# (attribute form plus a `from inspect import getclosurevars` alias).
|
|
(
|
|
isinstance(func, ast.Attribute)
|
|
and func.attr == "getclosurevars"
|
|
and _ast_name_matches(func.value, self.inspect_aliases)
|
|
)
|
|
or (isinstance(func, ast.Name) and func.id in self.getclosurevars_aliases)
|
|
):
|
|
dynamic_desc = "inspect.getclosurevars() recovers a guarded wrapper's closure"
|
|
elif (
|
|
# gc.get_referents / get_referrers / get_objects walk the object graph to a
|
|
# guard wrapper's closure cell (the original unguarded open/os.* callable)
|
|
# without spelling __closure__ / cell_contents, so a recovered original can
|
|
# then write/read outside the workdir. Block the graph-traversal APIs.
|
|
(
|
|
isinstance(func, ast.Attribute)
|
|
and func.attr in ("get_referents", "get_referrers", "get_objects")
|
|
and _ast_name_matches(func.value, self.gc_aliases)
|
|
)
|
|
or (isinstance(func, ast.Name) and func.id in self.gc_walk_aliases)
|
|
):
|
|
_gn = func.attr if isinstance(func, ast.Attribute) else func.id
|
|
dynamic_desc = (
|
|
f"gc.{_gn}() walks the object graph to a guarded wrapper's closure"
|
|
)
|
|
elif (
|
|
# cls.mro().__getitem__(1) / .pop(1) / cls.__mro__.__getitem__(1): the
|
|
# method-call twin of the subscripted-mro base extraction
|
|
# (visit_Subscript). Same gadget shape (io.FileIO.mro().pop(1) recovers
|
|
# the original FileIO base), so flag an element-extraction method on an
|
|
# mro()/__mro__ receiver.
|
|
isinstance(func, ast.Attribute)
|
|
and func.attr in ("__getitem__", "pop")
|
|
and (
|
|
(
|
|
isinstance(func.value, ast.Call)
|
|
and isinstance(func.value.func, ast.Attribute)
|
|
and func.value.func.attr == "mro"
|
|
and not func.value.args
|
|
)
|
|
or (isinstance(func.value, ast.Attribute) and func.value.attr == "__mro__")
|
|
)
|
|
and (
|
|
# pop() / pop(i) always extract an element; __getitem__ only when the
|
|
# index is a plain integer (not a slice object).
|
|
func.attr == "pop"
|
|
or (
|
|
len(node.args) == 1
|
|
and isinstance(_const_fold(node.args[0], _const_env), int)
|
|
)
|
|
)
|
|
):
|
|
dynamic_desc = f"mro().{func.attr}(...) extracts a base class (gadget)"
|
|
elif isinstance(func, ast.Attribute) and func.attr in (
|
|
"runcode",
|
|
"runsource",
|
|
"load_module",
|
|
"exec_module",
|
|
):
|
|
# code.InteractiveInterpreter().runcode(c) / InteractiveConsole()
|
|
# .runsource(src) execute a code object / source string; an importlib file
|
|
# loader (SourceFileLoader(...).load_module() / spec.loader.exec_module(m))
|
|
# executes a local file. None run through the recursive analysis exec/eval
|
|
# receive, so an opaque payload (a written evil.py, a compile() result, or
|
|
# raw source) runs un-analyzed. These method names are unique to those
|
|
# interpreters / loaders, so flag the call regardless of receiver.
|
|
dynamic_desc = (
|
|
f"{func.attr}() executes code / a file without static analysis "
|
|
"(code interpreter / importlib file loader)"
|
|
)
|
|
if dynamic_desc:
|
|
dynamic_exec.append(
|
|
{
|
|
"type": "dynamic_exec",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": dynamic_desc,
|
|
}
|
|
)
|
|
|
|
self.generic_visit(node)
|
|
|
|
def visit_Attribute(self, node):
|
|
# Introspection gadget dunders (``__subclasses__``, ``__globals__``, ...) are the
|
|
# standard way to walk from a benign object to os/builtins, bypassing the name-based
|
|
# checks. Flag the attribute access itself, then keep descending.
|
|
if node.attr in _GADGET_DUNDERS:
|
|
dynamic_exec.append(
|
|
{
|
|
"type": "dynamic_exec",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": f"introspection gadget attribute {node.attr}",
|
|
}
|
|
)
|
|
elif node.attr == "__dict__" and _ast_name_matches(
|
|
node.value, self._attr_obfuscation_targets()
|
|
):
|
|
# os.__dict__['system']('id') reaches the sink with no getattr call for
|
|
# the name-based checks to see. __dict__ on ordinary objects stays allowed.
|
|
dynamic_exec.append(
|
|
{
|
|
"type": "dynamic_exec",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": "__dict__ access on a sensitive module",
|
|
}
|
|
)
|
|
elif node.attr in ("__mro__", "mro") and self._is_fileclass_recovery_expr(node.value):
|
|
# io.FileIO.mro() / io.FileIO.__mro__ / open.__class__.mro(): the guard
|
|
# replaces io.FileIO with a confining subclass, but its MRO still exposes the
|
|
# UNGUARDED C base. Plain iteration recovers it (for c in io.FileIO.mro(): c(
|
|
# '/etc/x','w')) without ever indexing, so flag any whole-MRO access on a
|
|
# file-class-recovery receiver. Benign int.mro() / cls.__mro__ do not match.
|
|
dynamic_exec.append(
|
|
{
|
|
"type": "dynamic_exec",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": "MRO access on a file class recovers an unguarded base (gadget)",
|
|
}
|
|
)
|
|
self.generic_visit(node)
|
|
|
|
def _is_fileclass_recovery_expr(self, expr):
|
|
"""True when ``expr`` denotes a file/IO class whose MRO walk recovers an UNGUARDED
|
|
file primitive: the guarded ``io.FileIO`` / ``_io.FileIO`` (``.FileIO`` attribute)
|
|
or the type of a file object reached through ``.__class__`` (``open.__class__``,
|
|
``f.__class__``). Ordinary class receivers (``int``, ``cls``, ``type('X', (), {})``)
|
|
are plain Names / Calls and do not match, so benign MRO introspection stays allowed."""
|
|
return isinstance(expr, ast.Attribute) and expr.attr in ("FileIO", "__class__")
|
|
|
|
def _is_unbound_mro_gadget(self, node):
|
|
"""True when ``node`` is an UNBOUND MRO / getattribute call that recovers a file
|
|
class's MRO without spelling ``.mro`` / ``.__mro__`` on the receiver:
|
|
``type.mro(io.FileIO)``, ``type.__getattribute__(io.FileIO, '__mro__')``,
|
|
``object.__getattribute__(io.FileIO, 'mro')`` or ``getattr(io.FileIO, '__mro__')``.
|
|
Iterating the result exposes the unguarded ``_io.FileIO`` C base, so treat it as the
|
|
same recovery gadget as ``io.FileIO.__mro__``."""
|
|
f = node.func
|
|
# getattr(<fileclass>, 'mro' | '__mro__')
|
|
if (
|
|
isinstance(f, ast.Name)
|
|
and f.id == "getattr"
|
|
and len(node.args) >= 2
|
|
and self._is_fileclass_recovery_expr(node.args[0])
|
|
and _const_fold(node.args[1], _const_env) in ("mro", "__mro__")
|
|
):
|
|
return True
|
|
if not isinstance(f, ast.Attribute):
|
|
return False
|
|
# type.mro(<fileclass>)
|
|
if (
|
|
f.attr == "mro"
|
|
and isinstance(f.value, ast.Name)
|
|
and f.value.id == "type"
|
|
and node.args
|
|
and self._is_fileclass_recovery_expr(node.args[0])
|
|
):
|
|
return True
|
|
# type.__getattribute__(<fileclass>, 'mro' | '__mro__') / object.__getattribute__(...)
|
|
if (
|
|
f.attr in ("__getattribute__", "__getattr__")
|
|
and isinstance(f.value, ast.Name)
|
|
and f.value.id in ("type", "object")
|
|
and len(node.args) >= 2
|
|
and self._is_fileclass_recovery_expr(node.args[0])
|
|
and _const_fold(node.args[1], _const_env) in ("mro", "__mro__")
|
|
):
|
|
return True
|
|
return False
|
|
|
|
def visit_Subscript(self, node):
|
|
# An INTEGER-indexed __mro__ (cls.__mro__[1]) or the equivalent method call
|
|
# (cls.mro()[1]) extracts a specific base class the way __bases__[0] does -- the
|
|
# shape used to reach the original FileIO C base class (io.FileIO.mro()[1]) or
|
|
# walk to object/subclasses. Plain iteration (for c in cls.__mro__ / cls.mro())
|
|
# and slicing (cls.__mro__[1:]) yield the whole tuple/list for legitimate
|
|
# introspection, so only a non-slice index is flagged.
|
|
if (
|
|
isinstance(node.ctx, ast.Load)
|
|
and not isinstance(node.slice, ast.Slice)
|
|
and (
|
|
(isinstance(node.value, ast.Attribute) and node.value.attr == "__mro__")
|
|
or (
|
|
isinstance(node.value, ast.Call)
|
|
and isinstance(node.value.func, ast.Attribute)
|
|
and node.value.func.attr == "mro"
|
|
and not node.value.args
|
|
)
|
|
)
|
|
):
|
|
_mro_shape = "__mro__" if isinstance(node.value, ast.Attribute) else "mro()"
|
|
dynamic_exec.append(
|
|
{
|
|
"type": "dynamic_exec",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": f"subscripted {_mro_shape} extracts a base class (gadget)",
|
|
}
|
|
)
|
|
# type(open).__dict__['__closure__'].__get__(open) / type(cell).__dict__[
|
|
# 'cell_contents'].__get__(cell): fetch a gadget descriptor from a type's __dict__
|
|
# BY NAME (a subscript, not an attribute node), then invoke __get__ to recover the
|
|
# guarded wrapper's original callable. Flag a __dict__ subscript keyed by a gadget
|
|
# dunder so the attribute-node gadget scan cannot be side-stepped this way. The same
|
|
# mapping is reachable through vars(type(obj))[...], so cover that form too.
|
|
_dunder_dict = (
|
|
isinstance(node.value, ast.Attribute) and node.value.attr == "__dict__"
|
|
) or (
|
|
isinstance(node.value, ast.Call)
|
|
and isinstance(node.value.func, ast.Name)
|
|
and node.value.func.id == "vars"
|
|
and node.value.args
|
|
)
|
|
if isinstance(node.ctx, ast.Load) and _dunder_dict:
|
|
_dk = _const_fold(node.slice, _const_env)
|
|
if isinstance(_dk, str) and _dk in _GADGET_DUNDERS:
|
|
dynamic_exec.append(
|
|
{
|
|
"type": "dynamic_exec",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": f"__dict__[{_dk!r}] descriptor lookup (gadget)",
|
|
}
|
|
)
|
|
# sys.modules['os'] pulls an already-loaded dangerous module out of the
|
|
# loader table (os/subprocess are loaded by the host). Scope to a Load of a
|
|
# dangerous LITERAL key so legit uses ("x" in sys.modules, sys.modules.get(
|
|
# name), sys.modules[name] = ...) stay allowed.
|
|
v = node.value
|
|
# sys.modules[...] (attribute form), getattr(sys, 'modules')[...], or
|
|
# object.__getattribute__(sys, 'modules')[...] all index the loader table -- as
|
|
# does a single-assignment alias (m = sys.modules; m['os']), so use the alias-aware
|
|
# helper the mutation checks already use.
|
|
is_sys_modules = self._is_sys_modules(v)
|
|
if isinstance(node.ctx, ast.Load) and is_sys_modules:
|
|
# Constant-fold the key so sys.modules['o' + 's'] is caught, not just a
|
|
# bare literal; a truly dynamic key (sys.modules[name]) stays allowed.
|
|
key = _const_fold(node.slice, _const_env)
|
|
if isinstance(key, str) and key.split(".")[0] in _DANGEROUS_IMPORT_NAMES:
|
|
dynamic_exec.append(
|
|
{
|
|
"type": "dynamic_exec",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": "sys.modules[...] access to a sensitive module",
|
|
}
|
|
)
|
|
if isinstance(node.ctx, (ast.Store, ast.Del)) and is_sys_modules:
|
|
# `del sys.modules['posix']; import posix` (or reassigning the entry) drops
|
|
# the guard-patched module object so a fresh, UNWRAPPED C module is imported,
|
|
# bypassing the Stage 5 wrappers. Mutating the loader table has no legitimate
|
|
# use in sandboxed compute, so deny any Store/Del on sys.modules[...].
|
|
dynamic_exec.append(
|
|
{
|
|
"type": "dynamic_exec",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": "sys.modules mutation (del / assign) can drop a guarded module",
|
|
}
|
|
)
|
|
# globals()['__builtins__'] / locals()[...] / vars()[...] pulls the builtins
|
|
# namespace (or a dangerous module) out of the namespace dict, e.g.
|
|
# getattr(globals()['__builtins__'], '__import__')('os'). Flag a Load of a
|
|
# dangerous literal key off a bare globals()/locals()/vars() call.
|
|
if isinstance(node.ctx, ast.Load) and self._is_namespace_dict_expr(v):
|
|
key = _const_fold(node.slice, _const_env)
|
|
if isinstance(key, str) and (
|
|
key in ("__builtins__", "__builtin__")
|
|
or key.split(".")[0] in _DANGEROUS_IMPORT_NAMES
|
|
):
|
|
dynamic_exec.append(
|
|
{
|
|
"type": "dynamic_exec",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": "namespace-dict access to builtins / a sensitive module",
|
|
}
|
|
)
|
|
self.generic_visit(node)
|
|
|
|
def visit_ExceptHandler(self, node):
|
|
if self.loop_depth == 0:
|
|
self.generic_visit(node)
|
|
return
|
|
if node.type is None:
|
|
exception_catching.append(
|
|
{
|
|
"type": "bare_except_in_loop",
|
|
"line": node.lineno,
|
|
"description": "Bare except in loop catches TimeoutError and continues looping",
|
|
}
|
|
)
|
|
elif isinstance(node.type, ast.Name):
|
|
# Flag BaseException/TimeoutError but NOT Exception: `except
|
|
# Exception` can't catch SystemExit/KeyboardInterrupt, so it
|
|
# can't suppress timeout enforcement.
|
|
if node.type.id in ("TimeoutError", "BaseException"):
|
|
exception_catching.append(
|
|
{
|
|
"type": f"catches_{node.type.id}_in_loop",
|
|
"line": node.lineno,
|
|
"description": f"Catches {node.type.id} in loop - may suppress timeout and continue",
|
|
}
|
|
)
|
|
elif isinstance(node.type, ast.Tuple):
|
|
for elt in node.type.elts:
|
|
if isinstance(elt, ast.Name):
|
|
if elt.id in ("TimeoutError", "BaseException"):
|
|
exception_catching.append(
|
|
{
|
|
"type": f"catches_{elt.id}_in_loop",
|
|
"line": node.lineno,
|
|
"description": f"Catches {elt.id} in loop - may suppress timeout and continue",
|
|
}
|
|
)
|
|
self.generic_visit(node)
|
|
|
|
visitor = SignalEscapeVisitor()
|
|
visitor.visit(tree)
|
|
|
|
if visitor.imports_signal and not signal_tampering:
|
|
warnings.append("Code imports 'signal' module - review manually for safety")
|
|
|
|
# Static host policy: block metadata hosts and any literal host outside the
|
|
# trusted allowlist; uploads blocked regardless of host. Dynamic hosts are
|
|
# caught by the bash blocklist.
|
|
network_calls: list[dict] = []
|
|
sensitive_file_reads: list[dict] = []
|
|
_NETWORK_FQ_PREFIXES = (
|
|
"socket.socket",
|
|
"socket.create_connection",
|
|
"socket.getaddrinfo",
|
|
"urllib.request.urlopen",
|
|
"urllib.request.urlretrieve",
|
|
"urllib3.",
|
|
"requests.get",
|
|
"requests.post",
|
|
"requests.put",
|
|
"requests.delete",
|
|
"requests.patch",
|
|
"requests.head",
|
|
"requests.request",
|
|
"requests.Session",
|
|
"http.client.HTTPConnection",
|
|
"http.client.HTTPSConnection",
|
|
"httpx.get",
|
|
"httpx.post",
|
|
"httpx.put",
|
|
"httpx.patch",
|
|
"httpx.delete",
|
|
"httpx.request",
|
|
"httpx.Client",
|
|
"httpx.AsyncClient",
|
|
"aiohttp.ClientSession",
|
|
)
|
|
_UPLOAD_HTTP_METHODS = (
|
|
"requests.post",
|
|
"requests.put",
|
|
"requests.patch",
|
|
"requests.delete",
|
|
"requests.request",
|
|
"httpx.post",
|
|
"httpx.put",
|
|
"httpx.patch",
|
|
"httpx.delete",
|
|
"httpx.request",
|
|
"urllib.request.urlopen",
|
|
"urllib.request.Request",
|
|
)
|
|
_UPLOAD_HF_FQ = (
|
|
"huggingface_hub.upload_file",
|
|
"huggingface_hub.upload_folder",
|
|
"huggingface_hub.upload_large_folder",
|
|
"huggingface_hub.create_commit",
|
|
)
|
|
_UPLOAD_HF_METHODS = frozenset(
|
|
{
|
|
"upload_file",
|
|
"upload_folder",
|
|
"upload_large_folder",
|
|
"create_commit",
|
|
}
|
|
)
|
|
# Cloud-metadata / link-local hosts.
|
|
_METADATA_HOST_LITERALS = {
|
|
"169.254.169.254",
|
|
"fd00:ec2::254",
|
|
"metadata.google.internal",
|
|
"metadata",
|
|
"metadata.tencentyun.com",
|
|
"100.100.100.200",
|
|
"100.100.100.110",
|
|
"169.254.170.2",
|
|
"169.254.170.23",
|
|
}
|
|
_METADATA_HOST_PREFIXES = (
|
|
"169.254.",
|
|
"100.64.",
|
|
)
|
|
# Allowlist kept explicit so each entry is auditable.
|
|
_TRUSTED_PUBLIC_HOST_LITERALS = frozenset(
|
|
{
|
|
# search
|
|
"www.google.com",
|
|
"google.com",
|
|
"www.bing.com",
|
|
"bing.com",
|
|
"duckduckgo.com",
|
|
"html.duckduckgo.com",
|
|
# encyclopedic / reference
|
|
"wikipedia.org",
|
|
"www.wikipedia.org",
|
|
"wikimedia.org",
|
|
"www.wikimedia.org",
|
|
"wikidata.org",
|
|
"www.wikidata.org",
|
|
"commons.wikimedia.org",
|
|
"www.britannica.com",
|
|
"openlibrary.org",
|
|
"www.openstreetmap.org",
|
|
# ML / dev / data
|
|
"huggingface.co",
|
|
"hf.co",
|
|
"github.com",
|
|
"api.github.com",
|
|
"raw.githubusercontent.com",
|
|
"gist.github.com",
|
|
"docs.github.com",
|
|
"pypi.org",
|
|
"files.pythonhosted.org",
|
|
"www.npmjs.com",
|
|
"registry.npmjs.org",
|
|
"crates.io",
|
|
"static.crates.io",
|
|
# docs
|
|
"docs.python.org",
|
|
"python.org",
|
|
"www.python.org",
|
|
"developer.mozilla.org",
|
|
"developer.apple.com",
|
|
"learn.microsoft.com",
|
|
"docs.docker.com",
|
|
"pytorch.org",
|
|
"docs.pytorch.org",
|
|
"tensorflow.org",
|
|
"www.tensorflow.org",
|
|
"numpy.org",
|
|
"pandas.pydata.org",
|
|
"scipy.org",
|
|
"scikit-learn.org",
|
|
"matplotlib.org",
|
|
"fastapi.tiangolo.com",
|
|
"starlette.io",
|
|
# academic
|
|
"arxiv.org",
|
|
"export.arxiv.org",
|
|
"scholar.google.com",
|
|
"openreview.net",
|
|
"semanticscholar.org",
|
|
"www.semanticscholar.org",
|
|
"biorxiv.org",
|
|
"www.biorxiv.org",
|
|
"medrxiv.org",
|
|
"www.medrxiv.org",
|
|
"pubmed.ncbi.nlm.nih.gov",
|
|
"www.ncbi.nlm.nih.gov",
|
|
# Q&A / community
|
|
"stackoverflow.com",
|
|
"stackexchange.com",
|
|
"askubuntu.com",
|
|
"superuser.com",
|
|
"serverfault.com",
|
|
# standards
|
|
"www.w3.org",
|
|
"tools.ietf.org",
|
|
"datatracker.ietf.org",
|
|
"www.rfc-editor.org",
|
|
# reputable news
|
|
"www.bbc.com",
|
|
"www.bbc.co.uk",
|
|
"www.reuters.com",
|
|
"apnews.com",
|
|
"www.nature.com",
|
|
"www.science.org",
|
|
# government / open data
|
|
"data.gov",
|
|
"catalog.data.gov",
|
|
"www.census.gov",
|
|
"www.nasa.gov",
|
|
"data.nasa.gov",
|
|
"www.cdc.gov",
|
|
"www.nih.gov",
|
|
"www.who.int",
|
|
# weather / time
|
|
"api.weather.gov",
|
|
"worldtimeapi.org",
|
|
}
|
|
)
|
|
_TRUSTED_PUBLIC_HOST_SUFFIXES = (
|
|
".wikipedia.org",
|
|
".wikimedia.org",
|
|
".wiktionary.org",
|
|
".wikibooks.org",
|
|
".wikiquote.org",
|
|
".wikisource.org",
|
|
".wikiversity.org",
|
|
".wikivoyage.org",
|
|
".stackexchange.com",
|
|
".hf.co",
|
|
".huggingface.co",
|
|
".githubusercontent.com",
|
|
".github.io",
|
|
".arxiv.org",
|
|
".readthedocs.io",
|
|
".readthedocs.org",
|
|
)
|
|
_SENSITIVE_FILE_PREFIXES = (
|
|
"/etc/passwd",
|
|
"/etc/shadow",
|
|
"/etc/sudoers",
|
|
"/etc/ssh/",
|
|
)
|
|
_SENSITIVE_FILE_RE = re.compile(r"^/proc/(?:self|\d+)/(?:environ|cmdline|task/\d+/environ)$")
|
|
|
|
def _normalize_host(host: str) -> str:
|
|
if not host:
|
|
return ""
|
|
h = host.strip().lower().rstrip(".")
|
|
if "@" in h:
|
|
h = h.split("@", 1)[1]
|
|
if h.startswith("[") and "]" in h:
|
|
h = h[1 : h.index("]")]
|
|
elif h.count(":") == 1:
|
|
h = h.split(":", 1)[0]
|
|
return h
|
|
|
|
def _is_metadata_host(host: str) -> bool:
|
|
h = _normalize_host(host)
|
|
if not h:
|
|
return False
|
|
if h in _METADATA_HOST_LITERALS:
|
|
return True
|
|
if any(h.startswith(p) for p in _METADATA_HOST_PREFIXES):
|
|
return True
|
|
return False
|
|
|
|
def _is_trusted_host(host: str) -> bool:
|
|
h = _normalize_host(host)
|
|
if not h:
|
|
return False
|
|
if h in _TRUSTED_PUBLIC_HOST_LITERALS:
|
|
return True
|
|
return any(h.endswith(s) for s in _TRUSTED_PUBLIC_HOST_SUFFIXES)
|
|
|
|
def _call_is_upload_shape(node: ast.Call, fq: str) -> bool:
|
|
"""True for statically obvious upload shapes (files=, data=open(), bytes literal)."""
|
|
if fq in _UPLOAD_HF_FQ:
|
|
return True
|
|
if fq not in _UPLOAD_HTTP_METHODS:
|
|
return False
|
|
for kw in node.keywords or []:
|
|
if kw.arg == "files":
|
|
return True
|
|
if kw.arg == "data":
|
|
v = kw.value
|
|
if isinstance(v, ast.Call) and isinstance(v.func, ast.Name) and v.func.id == "open":
|
|
return True
|
|
if isinstance(v, ast.Constant) and isinstance(v.value, (bytes, bytearray)):
|
|
return True
|
|
return False
|
|
|
|
# Bare method-name fallback (`x.upload_file(...)`) is fuzzy, so it fires only
|
|
# when huggingface_hub/hf_api is imported; else paramiko.upload_file,
|
|
# boto3.create_commit, etc. would false-positive. Pre-scan for the imports.
|
|
_HF_IMPORT_MODULES = (
|
|
"huggingface_hub",
|
|
"hf_api",
|
|
"huggingface_hub.hf_api",
|
|
)
|
|
|
|
def _module_has_hf_import(tree: ast.AST) -> bool:
|
|
for n in ast.walk(tree):
|
|
if isinstance(n, ast.Import):
|
|
for alias in n.names:
|
|
if alias.name.split(".", 1)[0] in _HF_IMPORT_MODULES:
|
|
return True
|
|
elif isinstance(n, ast.ImportFrom):
|
|
root = (n.module or "").split(".", 1)[0]
|
|
if root in _HF_IMPORT_MODULES:
|
|
return True
|
|
elif isinstance(n, ast.Call) and n.args:
|
|
# __import__('huggingface_hub'), importlib.import_module(...),
|
|
# and bare import_module(...) (via `from importlib import ...`).
|
|
arg0 = n.args[0]
|
|
if not (isinstance(arg0, ast.Constant) and isinstance(arg0.value, str)):
|
|
continue
|
|
if arg0.value.split(".", 1)[0] not in _HF_IMPORT_MODULES:
|
|
continue
|
|
func = n.func
|
|
if isinstance(func, ast.Name) and func.id in {
|
|
"__import__",
|
|
"import_module",
|
|
}:
|
|
return True
|
|
if isinstance(func, ast.Attribute) and func.attr == "import_module":
|
|
return True
|
|
return False
|
|
|
|
_hf_in_scope = _module_has_hf_import(tree)
|
|
|
|
def _method_call_hf_upload_name(node: ast.Call) -> str | None:
|
|
"""Return the HF upload method name (`upload_file`, ...) or None. Covers
|
|
the Attribute and bare-Name forms; the bare-name branch fires only when
|
|
an HF import is in scope so paramiko/boto3 don't false-positive."""
|
|
if not _hf_in_scope:
|
|
return None
|
|
f = node.func
|
|
if isinstance(f, ast.Attribute) and f.attr in _UPLOAD_HF_METHODS:
|
|
return f.attr
|
|
if isinstance(f, ast.Name) and f.id in _UPLOAD_HF_METHODS:
|
|
return f.id
|
|
return None
|
|
|
|
# Kwargs that ship a credential over the wire. The sandbox env strips
|
|
# credentials up front, so any value here is hard-coded or lifted from parent.
|
|
_HF_SENSITIVE_KWARGS = frozenset(
|
|
{
|
|
"token",
|
|
"hf_token",
|
|
"api_token",
|
|
"api_key",
|
|
"auth_token",
|
|
"access_token",
|
|
"password",
|
|
"secret",
|
|
}
|
|
)
|
|
|
|
def _is_os_environ(node: ast.AST) -> bool:
|
|
return (
|
|
isinstance(node, ast.Attribute)
|
|
and node.attr == "environ"
|
|
and isinstance(node.value, ast.Name)
|
|
and node.value.id == "os"
|
|
)
|
|
|
|
def _reads_env_or_secret(node: ast.AST | None) -> bool:
|
|
"""True if any node in the subtree resolves to an env/process read.
|
|
|
|
Walks the whole subtree (not just the root) to catch wrappers like
|
|
`str(os.environ)`. Covers os.environ[/.get]/os.getenv, bare getenv, and
|
|
subprocess.{run,check_output,...} that could lift parent env via printenv.
|
|
"""
|
|
if node is None:
|
|
return False
|
|
for sub in ast.walk(node):
|
|
if _is_os_environ(sub):
|
|
return True
|
|
if isinstance(sub, ast.Call):
|
|
f = sub.func
|
|
if isinstance(f, ast.Attribute):
|
|
if (
|
|
f.attr in {"getenv", "getenvb"}
|
|
and isinstance(f.value, ast.Name)
|
|
and f.value.id == "os"
|
|
):
|
|
return True
|
|
if (
|
|
f.attr
|
|
in {
|
|
"check_output",
|
|
"run",
|
|
"Popen",
|
|
"getoutput",
|
|
"getstatusoutput",
|
|
}
|
|
and isinstance(f.value, ast.Name)
|
|
and f.value.id in {"subprocess", "commands"}
|
|
):
|
|
return True
|
|
if isinstance(f, ast.Name) and f.id in {"getenv", "getenvb"}:
|
|
return True
|
|
return False
|
|
|
|
def _is_safe_relative_path(path: str) -> bool:
|
|
"""Relative path with no leading `/`, `~`, drive letter, or `..` segments."""
|
|
if not isinstance(path, str) or not path:
|
|
return False
|
|
if path[0] in ("/", "\\", "~"):
|
|
return False
|
|
if len(path) >= 2 and path[1] == ":":
|
|
return False
|
|
return ".." not in path.replace("\\", "/").split("/")
|
|
|
|
def _path_arg_is_sandbox_local(node: ast.AST | None) -> bool:
|
|
"""Whether the path argument resolves to a sandbox-local literal."""
|
|
if node is None:
|
|
return False
|
|
if isinstance(node, ast.Constant) and isinstance(node.value, (bytes, bytearray)):
|
|
return True # inline bytes, no file access
|
|
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
|
return _is_safe_relative_path(node.value)
|
|
if isinstance(node, ast.Call):
|
|
f = node.func
|
|
is_open = (isinstance(f, ast.Name) and f.id == "open") or (
|
|
isinstance(f, ast.Attribute) and f.attr == "open"
|
|
)
|
|
if is_open and node.args:
|
|
a0 = node.args[0]
|
|
return (
|
|
isinstance(a0, ast.Constant)
|
|
and isinstance(a0.value, str)
|
|
and _is_safe_relative_path(a0.value)
|
|
)
|
|
return False
|
|
|
|
def _hf_upload_violation(node: ast.Call, method_name: str) -> str | None:
|
|
"""Inspect an HF upload call; return a violation reason or None.
|
|
|
|
Policy: HF uploads are allowed only when (a) no sensitive kwarg is set,
|
|
(b) no positional / keyword value reads `os.environ` or related env
|
|
readers, and (c) the path arg is a sandbox-local literal: a relative
|
|
string with no `..`, an `open(<literal>)`, or inline bytes. Dynamic /
|
|
variable paths are rejected since safety can't be proven statically and
|
|
a wrong-allow means credential exfiltration.
|
|
"""
|
|
for kw in node.keywords or []:
|
|
if kw.arg in _HF_SENSITIVE_KWARGS:
|
|
return (
|
|
f"HF upload {kw.arg}= cannot be set from sandboxed code; "
|
|
"uploads run with the sandbox identity only"
|
|
)
|
|
all_values = list(node.args or []) + [kw.value for kw in (node.keywords or [])]
|
|
for v in all_values:
|
|
if _reads_env_or_secret(v):
|
|
return (
|
|
"HF upload cannot include os.environ / os.getenv / subprocess "
|
|
"env reads; secrets and tokens must not be exfiltrated"
|
|
)
|
|
if method_name == "create_commit":
|
|
for kw in node.keywords or []:
|
|
if kw.arg == "operations" and isinstance(kw.value, ast.List):
|
|
for elt in kw.value.elts:
|
|
if isinstance(elt, ast.Call):
|
|
inner = _hf_upload_violation(elt, "upload_file")
|
|
if inner:
|
|
return inner
|
|
return None
|
|
path_node: ast.AST | None = node.args[0] if node.args else None
|
|
for kw in node.keywords or []:
|
|
if kw.arg in ("path_or_fileobj", "folder_path"):
|
|
path_node = kw.value
|
|
break
|
|
if not _path_arg_is_sandbox_local(path_node):
|
|
return (
|
|
"HF upload path must be a sandbox-local relative-path literal "
|
|
"(no absolute paths, no '..' segments, no dynamic expressions)"
|
|
)
|
|
return None
|
|
|
|
# Network-module import aliases so the FQ prefix match sees the canonical module even
|
|
# when it is renamed: import requests as r -> {"r": "requests"}, import urllib.request as
|
|
# u -> {"u": "urllib.request"}, from urllib import request as req -> {"req":
|
|
# "urllib.request"}. Without this, r.get('http://169.254.169.254/') builds fq="r.get"
|
|
# and skips every metadata / allowlist / upload check.
|
|
_NET_TOP_MODULES = ("socket", "urllib", "urllib3", "requests", "http", "httpx", "aiohttp")
|
|
_net_aliases: dict[str, str] = {}
|
|
for _n in ast.walk(tree):
|
|
if isinstance(_n, ast.Import):
|
|
for _a in _n.names:
|
|
if _a.asname and _a.name.split(".")[0] in _NET_TOP_MODULES:
|
|
_net_aliases[_a.asname] = _a.name
|
|
elif isinstance(_n, ast.ImportFrom) and _n.module:
|
|
if _n.module.split(".")[0] in _NET_TOP_MODULES:
|
|
for _a in _n.names:
|
|
_net_aliases[_a.asname or _a.name] = f"{_n.module}.{_a.name}"
|
|
|
|
# The URL / address keyword arguments the stdlib + common HTTP clients accept, so a
|
|
# keyword host (requests.get(url=...), urlopen(url=...), create_connection(address=...))
|
|
# is extracted the same as a positional one.
|
|
_NET_URL_KWARGS = ("url",)
|
|
_NET_ADDR_KWARGS = ("address", "sock_addr")
|
|
|
|
class NetworkAndIoVisitor(ast.NodeVisitor):
|
|
def visit_Call(self, node):
|
|
parts: list[str] = []
|
|
cur = node.func
|
|
while isinstance(cur, ast.Attribute):
|
|
parts.insert(0, cur.attr)
|
|
cur = cur.value
|
|
if isinstance(cur, ast.Name):
|
|
parts.insert(0, cur.id)
|
|
# Resolve a renamed network module (import requests as r) to its canonical name
|
|
# so the FQ-prefix match below still fires.
|
|
if parts and parts[0] in _net_aliases:
|
|
parts = _net_aliases[parts[0]].split(".") + parts[1:]
|
|
fq = ".".join(parts) if parts else ""
|
|
|
|
hf_upload_name = _method_call_hf_upload_name(node)
|
|
if hf_upload_name is not None:
|
|
violation = _hf_upload_violation(node, hf_upload_name)
|
|
if violation is not None:
|
|
network_calls.append(
|
|
{
|
|
"type": "upload_blocked",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": f"Blocked: {violation}",
|
|
}
|
|
)
|
|
|
|
# Direct sock.connect((host, port)) bypasses the FQ-prefix branch.
|
|
if isinstance(node.func, ast.Attribute) and node.func.attr == "connect":
|
|
a0 = node.args[0] if node.args else None
|
|
if a0 is None:
|
|
for _kw in node.keywords or []:
|
|
if _kw.arg == "address":
|
|
a0 = _kw.value
|
|
break
|
|
host_lit = None
|
|
if isinstance(a0, ast.Tuple) and a0.elts:
|
|
e0 = a0.elts[0]
|
|
if isinstance(e0, ast.Constant) and isinstance(e0.value, str):
|
|
host_lit = e0.value
|
|
elif isinstance(a0, ast.Constant) and isinstance(a0.value, str):
|
|
host_lit = a0.value
|
|
if host_lit:
|
|
if _is_metadata_host(host_lit):
|
|
network_calls.append(
|
|
{
|
|
"type": "metadata_host_blocked",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": "Blocked: cloud-metadata host",
|
|
}
|
|
)
|
|
elif not _is_trusted_host(host_lit):
|
|
network_calls.append(
|
|
{
|
|
"type": "untrusted_host_blocked",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": (
|
|
"Blocked: host not in sandbox allowlist; "
|
|
"use an allowed informational source"
|
|
),
|
|
}
|
|
)
|
|
|
|
if fq and any(fq.startswith(p) for p in _NETWORK_FQ_PREFIXES):
|
|
# 1) Upload-shape check (host-independent).
|
|
if _call_is_upload_shape(node, fq):
|
|
network_calls.append(
|
|
{
|
|
"type": "upload_blocked",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": ("Blocked: file upload disallowed in sandbox"),
|
|
}
|
|
)
|
|
|
|
# 2) Extract literal host (URL string or (host, port) tuple). The host may be
|
|
# a positional first arg OR a keyword (requests.get(url=...),
|
|
# urlopen(url=...), create_connection(address=(host, port))).
|
|
host_arg = None
|
|
url_arg = None
|
|
a0 = node.args[0] if node.args else None
|
|
if a0 is None:
|
|
for _kw in node.keywords or []:
|
|
if _kw.arg in _NET_URL_KWARGS:
|
|
a0 = _kw.value
|
|
break
|
|
if _kw.arg in _NET_ADDR_KWARGS:
|
|
a0 = _kw.value
|
|
break
|
|
if a0 is not None:
|
|
if isinstance(a0, ast.Constant) and isinstance(a0.value, str):
|
|
url_arg = a0.value
|
|
elif isinstance(a0, ast.Tuple) and a0.elts:
|
|
e0 = a0.elts[0]
|
|
if isinstance(e0, ast.Constant) and isinstance(e0.value, str):
|
|
host_arg = e0.value
|
|
if url_arg and host_arg is None:
|
|
m = re.match(r"^\w+://([^/?#]+)", url_arg)
|
|
if m:
|
|
host_arg = m.group(1)
|
|
|
|
if host_arg:
|
|
if _is_metadata_host(host_arg):
|
|
network_calls.append(
|
|
{
|
|
"type": "metadata_host_blocked",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": "Blocked: cloud-metadata host",
|
|
}
|
|
)
|
|
elif not _is_trusted_host(host_arg):
|
|
network_calls.append(
|
|
{
|
|
"type": "untrusted_host_blocked",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": (
|
|
"Blocked: host not in sandbox allowlist; "
|
|
"use an allowed informational source"
|
|
),
|
|
}
|
|
)
|
|
|
|
is_open_call = (
|
|
(isinstance(node.func, ast.Name) and node.func.id == "open")
|
|
or fq in ("io.open", "pathlib.Path.open")
|
|
or fq.endswith(".open")
|
|
)
|
|
if is_open_call and node.args:
|
|
a0 = node.args[0]
|
|
path_lit = None
|
|
if isinstance(a0, ast.Constant) and isinstance(a0.value, str):
|
|
path_lit = a0.value
|
|
if path_lit:
|
|
flagged = False
|
|
if any(path_lit.startswith(p) for p in _SENSITIVE_FILE_PREFIXES):
|
|
flagged = True
|
|
elif _SENSITIVE_FILE_RE.match(path_lit):
|
|
flagged = True
|
|
if flagged:
|
|
sensitive_file_reads.append(
|
|
{
|
|
"type": "sensitive_file_read",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": (
|
|
f"open({path_lit!r}) targets a host identity / "
|
|
"credential file; sandboxed code may not read it"
|
|
),
|
|
}
|
|
)
|
|
self.generic_visit(node)
|
|
|
|
def _fs_block(node, description):
|
|
filesystem_violations.append(
|
|
{
|
|
"type": "filesystem_violation",
|
|
"line": getattr(node, "lineno", -1),
|
|
"description": description,
|
|
}
|
|
)
|
|
|
|
# Read-only scanner: filesystem WRITES are confined at runtime by the Stage 5
|
|
# realpath backstop, so this static pass only blocks host-secret READS (the
|
|
# backstop leaves reads unpatched). A sensitive absolute / ~-rooted literal in
|
|
# ANY call arg is flagged -- this covers open()/os.open and library loaders that
|
|
# internally open the path (pandas.read_csv('/etc/shadow'), numpy.load('/etc/passwd')).
|
|
# The `..` / `~` traversal-escape form is flagged only for the dedicated
|
|
# open()/read callees, so benign relative-path building (os.path.join('..','x'))
|
|
# is not caught. Dynamic (non-foldable) paths are left to the runtime backstop.
|
|
_READ_METHODS = ("read_text", "read_bytes")
|
|
# Pathlib read methods carry the path on the RECEIVER, not in an argument:
|
|
# Path('../../.ssh/id_rsa').read_text() has no call args, so the constructor path
|
|
# must be inspected separately.
|
|
_PATHLIB_READ_METHODS = ("read_text", "read_bytes", "open")
|
|
_PATHLIB_CTORS = (
|
|
"Path",
|
|
"PurePath",
|
|
"PosixPath",
|
|
"PurePosixPath",
|
|
"WindowsPath",
|
|
"PureWindowsPath",
|
|
)
|
|
# shutil.copy*/move read their SOURCE (first arg) from the host, so a `..` traversal
|
|
# or ~ source copies a host secret into the workdir even though it is not an open()/
|
|
# read callee. Treat them as read callees so the traversal/sensitive check applies.
|
|
_SHUTIL_COPY_SINKS = (
|
|
"shutil.copy",
|
|
"shutil.copy2",
|
|
"shutil.copyfile",
|
|
"shutil.copytree",
|
|
"shutil.move",
|
|
)
|
|
_SHUTIL_COPY_METHODS = ("copy", "copy2", "copyfile", "copytree", "move")
|
|
# Import aliases so the traversal check still recognizes a renamed callee:
|
|
# from pathlib import Path as P -> P('../../etc/passwd').read_text()
|
|
# import shutil as sh -> sh.copy('../../etc/passwd', 'x')
|
|
_pathlib_ctor_aliases = set(_PATHLIB_CTORS)
|
|
_shutil_aliases = {"shutil"}
|
|
# from shutil import copy as c / copyfile / move -> bare-name aliases whose SOURCE (first
|
|
# arg) is a host read, e.g. c('../../../etc/passwd', 'x'). Tracked so the traversal check
|
|
# treats them as read callees like the attribute form shutil.copy(...).
|
|
_shutil_copy_from_aliases: set[str] = set()
|
|
# from os import open as oo / from io import open as X / from builtins import open as X:
|
|
# the read-only os.open is deliberately allowed OUTSIDE the workdir by the runtime
|
|
# guard, so a traversal read via such an alias must be caught statically.
|
|
_open_from_aliases: set[str] = set()
|
|
# Receiver-module aliases for the open() attribute form (import builtins as b; b.open(...),
|
|
# import io as i; i.open(...), import os as o; o.open(...)), so an aliased-module read is
|
|
# recognized like the literal builtins/io/os.open forms.
|
|
_open_mod_aliases = {"builtins", "__builtins__", "io", "os"}
|
|
# os/subprocess module aliases + from-import shell-name aliases, so a shell command
|
|
# string that reads a host secret (os.system('cat /etc/passwd')) is scanned even when
|
|
# os/subprocess is renamed.
|
|
_os_mod_aliases = {"os"}
|
|
_subprocess_mod_aliases = {"subprocess"}
|
|
_shell_name_aliases: dict[str, str] = {}
|
|
# from subprocess import run as r / call / check_call / check_output / Popen -> bare-name
|
|
# aliases that run an unguarded child, so r(['cat', '../../etc/shadow']) reads a host
|
|
# secret. Tracked so the read-callee traversal check recognizes them like subprocess.run.
|
|
_subprocess_exec_from_aliases: set[str] = set()
|
|
# from os.path import join as j / normpath / abspath -> {alias: 'join'} so a path builder
|
|
# folder recognizes the bare-name form open(join('/etc', 'passwd')).
|
|
_pathfunc_from_aliases: dict[str, str] = {}
|
|
# operator module + `from operator import methodcaller` aliases, so a deferred method
|
|
# applied to an os/subprocess receiver (methodcaller('popen', 'cat /etc/passwd')(os)) is
|
|
# rewritten to the direct call before the read scanner runs.
|
|
_operator_mod_aliases = {"operator"}
|
|
_methodcaller_from_aliases: set[str] = set()
|
|
for _imp in ast.walk(tree):
|
|
if isinstance(_imp, ast.ImportFrom) and _imp.module == "pathlib":
|
|
for _a in _imp.names:
|
|
if _a.name in _PATHLIB_CTORS:
|
|
_pathlib_ctor_aliases.add(_a.asname or _a.name)
|
|
elif isinstance(_imp, ast.ImportFrom) and _imp.module in ("os", "io", "builtins"):
|
|
for _a in _imp.names:
|
|
if _a.name == "open":
|
|
_open_from_aliases.add(_a.asname or "open")
|
|
# `from os import system as s` / popen: record the os shell-exec alias here too
|
|
# (this elif consumes the `os` module, so the subprocess branch below never sees
|
|
# it), else _scan_shell_string_reads skips s('cat /etc/passwd').
|
|
_fq = f"{_imp.module}.{_a.name}"
|
|
if _fq in _SHELL_EXEC_FUNCS:
|
|
_shell_name_aliases[_a.asname or _a.name] = _fq
|
|
elif isinstance(_imp, ast.ImportFrom) and _imp.module == "shutil":
|
|
for _a in _imp.names:
|
|
if _a.name in _SHUTIL_COPY_METHODS:
|
|
_shutil_copy_from_aliases.add(_a.asname or _a.name)
|
|
elif isinstance(_imp, ast.ImportFrom) and _imp.module == "subprocess":
|
|
for _a in _imp.names:
|
|
_fq = f"subprocess.{_a.name}"
|
|
if _fq in _SHELL_EXEC_FUNCS:
|
|
_shell_name_aliases[_a.asname or _a.name] = _fq
|
|
if _a.name in ("run", "call", "check_call", "check_output", "Popen"):
|
|
_subprocess_exec_from_aliases.add(_a.asname or _a.name)
|
|
elif isinstance(_imp, ast.ImportFrom) and _imp.module in (
|
|
"os.path",
|
|
"posixpath",
|
|
"ntpath",
|
|
):
|
|
for _a in _imp.names:
|
|
if _a.name in ("join", "normpath", "abspath"):
|
|
_pathfunc_from_aliases[_a.asname or _a.name] = _a.name
|
|
elif isinstance(_imp, ast.ImportFrom) and _imp.module == "operator":
|
|
for _a in _imp.names:
|
|
if _a.name == "methodcaller":
|
|
_methodcaller_from_aliases.add(_a.asname or _a.name)
|
|
elif isinstance(_imp, ast.Import):
|
|
for _a in _imp.names:
|
|
if _a.name == "shutil":
|
|
_shutil_aliases.add(_a.asname or "shutil")
|
|
elif _a.name == "operator":
|
|
_operator_mod_aliases.add(_a.asname or "operator")
|
|
elif _a.name == "os":
|
|
_os_mod_aliases.add(_a.asname or "os")
|
|
_open_mod_aliases.add(_a.asname or "os") # o.open(...)
|
|
elif _a.name in ("posix", "nt"):
|
|
# posix / nt are the os C backend (posix.system == os.system), so a shell
|
|
# string passed to them must be scanned for embedded secret reads too.
|
|
_os_mod_aliases.add(_a.asname or _a.name)
|
|
elif _a.name in ("io", "builtins"):
|
|
_open_mod_aliases.add(_a.asname or _a.name) # i.open(...) / b.open(...)
|
|
elif _a.name == "subprocess":
|
|
_subprocess_mod_aliases.add(_a.asname or "subprocess")
|
|
|
|
def _fold_pathjoin_call(call):
|
|
# Fold an os.path.join/normpath/abspath call that _const_fold's owner check misses
|
|
# because os is aliased (import os as o -> o.path.join) or the function is
|
|
# from-imported (from os.path import join -> join(...)). Recurses through
|
|
# _fold_read_arg so scope-local constants inside the args still resolve.
|
|
if not isinstance(call, ast.Call):
|
|
return None
|
|
fn = call.func
|
|
pname = None
|
|
if isinstance(fn, ast.Attribute) and fn.attr in ("join", "normpath", "abspath"):
|
|
owner = fn.value
|
|
if (
|
|
isinstance(owner, ast.Attribute)
|
|
and owner.attr == "path"
|
|
and isinstance(owner.value, ast.Name)
|
|
and owner.value.id in _os_mod_aliases
|
|
):
|
|
pname = fn.attr
|
|
elif isinstance(owner, ast.Name) and owner.id in ("posixpath", "ntpath"):
|
|
pname = fn.attr
|
|
elif isinstance(fn, ast.Name) and fn.id in _pathfunc_from_aliases:
|
|
pname = _pathfunc_from_aliases[fn.id]
|
|
if pname is None or not call.args:
|
|
return None
|
|
parts = []
|
|
for a in call.args:
|
|
v = _fold_read_arg(a)
|
|
if v is None:
|
|
return None
|
|
parts.append(v)
|
|
try:
|
|
if pname == "join":
|
|
return os.path.join(*parts)
|
|
if len(parts) == 1:
|
|
return (
|
|
os.path.normpath(parts[0]) if pname == "normpath" else os.path.abspath(parts[0])
|
|
)
|
|
except Exception:
|
|
return None
|
|
return None
|
|
|
|
def _unwrap_container_node(n):
|
|
# `[open][0]` / `(open,)[0]` / `{'k': open}['k']`: resolve an inline literal-container
|
|
# index to the element node so a container-hidden alias is seen through.
|
|
if not isinstance(n, ast.Subscript):
|
|
return n
|
|
container = n.value
|
|
ci = _const_fold(n.slice, _const_env)
|
|
if isinstance(container, (ast.List, ast.Tuple)) and isinstance(ci, int):
|
|
if -len(container.elts) <= ci < len(container.elts):
|
|
return container.elts[ci]
|
|
if isinstance(container, ast.Dict) and ci is not None:
|
|
for k, v in zip(container.keys, container.values):
|
|
if k is not None and _const_fold(k, _const_env) == ci:
|
|
return v
|
|
return n
|
|
|
|
def _resolves_to_open(fn):
|
|
# A callee that is `open`, a `from os/io/builtins import open as X` alias, a
|
|
# single-assignment alias (o = open; o('../../etc/passwd').read()), a
|
|
# container-hidden alias (o = [open][0]; o(...)), the attribute forms
|
|
# builtins.open / __builtins__.open / io.open / os.open, or any of these behind a
|
|
# trailing .__call__ (open.__call__('../../etc/passwd')).
|
|
while isinstance(fn, ast.Attribute) and fn.attr == "__call__":
|
|
fn = fn.value
|
|
if isinstance(fn, ast.Name):
|
|
if fn.id == "open" or fn.id in _open_from_aliases:
|
|
return True
|
|
rhs = _unwrap_container_node(_scope_idx.resolve(fn.id, fn, "rhsnode"))
|
|
if isinstance(rhs, ast.Name) and rhs.id == "open":
|
|
return True
|
|
if (
|
|
isinstance(rhs, ast.Attribute)
|
|
and rhs.attr == "open"
|
|
and isinstance(rhs.value, ast.Name)
|
|
and rhs.value.id in _open_mod_aliases
|
|
):
|
|
return True
|
|
if (
|
|
isinstance(fn, ast.Attribute)
|
|
and fn.attr == "open"
|
|
and isinstance(fn.value, ast.Name)
|
|
and fn.value.id in _open_mod_aliases
|
|
):
|
|
return True
|
|
return False
|
|
|
|
def _is_shutil_copy_callee(fn):
|
|
while isinstance(fn, ast.Attribute) and fn.attr == "__call__":
|
|
fn = fn.value
|
|
# A single-assignment alias (c = shutil.copy; c('../../etc/passwd', 'x')) hides the
|
|
# shutil.copy attribute form behind a bare Name, so resolve the RHS before matching.
|
|
if isinstance(fn, ast.Name):
|
|
fn = _unwrap_container_node(_scope_idx.resolve(fn.id, fn, "rhsnode"))
|
|
return (
|
|
isinstance(fn, ast.Attribute)
|
|
and fn.attr in _SHUTIL_COPY_METHODS
|
|
and isinstance(fn.value, ast.Name)
|
|
and fn.value.id in _shutil_aliases
|
|
)
|
|
|
|
def _is_subprocess_exec_callee(fn):
|
|
# subprocess.run/call/check_call/check_output/Popen run an unguarded child, so a
|
|
# `..` traversal in a literal argv (subprocess.run(['cat', '../../root/.ssh/id_rsa']))
|
|
# reads a host secret. Treat these as read callees so the traversal check fires on
|
|
# their argv path elements (absolute-sensitive elements already block regardless).
|
|
while isinstance(fn, ast.Attribute) and fn.attr == "__call__":
|
|
fn = fn.value
|
|
# A from-import (from subprocess import run as r -> r([...])) or a single-assignment
|
|
# alias (r = subprocess.run) both hide the exec attribute form behind a bare Name;
|
|
# resolve/recognize them before matching the attribute form.
|
|
if isinstance(fn, ast.Name):
|
|
if fn.id in _subprocess_exec_from_aliases:
|
|
return True
|
|
fn = _unwrap_container_node(_scope_idx.resolve(fn.id, fn, "rhsnode"))
|
|
return (
|
|
isinstance(fn, ast.Attribute)
|
|
and isinstance(fn.value, ast.Name)
|
|
and fn.value.id in _subprocess_mod_aliases
|
|
and fn.attr in ("run", "call", "check_call", "check_output", "Popen")
|
|
)
|
|
|
|
def _fold_read_arg(arg):
|
|
# Fold a read-path argument to a concrete string, resolving a module-level
|
|
# constant (via _const_env) OR a function-local single-assignment string
|
|
# constant (p = '/etc/passwd' inside a def) via the scope index.
|
|
v = _const_fold(arg, _const_env)
|
|
if isinstance(v, (str, bytes, bytearray)):
|
|
return _to_text(v)
|
|
if isinstance(arg, ast.Name):
|
|
sv = _scope_idx.resolve(arg.id, arg, "strconst")
|
|
if isinstance(sv, (str, bytes, bytearray)):
|
|
return _to_text(sv)
|
|
return None
|
|
# os-aliased / from-imported path builder (o.path.join(...), join(...)) that
|
|
# _const_fold's literal-`os` owner check misses.
|
|
pj = _fold_pathjoin_call(arg)
|
|
if isinstance(pj, (str, bytes, bytearray)):
|
|
return _to_text(pj)
|
|
# A path-builder call (os.path.join(p, 'passwd'), normpath, ...) whose arguments
|
|
# include function-local single-assignment string constants stays opaque to the
|
|
# module-level _const_env. Augment the fold env with those scope-local names' RHS
|
|
# NODES and re-fold so `p = '/etc'; open(os.path.join(p, 'passwd'))` is caught.
|
|
# (_const_fold maps names to RHS nodes, not values.)
|
|
_local_env = None
|
|
for _sub in ast.walk(arg):
|
|
if isinstance(_sub, ast.Name) and isinstance(_sub.ctx, ast.Load):
|
|
if _const_env is not None and _sub.id in _const_env:
|
|
continue
|
|
_svn = _scope_idx.resolve(_sub.id, _sub, "rhsnode")
|
|
if _svn is not None:
|
|
if _local_env is None:
|
|
_local_env = dict(_const_env or {})
|
|
_local_env[_sub.id] = _svn
|
|
if _local_env is not None:
|
|
v = _const_fold(arg, _local_env)
|
|
if isinstance(v, (str, bytes, bytearray)):
|
|
return _to_text(v)
|
|
return None
|
|
|
|
def _pathlib_receiver_path(recv, _seen = None):
|
|
# Resolve a pathlib receiver to a concrete path: Path(...) / pathlib.Path(...)
|
|
# (all constructor args joined), a `/` join (Path('/etc') / 'passwd'), a
|
|
# .joinpath(...) chain, or a single-assignment name bound to any of these
|
|
# (p = Path('..') / 'etc' / 'passwd'; p.read_text()).
|
|
if isinstance(recv, ast.Name):
|
|
# Resolve the name to its single-assignment RHS (cycle-guarded).
|
|
if _seen is None:
|
|
_seen = set()
|
|
if recv.id in _seen:
|
|
return None
|
|
_seen.add(recv.id)
|
|
rhs = _scope_idx.resolve(recv.id, recv, "rhsnode")
|
|
if rhs is None:
|
|
return None
|
|
return _pathlib_receiver_path(rhs, _seen)
|
|
if isinstance(recv, ast.BinOp) and isinstance(recv.op, ast.Div):
|
|
base = _pathlib_receiver_path(recv.left, _seen)
|
|
rv = _fold_read_arg(recv.right)
|
|
if base is None or rv is None:
|
|
return None
|
|
try:
|
|
return os.path.join(base, rv)
|
|
except Exception:
|
|
return None
|
|
if not isinstance(recv, ast.Call):
|
|
return None
|
|
rf = recv.func
|
|
# No-op path-identity methods (resolve/absolute/expanduser) return the same file, so
|
|
# look through them: Path('/etc').joinpath('passwd').resolve().read_text() still
|
|
# reads /etc/passwd. expanduser() only makes a leading ~ concrete, which the
|
|
# sensitive check already handles on the pre-expansion form.
|
|
if isinstance(rf, ast.Attribute) and rf.attr in ("resolve", "absolute", "expanduser"):
|
|
return _pathlib_receiver_path(rf.value, _seen)
|
|
if isinstance(rf, ast.Attribute) and rf.attr == "joinpath":
|
|
base = _pathlib_receiver_path(rf.value, _seen)
|
|
if base is None:
|
|
return None
|
|
parts = [base]
|
|
for a in recv.args:
|
|
v = _fold_read_arg(a)
|
|
if v is None:
|
|
return None
|
|
parts.append(v)
|
|
try:
|
|
return os.path.join(*parts)
|
|
except Exception:
|
|
return None
|
|
# A single-assignment alias of the constructor (P = pathlib.Path / P = Path) is not
|
|
# in _pathlib_ctor_aliases, so resolve a Name callee's RHS to see if it binds a
|
|
# pathlib constructor before giving up.
|
|
_ctor_name = isinstance(rf, ast.Name) and rf.id in _pathlib_ctor_aliases
|
|
if not _ctor_name and isinstance(rf, ast.Name):
|
|
_crhs = _unwrap_container_node(_scope_idx.resolve(rf.id, rf, "rhsnode"))
|
|
_ctor_name = (isinstance(_crhs, ast.Name) and _crhs.id in _pathlib_ctor_aliases) or (
|
|
isinstance(_crhs, ast.Attribute) and _crhs.attr in _PATHLIB_CTORS
|
|
)
|
|
ctor = _ctor_name or (isinstance(rf, ast.Attribute) and rf.attr in _PATHLIB_CTORS)
|
|
if not ctor or not recv.args:
|
|
return None
|
|
parts = []
|
|
for a in recv.args:
|
|
v = _fold_read_arg(a)
|
|
if v is None:
|
|
return None
|
|
parts.append(v)
|
|
if not parts:
|
|
return None
|
|
try:
|
|
return os.path.join(*parts)
|
|
except Exception:
|
|
return None
|
|
|
|
def _flag_read_path(node, s, is_read_callee):
|
|
norm = s.replace("\\", "/")
|
|
# Collapse redundant separators / '.' segments and resolve '..' so equivalent
|
|
# spellings (/etc//passwd, /etc/./passwd, /tmp/../etc/passwd) still match the
|
|
# sensitive exact / dir checks. Keep the raw form for the traversal check below.
|
|
try:
|
|
canon = os.path.normpath(norm)
|
|
except Exception:
|
|
canon = norm
|
|
if _is_sensitive_abs_path(norm) or _is_sensitive_abs_path(canon):
|
|
_fs_block(node, f"{s!r} is a sensitive host identity / credential file")
|
|
return True
|
|
if is_read_callee and (s[:1] == "~" or ".." in norm.split("/")):
|
|
_fs_block(node, f"{s!r} escapes the session workdir via path traversal")
|
|
return True
|
|
return False
|
|
|
|
# Shell sinks whose first argument is always interpreted as a shell command STRING
|
|
# (os.system('cat /etc/passwd') runs an unguarded child that leaks the file in stdout).
|
|
_STRING_SHELL_SINKS = frozenset(
|
|
{
|
|
"os.system",
|
|
"os.popen",
|
|
"os.popen2",
|
|
"os.popen3",
|
|
"os.popen4",
|
|
"subprocess.getoutput",
|
|
"subprocess.getstatusoutput",
|
|
}
|
|
)
|
|
|
|
def _shell_string_sink_fq(f):
|
|
# Resolve a callee to its fq shell-sink name honoring os/subprocess module aliases
|
|
# and from-import name aliases (from subprocess import getoutput as g), else None.
|
|
if isinstance(f, ast.Attribute):
|
|
v = f.value
|
|
# os / posix / nt receiver as a simple name (posix.system) or a module that
|
|
# re-exports os as an attribute (pathlib.os.system, tempfile.os.system).
|
|
if (isinstance(v, ast.Name) and v.id in _os_mod_aliases) or (
|
|
isinstance(v, ast.Attribute) and v.attr in ("os", "posix", "nt")
|
|
):
|
|
cand = f"os.{f.attr}"
|
|
elif (isinstance(v, ast.Name) and v.id in _subprocess_mod_aliases) or (
|
|
isinstance(v, ast.Attribute) and v.attr == "subprocess"
|
|
):
|
|
cand = f"subprocess.{f.attr}"
|
|
else:
|
|
cand = None
|
|
if cand in _SHELL_EXEC_FUNCS:
|
|
return cand
|
|
elif isinstance(f, ast.Name):
|
|
return _shell_name_aliases.get(f.id)
|
|
return None
|
|
|
|
def _rewrite_methodcaller_call(node):
|
|
# operator.methodcaller('popen', 'cat /etc/passwd')(os) applies a deferred method to a
|
|
# module receiver; rewrite it to the direct os.popen('cat /etc/passwd') call so the
|
|
# read scanner tokenizes the embedded secret read. Only os/subprocess receivers are
|
|
# rewritten, so a methodcaller aimed at a benign object is left untouched.
|
|
if len(node.args) != 1 or node.keywords:
|
|
return None
|
|
mc = node.func
|
|
while isinstance(mc, ast.Attribute) and mc.attr == "__call__":
|
|
mc = mc.value
|
|
if not isinstance(mc, ast.Call) or not mc.args:
|
|
return None
|
|
mf = mc.func
|
|
is_mc = (
|
|
isinstance(mf, ast.Attribute)
|
|
and mf.attr == "methodcaller"
|
|
and isinstance(mf.value, ast.Name)
|
|
and mf.value.id in _operator_mod_aliases
|
|
) or (isinstance(mf, ast.Name) and mf.id in _methodcaller_from_aliases)
|
|
if not is_mc:
|
|
return None
|
|
meth = _fold_read_arg(mc.args[0])
|
|
if not isinstance(meth, str) or not meth.isidentifier():
|
|
return None
|
|
receiver = node.args[0]
|
|
if not (
|
|
isinstance(receiver, ast.Name)
|
|
and (receiver.id in _os_mod_aliases or receiver.id in _subprocess_mod_aliases)
|
|
):
|
|
return None
|
|
synth = ast.Call(
|
|
func = ast.Attribute(value = receiver, attr = meth, ctx = ast.Load()),
|
|
args = list(mc.args[1:]),
|
|
keywords = list(mc.keywords),
|
|
)
|
|
ast.copy_location(synth, node)
|
|
ast.fix_missing_locations(synth)
|
|
return synth
|
|
|
|
def _is_exec_family_callee(f):
|
|
# os.execv / os.execl / os.spawnv / os.posix_spawn ... replace or fork the guarded
|
|
# process with an unguarded program, so a `..` traversal in their argv reads a host
|
|
# secret the same way subprocess argv does (os.execv('/bin/cat', ['cat',
|
|
# '../../etc/shadow'])). Treat them as read callees for the traversal check.
|
|
while isinstance(f, ast.Attribute) and f.attr == "__call__":
|
|
f = f.value
|
|
fq = _shell_string_sink_fq(f)
|
|
return fq is not None and (fq.startswith("os.exec") or fq.startswith("os.spawn"))
|
|
|
|
def _iter_call_kwargs(call):
|
|
# Yield (name, value_node) for every keyword argument, EXPANDING a literal **{...}
|
|
# unpack (subprocess.run(cmd, **{'shell': True, 'cwd': p})) so a shell / cwd argument
|
|
# smuggled through a dict unpack is seen exactly like an explicit shell= / cwd= kwarg.
|
|
for _kw in call.keywords or []:
|
|
if _kw.arg is not None:
|
|
yield _kw.arg, _kw.value
|
|
elif isinstance(_kw.value, ast.Dict):
|
|
for _dk, _dv in zip(_kw.value.keys, _kw.value.values):
|
|
if (
|
|
_dk is not None
|
|
and isinstance(_dk, ast.Constant)
|
|
and isinstance(_dk.value, str)
|
|
):
|
|
yield _dk.value, _dv
|
|
|
|
def _scan_shell_string_reads(node, f):
|
|
# os.system('cat /etc/passwd') / subprocess.run('cat /etc/passwd', shell=True): the
|
|
# read scanner otherwise treats the whole command as one opaque path candidate, and
|
|
# _is_sensitive_abs_path ignores strings with whitespace. Tokenize the command and
|
|
# check each token as a read path so an embedded host-secret read is caught.
|
|
def _first_cmd_arg():
|
|
# The command may be positional OR the public `args=` keyword
|
|
# (subprocess.run(args='cat /etc/passwd', shell=True) / run(args=['cat', p])),
|
|
# including a literal **{'args': ...} unpack.
|
|
if node.args:
|
|
return node.args[0]
|
|
for _name, _val in _iter_call_kwargs(node):
|
|
if _name == "args":
|
|
return _val
|
|
return None
|
|
|
|
# subprocess.run('cat passwd', shell=True, cwd='/etc') runs the payload in an unguarded
|
|
# shell whose cwd is /etc, so a relative reader arg reads /etc/passwd; a NON-literal cwd
|
|
# cannot be proven sandbox-local. Extract cwd= once and thread it into the payload scan.
|
|
_cwd_lit = None
|
|
_cwd_dyn = False
|
|
if _is_subprocess_exec_callee(f):
|
|
for _name, _val in _iter_call_kwargs(node):
|
|
if _name == "cwd":
|
|
_cv = _fold_read_arg(_val)
|
|
if isinstance(_cv, str):
|
|
_cwd_lit = _cv
|
|
elif not (isinstance(_val, ast.Constant) and _val.value is None):
|
|
_cwd_dyn = True
|
|
break
|
|
|
|
def _scan_one_command(cmd):
|
|
# Scan a shell command STRING (folded to a literal) for an embedded host-secret
|
|
# read and record a violation. Delegates to the shared scanner in strict-traversal
|
|
# mode (the os.system() shell-string policy blocks ANY .. / ~ read path), which also
|
|
# resolves the reader past assignment / wrapper prefixes and recurses nested shells.
|
|
if cmd is None:
|
|
return False
|
|
_r = _scan_command_string_for_reads(
|
|
cmd, strict_traversal = True, cwd = _cwd_lit, cwd_dynamic = _cwd_dyn
|
|
)
|
|
if _r is not None:
|
|
_fs_block(node, _r)
|
|
return True
|
|
return False
|
|
|
|
# A subprocess argv that invokes a shell with -c runs the payload in an unguarded
|
|
# child (subprocess.run(['sh', '-c', 'head -1 /etc/passwd'])). The blocked-command
|
|
# scanner finds no blocked command (head is benign), so scan the -c payload for
|
|
# sensitive reads here the same way a string shell sink is scanned.
|
|
if _is_subprocess_exec_callee(f):
|
|
argv = _first_cmd_arg()
|
|
if isinstance(argv, (ast.List, ast.Tuple)) and argv.elts:
|
|
_elts = [_fold_read_arg(_e) for _e in argv.elts]
|
|
# Resolve the executed command word past wrapper prefixes (env / timeout /
|
|
# nice / ...): subprocess.run(['env', 'bash', '-c', payload]) runs the nested
|
|
# shell just like subprocess.run(['bash', '-c', payload]), so scan the -c
|
|
# payload regardless of the wrapper hiding argv[0].
|
|
_ci = _blocked_in_argv(_elts)[1]
|
|
_sh = _elts[_ci] if _ci is not None and _ci < len(_elts) else None
|
|
if _sh is not None and os.path.basename(_sh).lower() in _SHELL_BINARIES:
|
|
for _k in range(_ci + 1, len(_elts)):
|
|
_ev = _elts[_k]
|
|
if _ev is not None and (
|
|
_ev == "-c"
|
|
or (
|
|
_ev.startswith("-")
|
|
and not _ev.startswith("--")
|
|
and _ev.endswith("c")
|
|
)
|
|
):
|
|
if _k + 1 < len(_elts) and _scan_one_command(_elts[_k + 1]):
|
|
return True
|
|
break
|
|
|
|
_fq = _shell_string_sink_fq(f)
|
|
_is_str = _fq in _STRING_SHELL_SINKS
|
|
if not _is_str:
|
|
# subprocess.run/call/Popen/check_output/check_call(cmd, shell=True): a string
|
|
# command with shell=True runs through /bin/sh (these are in _SHELL_EXEC_FUNCS
|
|
# but not in _STRING_SHELL_SINKS, so check the shell= kwarg explicitly). Uses the
|
|
# subprocess-exec callee resolver so the attribute, from-import (from subprocess
|
|
# import run as r) and single-assignment (r = subprocess.run) forms are all seen.
|
|
if _is_subprocess_exec_callee(f):
|
|
for _name, _val in _iter_call_kwargs(node):
|
|
if _name == "shell" and not (
|
|
isinstance(_val, ast.Constant) and _val.value is False
|
|
):
|
|
_is_str = True
|
|
_cmd_node = _first_cmd_arg()
|
|
if not _is_str or _cmd_node is None:
|
|
return False
|
|
return _scan_one_command(_fold_read_arg(_cmd_node))
|
|
|
|
class _SensitiveReadVisitor(ast.NodeVisitor):
|
|
def visit_Call(self, node):
|
|
_rw = _rewrite_methodcaller_call(node)
|
|
if _rw is not None:
|
|
# methodcaller('popen', 'cat /etc/passwd')(os): analyze the direct call form.
|
|
self.visit_Call(_rw)
|
|
return
|
|
f = node.func
|
|
fq = _fq_attr_name(f)
|
|
method = (
|
|
f.attr
|
|
if isinstance(f, ast.Attribute)
|
|
else (f.id if isinstance(f, ast.Name) else "")
|
|
)
|
|
# A shell-command STRING sink: scan the command for embedded sensitive reads.
|
|
if _scan_shell_string_reads(node, f):
|
|
return
|
|
_is_child_exec = _is_subprocess_exec_callee(f) or _is_exec_family_callee(f)
|
|
is_read_callee = (
|
|
_resolves_to_open(f)
|
|
or fq in ("io.open", "os.open")
|
|
or fq in _SHUTIL_COPY_SINKS
|
|
or _is_shutil_copy_callee(f)
|
|
or (isinstance(f, ast.Name) and f.id in _shutil_copy_from_aliases)
|
|
or _is_child_exec
|
|
or method in _READ_METHODS
|
|
)
|
|
# subprocess.run(['cat', 'passwd'], cwd='/etc') reads /etc/passwd in an unguarded
|
|
# child: the argv entry is relative and /etc alone is not sensitive, so combine a
|
|
# literal cwd= with each relative argv path before the sensitivity check. A
|
|
# NON-literal cwd (cwd=P) cannot be proven sandbox-local, so a relative read under
|
|
# it fails closed (handled below).
|
|
_sub_cwd = None
|
|
_sub_cwd_dynamic = False
|
|
if _is_child_exec:
|
|
for _name, _val in _iter_call_kwargs(node):
|
|
if _name == "cwd":
|
|
_cv = _fold_read_arg(_val)
|
|
if isinstance(_cv, str):
|
|
_sub_cwd = _cv
|
|
elif not (isinstance(_val, ast.Constant) and _val.value is None):
|
|
_sub_cwd_dynamic = True
|
|
break
|
|
# An env -C DIR / --chdir=DIR at the front of the argv chdirs the child before
|
|
# the reader runs (['env', '-C', '/etc', 'cat', 'passwd']), just like the
|
|
# shell-string env -C case; fold that dir into the cwd used for relative reads.
|
|
_argv_for_cd = node.args[0] if node.args else None
|
|
if _argv_for_cd is None:
|
|
for _name, _val in _iter_call_kwargs(node):
|
|
if _name == "args":
|
|
_argv_for_cd = _val
|
|
break
|
|
if isinstance(_argv_for_cd, (ast.List, ast.Tuple)):
|
|
_ec = _argv_env_chdir([_fold_read_arg(_e) for _e in _argv_for_cd.elts])
|
|
if _ec is not None:
|
|
if _ec.startswith("/") or _ec.startswith("~"):
|
|
_sub_cwd = _ec # absolute env -C overrides the ambient cwd
|
|
_sub_cwd_dynamic = False
|
|
elif not _sub_cwd_dynamic:
|
|
_sub_cwd = _join_chdir(_sub_cwd, _ec)
|
|
# A file-reading child (cat / head / ...) with a relative argv path under a
|
|
# non-literal cwd could read a host secret (cwd=P; P evaluates to /etc); the child
|
|
# is unguarded, so fail closed unless the cwd is proven sandbox-local. The argv may
|
|
# be positional OR the public args= keyword (run(args=['cat', p], cwd=P)).
|
|
if _sub_cwd_dynamic:
|
|
_argv0 = None
|
|
_argv_node = node.args[0] if node.args else None
|
|
if _argv_node is None:
|
|
for _name, _val in _iter_call_kwargs(node):
|
|
if _name == "args":
|
|
_argv_node = _val
|
|
break
|
|
if isinstance(_argv_node, (ast.List, ast.Tuple)):
|
|
_av = _argv_node.elts
|
|
_p0 = _fold_read_arg(_av[0]) if _av else None
|
|
if (
|
|
isinstance(_p0, str)
|
|
and os.path.basename(_p0).lower() in _SHELL_READ_COMMANDS
|
|
):
|
|
for _ae in _av[1:]:
|
|
_av_s = _fold_read_arg(_ae)
|
|
if (
|
|
isinstance(_av_s, str)
|
|
and _av_s
|
|
and not _av_s.startswith("-")
|
|
and not _av_s.startswith("/")
|
|
and not _av_s.startswith("~")
|
|
):
|
|
_fs_block(
|
|
node,
|
|
"child reader with a relative path under a non-literal cwd",
|
|
)
|
|
_argv0 = True
|
|
break
|
|
if _argv0:
|
|
return
|
|
# Pathlib read on a Path(...) / join receiver: check the resolved path.
|
|
if isinstance(f, ast.Attribute) and f.attr in _PATHLIB_READ_METHODS:
|
|
rp = _pathlib_receiver_path(f.value)
|
|
if rp is not None and _flag_read_path(node, rp, True):
|
|
return
|
|
# Build the arg list, expanding a literal **{...} unpack so its path value is
|
|
# scanned (open(**{'file': '../../etc/passwd'}) reads the same file that
|
|
# open('../../etc/passwd') would, which is otherwise treated as opaque).
|
|
scan_args = list(node.args)
|
|
for kw in node.keywords or []:
|
|
if kw.arg is None and isinstance(kw.value, ast.Dict):
|
|
scan_args.extend(v for v in kw.value.values if v is not None)
|
|
else:
|
|
scan_args.append(kw.value)
|
|
# Descend into a literal list/tuple argv so a sensitive path element is
|
|
# scanned: subprocess.run(['cat', '/etc/passwd']) reads the host file in an
|
|
# unguarded child even though the top-level arg is a list, not a string.
|
|
# A literal *[...] / *(...) starred arg is unpacked positionally, so scan its
|
|
# elements too: open(*['/etc/passwd']) reads the same file open('/etc/passwd')
|
|
# would, and os.open(*['/etc/shadow', os.O_RDONLY]) is otherwise opaque.
|
|
_expanded = []
|
|
for a in scan_args:
|
|
inner = a.value if isinstance(a, ast.Starred) else a
|
|
if isinstance(inner, (ast.List, ast.Tuple)):
|
|
_expanded.extend(inner.elts)
|
|
else:
|
|
_expanded.append(inner)
|
|
scan_args = _expanded
|
|
for arg in scan_args:
|
|
s = _fold_read_arg(arg)
|
|
if s is None:
|
|
# A pathlib expression carries no foldable string constant
|
|
# (open(Path('/etc') / 'passwd')), so resolve it the same way a
|
|
# Path(...).read_text() receiver is resolved before skipping.
|
|
rp = _pathlib_receiver_path(arg)
|
|
if rp is not None and _flag_read_path(node, rp, is_read_callee):
|
|
break
|
|
# An opaque read path assembled from obfuscation primitives
|
|
# (open(''.join(map(chr, [...]))).read()) can still target a host
|
|
# secret, and reads are not runtime-confined. Apply the same
|
|
# fail-closed obfuscation policy exec payloads get: block a read
|
|
# callee whose path is built from chr/join(map)/decode/fetch/... .
|
|
if is_read_callee and _payload_has_obfuscation_primitive(arg):
|
|
_fs_block(node, "read path assembled from obfuscation primitives")
|
|
break
|
|
continue
|
|
if _flag_read_path(node, s, is_read_callee):
|
|
break
|
|
# Resolve a relative argv entry against a literal subprocess cwd= (cat passwd
|
|
# + cwd='/etc' -> /etc/passwd) so the combined host-secret read is caught.
|
|
if _sub_cwd is not None and not s.startswith("/") and not s.startswith("~"):
|
|
if _flag_read_path(node, os.path.join(_sub_cwd, s), is_read_callee):
|
|
break
|
|
self.generic_visit(node)
|
|
|
|
NetworkAndIoVisitor().visit(tree)
|
|
|
|
if _analyzer_on:
|
|
try:
|
|
_SensitiveReadVisitor().visit(tree)
|
|
except Exception: # pragma: no cover - never crashier than legacy
|
|
logger.warning("sandbox filesystem analyzer failed; skipping", exc_info = True)
|
|
filesystem_violations.clear()
|
|
|
|
is_safe = (
|
|
len(signal_tampering) == 0
|
|
and len(exception_catching) == 0
|
|
and len(shell_escapes) == 0
|
|
and len(dynamic_exec) == 0
|
|
and len(network_calls) == 0
|
|
and len(sensitive_file_reads) == 0
|
|
and len(filesystem_violations) == 0
|
|
)
|
|
return is_safe, {
|
|
"signal_tampering": signal_tampering,
|
|
"exception_catching": exception_catching,
|
|
"shell_escapes": shell_escapes,
|
|
"dynamic_exec": dynamic_exec,
|
|
"network_calls": network_calls,
|
|
"sensitive_file_reads": sensitive_file_reads,
|
|
"filesystem_violations": filesystem_violations,
|
|
"warnings": warnings,
|
|
}
|
|
|
|
|
|
def _check_code_safety(code: str) -> str | None:
|
|
"""Validate code safety via static analysis.
|
|
|
|
Returns an error message string if the code is unsafe, or None if OK.
|
|
"""
|
|
safe, info = _check_signal_escape_patterns(code)
|
|
if not safe:
|
|
# Let SyntaxError from ast.parse through so the subprocess produces a
|
|
# normal Python traceback instead of a misleading "unsafe code" message.
|
|
if info.get("error"):
|
|
return None
|
|
|
|
reasons = [item.get("description", "") for item in info.get("signal_tampering", [])]
|
|
shell_reasons = [item.get("description", "") for item in info.get("shell_escapes", [])]
|
|
exception_reasons = [
|
|
item.get("description", "") for item in info.get("exception_catching", [])
|
|
]
|
|
dynamic_reasons = [item.get("description", "") for item in info.get("dynamic_exec", [])]
|
|
network_reasons = [item.get("description", "") for item in info.get("network_calls", [])]
|
|
file_reasons = [
|
|
item.get("description", "") for item in info.get("sensitive_file_reads", [])
|
|
]
|
|
fs_reasons = [item.get("description", "") for item in info.get("filesystem_violations", [])]
|
|
all_reasons = [
|
|
r
|
|
for r in reasons
|
|
+ shell_reasons
|
|
+ exception_reasons
|
|
+ dynamic_reasons
|
|
+ network_reasons
|
|
+ file_reasons
|
|
+ fs_reasons
|
|
if r
|
|
]
|
|
if all_reasons:
|
|
return (
|
|
f"Error: unsafe code detected ({'; '.join(all_reasons)}). "
|
|
f"Please remove unsafe patterns from your code."
|
|
)
|
|
|
|
return None
|
|
|
|
|
|
def _kill_process_tree(proc) -> None:
|
|
"""SIGKILL the setsid process group; fall back to single-pid kill."""
|
|
if proc.poll() is not None:
|
|
return
|
|
try:
|
|
pgid = os.getpgid(proc.pid)
|
|
except (ProcessLookupError, PermissionError):
|
|
pgid = None
|
|
if pgid is not None:
|
|
try:
|
|
os.killpg(pgid, signal.SIGKILL)
|
|
return
|
|
except (ProcessLookupError, PermissionError):
|
|
pass
|
|
try:
|
|
proc.kill()
|
|
except (ProcessLookupError, PermissionError):
|
|
pass
|
|
|
|
|
|
def _cancel_watcher(
|
|
proc,
|
|
cancel_event,
|
|
poll_interval = 0.2,
|
|
):
|
|
"""Daemon thread that kills a process when cancel_event is set."""
|
|
while proc.poll() is None:
|
|
if cancel_event is not None and cancel_event.is_set():
|
|
_kill_process_tree(proc)
|
|
return
|
|
cancel_event.wait(poll_interval) if cancel_event else None
|
|
|
|
|
|
def _truncate(text: str, limit: int = _MAX_OUTPUT_CHARS) -> str:
|
|
if len(text) > limit:
|
|
return text[:limit] + f"\n\n... (truncated, {len(text)} chars total)"
|
|
return text
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Stage 5: runtime realpath backstop injected into sandboxed Python.
|
|
#
|
|
# This child-side guard resolves the true realpath (following symlinks) of every
|
|
# MUTATING file op and refuses it unless it lands inside the session workdir. It is
|
|
# the PRIMARY filesystem-write boundary: it sees dynamic paths, pre-existing
|
|
# symlinks, and library writers that funnel through builtins.open / io.open / os.open
|
|
# (numpy.save, torch.save, pandas.to_csv, savefig, ...). Reads are left unpatched
|
|
# here -- host-secret reads are handled by the static sensitive-read scanner. Skipped
|
|
# entirely under disable_sandbox (Bypass Permissions).
|
|
#
|
|
# Accepted residuals (a Python monkeypatch layer cannot close these; OS-level
|
|
# isolation is the real boundary): native-C writers that never call a patched Python
|
|
# entry point (cv2.imwrite, some pyarrow/zipfile writers), ctypes/cffi direct syscalls,
|
|
# and the realpath-before-open TOCTOU window under adversarial in-sandbox threading.
|
|
# --------------------------------------------------------------------------
|
|
_SANDBOX_GUARD_SRC = r"""
|
|
import sys as _sys
|
|
# The exec script lives INSIDE the workdir, so Python prepends the workdir to sys.path[0].
|
|
# A malicious workdir/os.py / io.py / pathlib.py / re.py (dropped by a prior run or upload)
|
|
# would otherwise shadow the guard's OWN imports below and execute unguarded at import time,
|
|
# before any patch is installed. Import the guard's stdlib deps with the workdir / cwd
|
|
# stripped from the path, then restore it so ordinary user imports still resolve (os / io /
|
|
# pathlib / re are now cached as the real, patched modules). `import sys` is safe: sys is a
|
|
# built-in module, never loaded from a file.
|
|
_saved_path = list(_sys.path)
|
|
_sys.path = [_p for _p in _sys.path if _p not in ("", ".", __WORKDIR__, __WORKDIR__ + "/")]
|
|
import os as _os, builtins as _bi, io as _io, pathlib as _pl, re as _re
|
|
# Pin the builtins the guard predicates consult (isinstance / int / bytes / str / any) into
|
|
# THIS namespace so a sandboxed `builtins.isinstance = lambda *a: True` (etc.) cannot make a
|
|
# guard check lie -- e.g. isinstance(path, int) treating an outside path as an fd and
|
|
# approving an absolute write. Every guard function below resolves these names from here, not
|
|
# the mutable builtins module.
|
|
isinstance = _bi.isinstance
|
|
int = _bi.int
|
|
bytes = _bi.bytes
|
|
str = _bi.str
|
|
any = _bi.any
|
|
# NOTE: sys.path stays stripped for the WHOLE guard setup below (it also imports shutil,
|
|
# which is pure-Python and equally shadowable); it is restored at the very END of this
|
|
# prelude, just before user code runs, so ordinary user imports still resolve.
|
|
# io + pathlib are imported BEFORE any patching on purpose: on Python <= 3.11
|
|
# pathlib._NormalAccessor captures io.open / os.* into class attributes at import
|
|
# time. A C builtin captured there does not bind on instance access, but a Python
|
|
# wrapper does (self shifts into the next arg), which would corrupt Path.open /
|
|
# Path.write_text. Importing here makes the accessor capture the originals; the
|
|
# confinement is applied on the public os / io / Path.* APIs below instead.
|
|
# Capture the path helpers/separator into IMMUTABLE guard-local names BEFORE user code
|
|
# runs. _within() otherwise reads os.path.realpath / os.fspath / os.sep off the live
|
|
# module every call, so sandboxed code could reassign os.path.realpath (e.g. to a lambda
|
|
# that echoes an in-workdir path) right before a write and have the guard approve an
|
|
# outside target while the real open() still writes there. These references cannot be
|
|
# rebound by mutating the os module.
|
|
_realpath = _os.path.realpath
|
|
_fspath = _os.fspath
|
|
_fsdecode = _os.fsdecode
|
|
_sep = _os.sep
|
|
# os.path.realpath resolves symlinks by consulting the LIVE os.lstat / os.readlink (and
|
|
# os.getcwd for relative paths). Capture the originals so a monkeypatch of any of them --
|
|
# e.g. os.lstat raising so realpath stops FOLLOWING an in-workdir symlink that points
|
|
# outside -- cannot make _within() approve a path the real open() then escapes through.
|
|
_lstat = _os.lstat
|
|
_readlink = _os.readlink
|
|
_getcwd = _os.getcwd
|
|
_stat = _os.stat
|
|
# posixpath.realpath decides whether to FOLLOW a component by calling os.path.stat.S_ISLNK
|
|
# on the live stat module. Sandboxed code can set os.path.stat.S_ISLNK = lambda mode: False
|
|
# (or reassign os.path.stat) so realpath stops following an in-workdir symlink that escapes,
|
|
# leaving the target under _WD while the real open() follows it outside. Capture the module +
|
|
# S_ISLNK so both can be re-pinned before each resolution.
|
|
_stat_mod = _os.path.stat
|
|
_S_ISLNK = _stat_mod.S_ISLNK
|
|
_WD = _realpath(__WORKDIR__)
|
|
# Standard device sinks cannot persist data outside the workspace, so a write to one is
|
|
# allowed (mirrors the terminal shell redirect allowlist); benign patterns like
|
|
# open('/dev/null', 'w') to suppress output would otherwise be denied by the workdir check.
|
|
_SAFE_DEV_SINKS = frozenset(
|
|
{"/dev/null", "/dev/zero", "/dev/full", "/dev/stdout", "/dev/stderr", "/dev/tty"}
|
|
)
|
|
|
|
def _within(p):
|
|
try:
|
|
if isinstance(p, int):
|
|
return True
|
|
# Allow a write to an exact device sink. Checked on the REQUESTED path, not its
|
|
# realpath, so /dev/stdout is not followed to a redirected outside file. Normalize
|
|
# via the base str.replace (not p.replace): a str subclass could override replace()
|
|
# to return "/dev/null" while its real value is an outside file, so call the genuine
|
|
# method on the underlying buffer, which yields a plain str immune to the override.
|
|
_ps = p if isinstance(p, str) else (_fsdecode(p) if isinstance(p, (bytes, bytearray)) else None)
|
|
if _ps is not None and str.replace(_ps, "\\", "/") in _SAFE_DEV_SINKS:
|
|
return True
|
|
# os.path.realpath internally calls the LIVE os.fspath (posixpath.realpath does
|
|
# `filename = os.fspath(filename)`) and os.lstat / os.readlink / os.getcwd, so a
|
|
# sandboxed reassignment of any of them would poison the resolution even though we
|
|
# hold the original realpath. Re-pin them to the captured originals before
|
|
# resolving; the target then resolves truthfully (symlinks followed, cwd honest).
|
|
# Re-pinning per check keeps it self-healing if user code re-patches; the real
|
|
# open() receives the already-materialized str and does not route through these,
|
|
# so restoring them has no effect on the write itself.
|
|
_os.fspath = _fspath
|
|
_os.lstat = _lstat
|
|
_os.readlink = _readlink
|
|
_os.getcwd = _getcwd
|
|
_os.stat = _stat
|
|
_os.path.stat = _stat_mod
|
|
_stat_mod.S_ISLNK = _S_ISLNK
|
|
rp = _realpath(_fspath(p))
|
|
# A bytes path resolves to bytes; normalize to str so the prefix compare against
|
|
# the str _WD does not raise (which would deny a legitimate in-workdir bytes write
|
|
# such as open(b'local.txt', 'w')).
|
|
if isinstance(rp, bytes):
|
|
rp = _fsdecode(rp)
|
|
except Exception:
|
|
return False
|
|
return rp == _WD or rp.startswith(_WD + _sep)
|
|
|
|
def _deny(p, what):
|
|
raise PermissionError(
|
|
"sandbox: %s outside the session workdir is not permitted: %r" % (what, p)
|
|
)
|
|
|
|
def _gwraps(real):
|
|
# Like functools.wraps but WITHOUT publishing __wrapped__: functools.wraps stores
|
|
# the ORIGINAL unguarded callable on w.__wrapped__, and sandboxed code could reach
|
|
# it (builtins.open.__wrapped__('/etc/x', 'w'), os.rename.__wrapped__(...)) to call
|
|
# straight through every confinement below. Copy only the cosmetic metadata.
|
|
def _deco(w):
|
|
for _a in ("__module__", "__name__", "__qualname__", "__doc__"):
|
|
try:
|
|
setattr(w, _a, getattr(real, _a))
|
|
except Exception:
|
|
pass
|
|
return w
|
|
return _deco
|
|
|
|
def _fspath1(p):
|
|
# Materialize a path-like ONCE so a stateful __fspath__ cannot return a workdir
|
|
# path for the _within() check and a different path for the real syscall (TOCTOU).
|
|
# Uses the captured _fspath so a reassigned os.fspath cannot interpose here.
|
|
if isinstance(p, int):
|
|
return p
|
|
try:
|
|
return _fspath(p)
|
|
except Exception:
|
|
return p
|
|
|
|
def _mode_is_write(mode):
|
|
# Coerce through the *base* str: a str-subclass __contains__/__str__ must not be
|
|
# able to lie about whether the mode requests a write.
|
|
m = str.__str__(mode) if isinstance(mode, str) else "r"
|
|
return any(c in m for c in "wax+")
|
|
|
|
# Runtime sensitive-read backstop. The static scanner cannot fold every read path
|
|
# (open(globals()['x']), open(fetch_name()), open(''.join(...))), and reads are otherwise
|
|
# unconfined, so an opaque path could name a host secret. Deny a read whose REALPATH
|
|
# resolves to a known-sensitive host file OUTSIDE the workdir. In-workdir files are the
|
|
# sandbox's own and always allowed. The loose 'credentials' / '.pem' / '/root/' signals the
|
|
# static layer uses are intentionally NOT applied here: importing common libraries reads
|
|
# site-packages files such as google/auth/credentials.py and certifi/cacert.pem (and, under
|
|
# a root home, /root/.local/.../site-packages), so matching them at runtime would break
|
|
# imports. The specific SSH / cloud / kube / netrc / HF-token signals stay.
|
|
_SENS_EXACT = frozenset({
|
|
"/etc/passwd", "/etc/shadow", "/etc/sudoers", "/etc/gshadow", "/etc/master.passwd",
|
|
})
|
|
_SENS_DIRS = (
|
|
"/etc/ssh/", "/.ssh/", "/.aws/", "/.config/gcloud", "/.kube/", "/.docker/",
|
|
"/var/run/secrets/kubernetes.io/", "/run/secrets/kubernetes.io/",
|
|
)
|
|
_SENS_TOKENS = (
|
|
"id_rsa", "id_ed25519", ".netrc", ".git-credentials", "/.huggingface/token", ".kube/config",
|
|
)
|
|
_SENS_PROC = _re.compile(r"^/proc/(?:self|\d+)/(?:environ|cmdline|maps|mem|task/\d+/environ)$")
|
|
|
|
def _is_sensitive_read(rp):
|
|
n = rp.replace("\\", "/")
|
|
if n in _SENS_EXACT:
|
|
return True
|
|
# _SENS_DIRS entries carry a trailing slash to match a file UNDER the dir
|
|
# (/root/.ssh/id_rsa). Append one to n so the sensitive directory ITSELF
|
|
# (os.listdir('/root/.ssh') -> '/root/.ssh', no trailing slash) matches too.
|
|
if any(part in (n + "/") for part in _SENS_DIRS):
|
|
return True
|
|
if _SENS_PROC.match(n):
|
|
return True
|
|
# Dotfiles / caches under a root home hold credentials (/root/.bashrc, /root/.cache/...);
|
|
# an opaque path the static /root/ rule cannot fold could read them at runtime. Restore
|
|
# the /root/ protection here (including the root home ITSELF, /root, which a directory
|
|
# reader would enumerate), but carve out package / library trees so importing a library
|
|
# installed under a root home (site-packages, the stdlib) is not broken.
|
|
if (n == "/root" or n.startswith("/root/")) and not any(
|
|
_seg in n
|
|
for _seg in ("/site-packages/", "/dist-packages/", "/lib/python", "/lib64/python")
|
|
):
|
|
return True
|
|
low = n.lower()
|
|
return any(tok in low for tok in _SENS_TOKENS)
|
|
|
|
def _read_realpath(p):
|
|
# Resolve to a truthful realpath the same self-healing way _within does, so a
|
|
# sandboxed reassignment of os.fspath / os.lstat / os.readlink / os.getcwd cannot
|
|
# poison the resolution.
|
|
try:
|
|
_os.fspath = _fspath
|
|
_os.lstat = _lstat
|
|
_os.readlink = _readlink
|
|
_os.getcwd = _getcwd
|
|
_os.stat = _stat
|
|
_os.path.stat = _stat_mod
|
|
_stat_mod.S_ISLNK = _S_ISLNK
|
|
rp = _realpath(_fspath(p))
|
|
if isinstance(rp, bytes):
|
|
rp = _fsdecode(rp)
|
|
return rp
|
|
except Exception:
|
|
return None
|
|
|
|
def _deny_sensitive_read(p):
|
|
if isinstance(p, int):
|
|
return
|
|
rp = _read_realpath(p)
|
|
if rp is None:
|
|
return
|
|
# In-workdir files are the sandbox's own; never treat them as host secrets.
|
|
if rp == _WD or rp.startswith(_WD + _sep):
|
|
return
|
|
if _is_sensitive_read(rp):
|
|
raise PermissionError(
|
|
"sandbox: reading a sensitive host path is not permitted: %r" % (rp,)
|
|
)
|
|
|
|
def _guard_open_like(real):
|
|
@_gwraps(real)
|
|
def w(file, mode="r", *a, **k):
|
|
f = _fspath1(file)
|
|
if _mode_is_write(mode):
|
|
if not _within(f):
|
|
_deny(f, "write")
|
|
else:
|
|
_deny_sensitive_read(f)
|
|
return real(f, mode, *a, **k)
|
|
return w
|
|
|
|
_bi.open = _guard_open_like(_bi.open)
|
|
|
|
# Low-level os.open: builtins.open does not route through it, so it needs its own
|
|
# guard. Any mutating open flag confines the target; ANY dir_fd call (read or write) fails
|
|
# closed -- a string realpath against cwd is wrong for an fd-relative path, and a read-only
|
|
# dir_fd open can still read a host file under a directory fd opened outside the workdir
|
|
# (d = os.open('/etc', O_RDONLY); os.open('passwd', O_RDONLY, dir_fd=d)).
|
|
_WRITE_OFLAGS = (
|
|
getattr(_os, "O_WRONLY", 0) | getattr(_os, "O_RDWR", 0)
|
|
| getattr(_os, "O_CREAT", 0) | getattr(_os, "O_TRUNC", 0) | getattr(_os, "O_APPEND", 0)
|
|
)
|
|
def _make_osopen_guard(real_open):
|
|
@_gwraps(real_open)
|
|
def _guarded(path, flags, *a, **k):
|
|
if k.get("dir_fd") is not None:
|
|
_deny(path, "os.open (dir_fd)")
|
|
try:
|
|
fi = int.__index__(flags) # base int: an int-subclass __and__ must not lie
|
|
except Exception:
|
|
fi = None
|
|
mutating = (fi is None) or bool(fi & _WRITE_OFLAGS)
|
|
if mutating:
|
|
p = _fspath1(path)
|
|
if not _within(p):
|
|
_deny(p, "os.open write")
|
|
return real_open(p, flags, *a, **k)
|
|
# Read-only os.open: reads are unconfined, but a host secret is still off limits.
|
|
p = _fspath1(path)
|
|
_deny_sensitive_read(p)
|
|
return real_open(p, flags, *a, **k)
|
|
return _guarded
|
|
_os.open = _make_osopen_guard(_os.open)
|
|
|
|
def _wrap1(mod, name, what):
|
|
orig = getattr(mod, name, None)
|
|
if orig is None:
|
|
return
|
|
@_gwraps(orig)
|
|
def w(*a, **k):
|
|
# The path may be positional OR a public keyword: os.mkdir(path=...),
|
|
# os.makedirs(name=...), os.removedirs(name=...). Extract it from whichever slot it
|
|
# arrived in so a keyword call is not broken (missing positional) while still confined.
|
|
_pk = None
|
|
if a:
|
|
path = a[0]
|
|
elif "path" in k:
|
|
path, _pk = k["path"], "path"
|
|
elif "name" in k:
|
|
path, _pk = k["name"], "name"
|
|
else:
|
|
return orig(*a, **k) # let the original raise its own TypeError
|
|
if any(k.get(_f) is not None for _f in ("dir_fd", "src_dir_fd", "dst_dir_fd")):
|
|
_deny(path, what + " (dir_fd)") # fd-relative target: a realpath check is meaningless
|
|
if isinstance(path, int):
|
|
# A mutating op given an fd (os.chmod(fd), os.truncate(fd), ...) can hit a
|
|
# file opened read-only outside the workdir; a string realpath cannot
|
|
# confine an fd, so deny it (fchmod/fchown are already denied separately).
|
|
_deny(path, what + " (fd)")
|
|
p = _fspath1(path)
|
|
if not _within(p):
|
|
_deny(p, what)
|
|
# Pass the MATERIALIZED path back in the same slot it arrived (a stateful __fspath__
|
|
# cannot then return a different outside path to the real call).
|
|
if a:
|
|
return orig(p, *a[1:], **k)
|
|
k = dict(k)
|
|
k[_pk] = p
|
|
return orig(*a, **k)
|
|
setattr(mod, name, w)
|
|
|
|
# Path-first single-arg mutators. mkfifo/utime/setxattr/removexattr create or mutate
|
|
# host files/metadata; the platform-specific ones no-op via _wrap1 when absent.
|
|
_OS_MUTATORS1 = (
|
|
"remove", "unlink", "rmdir", "removedirs", "truncate", "chmod", "chown",
|
|
"mkdir", "makedirs", "mknod", "mkfifo", "utime", "setxattr", "removexattr",
|
|
"lchmod", "lchown", "chflags", "lchflags",
|
|
)
|
|
for _n in _OS_MUTATORS1:
|
|
_wrap1(_os, _n, _n)
|
|
|
|
# Directory readers (os.listdir / os.scandir) enumerate a directory's names/entries
|
|
# WITHOUT routing through open(), so a sensitive host directory whose path is opaque to
|
|
# the static scanner (P = globals()['P']; os.listdir(P)) would leak its contents past the
|
|
# open-like backstop. Apply the same sensitive-read check to the directory path. A bare
|
|
# call (cwd), in-workdir paths, and an fd argument (os.open already screens the fd's read)
|
|
# stay allowed.
|
|
def _guard_dir_reader(name, mod=_os):
|
|
orig = getattr(mod, name, None)
|
|
if orig is None:
|
|
return
|
|
@_gwraps(orig)
|
|
def w(path=".", *a, **k):
|
|
if isinstance(path, int):
|
|
return orig(path, *a, **k)
|
|
# Materialize the path ONCE and pass that same value to the real call, so a stateful
|
|
# __fspath__ cannot return an in-workdir path for the check and a sensitive outside
|
|
# directory for the real listdir/scandir.
|
|
p = _fspath1(path)
|
|
_deny_sensitive_read(p)
|
|
return orig(p, *a, **k)
|
|
setattr(mod, name, w)
|
|
|
|
for _n in ("listdir", "scandir"):
|
|
_guard_dir_reader(_n)
|
|
|
|
def _wrap2(mod, name, both):
|
|
orig = getattr(mod, name, None)
|
|
if orig is None:
|
|
return
|
|
@_gwraps(orig)
|
|
def w(src, dst, *a, **k):
|
|
if any(k.get(_f) is not None for _f in ("dir_fd", "src_dir_fd", "dst_dir_fd")):
|
|
_deny(dst, name + " (dir_fd)") # fd-relative target: a realpath check is meaningless
|
|
s, d = _fspath1(src), _fspath1(dst)
|
|
if both and not _within(s):
|
|
_deny(s, name + " source")
|
|
if not _within(d):
|
|
_deny(d, name + " destination")
|
|
return orig(s, d, *a, **k)
|
|
setattr(mod, name, w)
|
|
|
|
for _n in ("rename", "renames", "replace", "link", "symlink"):
|
|
_wrap2(_os, _n, True)
|
|
|
|
# posix (POSIX) / nt (Windows) is the low-level C module os re-exports from; patching
|
|
# os.* leaves posix.open / posix.rename / ... importable with the originals, so apply
|
|
# the same guards to that module too.
|
|
for _lowosname in ("posix", "nt"):
|
|
try:
|
|
_lowos = __import__(_lowosname)
|
|
except Exception:
|
|
_lowos = None
|
|
if _lowos is not None:
|
|
try:
|
|
if hasattr(_lowos, "open"):
|
|
_lowos.open = _make_osopen_guard(_lowos.open)
|
|
for _n in _OS_MUTATORS1:
|
|
_wrap1(_lowos, _n, _lowosname + "." + _n)
|
|
for _n in ("rename", "renames", "replace", "link", "symlink"):
|
|
_wrap2(_lowos, _n, True)
|
|
except Exception:
|
|
pass
|
|
|
|
# io.open is a separate reference from the (now patched) builtins.open -- guard
|
|
# direct io.open() writers (e.g. zipfile-based) the same way. (Path.open is handled
|
|
# explicitly below, not via this patch.)
|
|
try:
|
|
_io.open = _guard_open_like(_io.open)
|
|
except Exception:
|
|
pass
|
|
|
|
# The low-level C module _io is where io.open / builtins.open originate; patching the
|
|
# io alias above leaves _io.open untouched, so `import _io; _io.open(p, 'w')` would
|
|
# escape. Guard the underlying entry point too.
|
|
try:
|
|
import _io as _lowio
|
|
_lowio.open = _guard_open_like(_lowio.open)
|
|
except Exception:
|
|
_lowio = None
|
|
|
|
# io.FileIO / _io.FileIO is a C constructor that opens a file WITHOUT routing through
|
|
# open(), so `io.FileIO('/tmp/escape', 'w')` bypasses _guard_open_like. Subclass it to
|
|
# confine mutating modes (subclassing keeps guard-built objects real FileIO instances).
|
|
def _guard_fileio(_realcls):
|
|
class _GuardedFileIO(_realcls):
|
|
def __init__(self, name, mode="r", *a, **k):
|
|
f = _fspath1(name)
|
|
if _mode_is_write(mode):
|
|
if not _within(f):
|
|
_deny(f, "FileIO write")
|
|
else:
|
|
_deny_sensitive_read(f)
|
|
# Pass the MATERIALIZED path so a stateful __fspath__ cannot return a
|
|
# different (outside) path to the real constructor than we checked.
|
|
super().__init__(f, mode, *a, **k)
|
|
return _GuardedFileIO
|
|
|
|
for _iomod in (_io, _lowio):
|
|
try:
|
|
if _iomod is not None and hasattr(_iomod, "FileIO"):
|
|
_iomod.FileIO = _guard_fileio(_iomod.FileIO)
|
|
except Exception:
|
|
pass
|
|
|
|
# The guards above patch the EXISTING posix / nt / _io module objects, but sandboxed code
|
|
# can mint a FRESH copy with the original unwrapped C functions via
|
|
# _imp.create_builtin(posix.__spec__) (or the BuiltinImporter path) and call its open()
|
|
# directly. Wrap _imp.create_builtin / create_dynamic so a freshly created guard-relevant
|
|
# module (posix / nt with os.open-style open + mutators, _io / io with open + FileIO) gets
|
|
# the same wrappers re-applied before it is handed back. Other builtin modules carry no file
|
|
# primitives, so they pass through unchanged and ordinary lazy imports keep working.
|
|
def _reguard_created(m):
|
|
try:
|
|
_nm = getattr(m, "__name__", "") or ""
|
|
except Exception:
|
|
return m
|
|
try:
|
|
if _nm in ("posix", "nt"):
|
|
if hasattr(m, "open"):
|
|
m.open = _make_osopen_guard(m.open)
|
|
for _rn in _OS_MUTATORS1:
|
|
_wrap1(m, _rn, _nm + "." + _rn)
|
|
for _rn in ("rename", "renames", "replace", "link", "symlink"):
|
|
_wrap2(m, _rn, True)
|
|
# A fresh posix/nt module also re-exposes the ORIGINAL fd metadata mutators and
|
|
# directory readers; reapply the same fd deniers + read confinement applied to the
|
|
# already-loaded module (else fresh fchmod(fd, ...) / fresh listdir('/root') slip).
|
|
if hasattr(m, "chdir"):
|
|
_wrap1(m, "chdir", _nm + ".chdir")
|
|
for _rn in ("fchmod", "fchown"):
|
|
if hasattr(m, _rn):
|
|
setattr(m, _rn, _make_fd_denier(_nm + "." + _rn, getattr(m, _rn)))
|
|
for _rn in ("listdir", "scandir"):
|
|
if hasattr(m, _rn):
|
|
_guard_dir_reader(_rn, m)
|
|
elif _nm in ("_io", "io"):
|
|
if hasattr(m, "open"):
|
|
m.open = _guard_open_like(m.open)
|
|
if hasattr(m, "FileIO"):
|
|
m.FileIO = _guard_fileio(m.FileIO)
|
|
except Exception:
|
|
pass
|
|
return m
|
|
|
|
try:
|
|
import _imp as _lowimp
|
|
|
|
def _guard_create(_orig):
|
|
@_gwraps(_orig)
|
|
def w(spec, *a, **k):
|
|
return _reguard_created(_orig(spec, *a, **k))
|
|
return w
|
|
|
|
for _cn in ("create_builtin", "create_dynamic"):
|
|
_co = getattr(_lowimp, _cn, None)
|
|
if _co is not None:
|
|
setattr(_lowimp, _cn, _guard_create(_co))
|
|
except Exception:
|
|
pass
|
|
|
|
# Confine the current working directory: os.chdir to a dir outside the workdir would
|
|
# let a later relative write/read (which the static read scan treats as local) escape.
|
|
# os.fchdir takes an fd whose target we cannot cheaply realpath, so deny it outright.
|
|
_wrap1(_os, "chdir", "chdir")
|
|
try:
|
|
_real_fchdir = _os.fchdir
|
|
@_gwraps(_real_fchdir)
|
|
def _guarded_fchdir(fd):
|
|
_deny(fd, "fchdir")
|
|
_os.fchdir = _guarded_fchdir
|
|
except Exception:
|
|
pass
|
|
|
|
# fd-based metadata mutators operate on an already-open descriptor, so a read-only
|
|
# os.open of an outside file (allowed -- reads are not confined) could still be reused
|
|
# to mutate host metadata. Deny them; sandboxed compute has no need to chmod/chown by fd.
|
|
def _make_fd_denier(_name, _orig):
|
|
@_gwraps(_orig)
|
|
def _w(fd, *a, **k):
|
|
_deny(fd, _name)
|
|
return _w
|
|
for _n in ("fchmod", "fchown"):
|
|
try:
|
|
setattr(_os, _n, _make_fd_denier(_n, getattr(_os, _n)))
|
|
except Exception:
|
|
pass
|
|
|
|
# The low-level posix / nt modules re-export chdir / fchdir / fchmod / fchown with the
|
|
# ORIGINALS, so patching os.* leaves posix.chdir (cwd escape -> unconfined relative
|
|
# reads) and posix.fchmod / posix.fchown (host-metadata mutation on a read-only outside
|
|
# fd) reachable. Apply the same confinement / deniers to those module objects too.
|
|
for _lowosname in ("posix", "nt"):
|
|
try:
|
|
_lowos = __import__(_lowosname)
|
|
except Exception:
|
|
_lowos = None
|
|
if _lowos is None:
|
|
continue
|
|
try:
|
|
if hasattr(_lowos, "chdir"):
|
|
_wrap1(_lowos, "chdir", _lowosname + ".chdir")
|
|
if hasattr(_lowos, "fchdir"):
|
|
_lowos.fchdir = _make_fd_denier(_lowosname + ".fchdir", _lowos.fchdir)
|
|
for _n in ("fchmod", "fchown"):
|
|
if hasattr(_lowos, _n):
|
|
setattr(_lowos, _n, _make_fd_denier(_lowosname + "." + _n, getattr(_lowos, _n)))
|
|
# posix.listdir / posix.scandir re-export the ORIGINAL enumerators, so the os.* dir
|
|
# guard leaves them reachable (posix.listdir('/root')); apply the same sensitive-read
|
|
# confinement to the low-level module.
|
|
for _n in ("listdir", "scandir"):
|
|
if hasattr(_lowos, _n):
|
|
_guard_dir_reader(_n, _lowos)
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
import shutil as _sh
|
|
_wrap1(_sh, "rmtree", "rmtree")
|
|
_wrap1(_sh, "chown", "chown")
|
|
_wrap2(_sh, "move", True)
|
|
for _n in ("copy", "copy2", "copyfile", "copytree", "copymode", "copystat"):
|
|
_wrap2(_sh, _n, False)
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
# Path.open("w"): wrap the public method directly (mode-aware). Version-robust
|
|
# because pathlib's accessor holds the original io.open (captured at the top).
|
|
_real_path_open = _pl.Path.open
|
|
@_gwraps(_real_path_open)
|
|
def _guarded_path_open(self, mode="r", *a, **k):
|
|
# Coerce mode through the base str (a str-subclass __contains__ must not lie).
|
|
if _mode_is_write(mode):
|
|
if not _within(self):
|
|
_deny(str(self), "Path.open")
|
|
else:
|
|
# A dynamically assembled Path (Path(globals()['P']).read_text()) has no literal
|
|
# receiver for the static scanner. On Python <= 3.11 pathlib holds the ORIGINAL
|
|
# io.open, so the io.open sensitive-read backstop would not fire for pathlib
|
|
# reads; apply it here so Path reads are confined version-robustly. read_text /
|
|
# read_bytes route through this same self.open(), so they are covered too.
|
|
_deny_sensitive_read(self)
|
|
return _real_path_open(self, mode, *a, **k)
|
|
_pl.Path.open = _guarded_path_open
|
|
|
|
def _wrapp(name, targ):
|
|
orig = getattr(_pl.Path, name, None)
|
|
if orig is None:
|
|
return
|
|
@_gwraps(orig)
|
|
def w(self, *a, **k):
|
|
if not _within(self):
|
|
_deny(str(self), "Path." + name)
|
|
if targ:
|
|
# rename/replace/symlink_to/hardlink_to accept the target as the
|
|
# `target=` keyword too; on Python <= 3.10 pathlib routes through the
|
|
# accessor's ORIGINAL os.rename, so this wrapper is the only
|
|
# confinement -- check the keyword as well as the positional arg.
|
|
_t = a[0] if a else k.get("target")
|
|
if _t is not None:
|
|
# Materialize the target once so a stateful __fspath__ cannot return
|
|
# an in-workdir path here and an outside one to the real call.
|
|
_tm = _fspath1(_t)
|
|
if not _within(_tm):
|
|
_deny(str(_tm), "Path." + name + " target")
|
|
if a:
|
|
a = (_tm,) + tuple(a[1:])
|
|
else:
|
|
k = dict(k)
|
|
k["target"] = _tm
|
|
return orig(self, *a, **k)
|
|
setattr(_pl.Path, name, w)
|
|
for _n in ("write_text", "write_bytes", "unlink", "mkdir", "rmdir", "chmod", "touch"):
|
|
_wrapp(_n, False)
|
|
for _n in ("rename", "replace", "symlink_to", "hardlink_to"):
|
|
_wrapp(_n, True)
|
|
|
|
# Path.iterdir / glob / rglob enumerate a directory; a dynamically built receiver
|
|
# (Path(globals()['P']).iterdir(), Path(P).glob('*')) has no literal path for the static
|
|
# scanner and, on some CPython versions, routes through pathlib's captured original
|
|
# os.scandir rather than the patched one, so screen the directory read on the RECEIVER dir.
|
|
def _guard_path_dir_reader(_name):
|
|
_real = getattr(_pl.Path, _name, None)
|
|
if _real is None:
|
|
return
|
|
@_gwraps(_real)
|
|
def w(self, *a, **k):
|
|
_deny_sensitive_read(self)
|
|
return _real(self, *a, **k)
|
|
setattr(_pl.Path, _name, w)
|
|
for _n in ("iterdir", "glob", "rglob"):
|
|
_guard_path_dir_reader(_n)
|
|
except Exception:
|
|
pass
|
|
|
|
# All guard dependencies are now imported (and cached as the real, patched modules) with the
|
|
# workdir kept off sys.path, so no workdir/*.py could shadow them. Restore the original path
|
|
# for user code so ordinary sibling imports still resolve.
|
|
_sys.path = _saved_path
|
|
"""
|
|
|
|
|
|
def _sandbox_runtime_prelude(workdir: str) -> str:
|
|
"""One physical line that runs the realpath backstop before the user code.
|
|
|
|
The guard executes in its own namespace (helper names never leak into user
|
|
globals) while its monkeypatches persist on the os/shutil/pathlib/builtins
|
|
module objects. Emitting it on a single line keeps user traceback line numbers
|
|
shifted by exactly one."""
|
|
src = _SANDBOX_GUARD_SRC.replace("__WORKDIR__", repr(workdir))
|
|
return (
|
|
"exec(compile(%r, '<studio-sandbox-guard>', 'exec'), {'__builtins__': __builtins__})\n"
|
|
% src
|
|
)
|
|
|
|
|
|
def _inject_sandbox_guard(code: str, prelude: str) -> str:
|
|
"""Splice the runtime guard into ``code`` without displacing leading directives.
|
|
|
|
``from __future__`` imports must be the first statement of a module (only a
|
|
docstring and comments may precede them), so blindly prepending the guard line
|
|
turns any user program that opens with a future import into a SyntaxError. Parse
|
|
the code, keep a leading module docstring and any ``from __future__`` imports on
|
|
top, and insert the (inert compile-time) guard immediately after them -- it still
|
|
runs before the first real user statement, so the sandbox is established before
|
|
any file operation. Everything else (including unparsable code, where we want the
|
|
natural SyntaxError traceback) falls back to a plain prepend.
|
|
"""
|
|
try:
|
|
tree = ast.parse(code)
|
|
except SyntaxError:
|
|
return prelude + code
|
|
body = getattr(tree, "body", [])
|
|
idx = 0
|
|
split = 0
|
|
if (
|
|
body
|
|
and isinstance(body[0], ast.Expr)
|
|
and isinstance(getattr(body[0], "value", None), ast.Constant)
|
|
and isinstance(body[0].value.value, str)
|
|
):
|
|
idx = 1
|
|
split = body[0].end_lineno or 0
|
|
has_future = False
|
|
while (
|
|
idx < len(body)
|
|
and isinstance(body[idx], ast.ImportFrom)
|
|
and body[idx].module == "__future__"
|
|
):
|
|
has_future = True
|
|
split = body[idx].end_lineno or split
|
|
idx += 1
|
|
if not has_future or split <= 0:
|
|
return prelude + code
|
|
lines = code.splitlines(keepends = True)
|
|
head = "".join(lines[:split])
|
|
tail = "".join(lines[split:])
|
|
if head and not head.endswith(("\n", "\r")):
|
|
head += "\n"
|
|
return head + prelude + tail
|
|
|
|
|
|
def _python_exec(
|
|
code: str,
|
|
cancel_event = None,
|
|
timeout: int = _EXEC_TIMEOUT,
|
|
session_id: str | None = None,
|
|
disable_sandbox: bool = False,
|
|
) -> str:
|
|
"""Execute Python code in a subprocess sandbox.
|
|
|
|
disable_sandbox (Bypass Permissions): skip the safety analysis and rlimit
|
|
pre-exec, and use the host env minus secrets.
|
|
"""
|
|
if not code or not code.strip():
|
|
return "No code provided."
|
|
|
|
# Validate imports and code safety (skipped when the sandbox is disabled)
|
|
if not disable_sandbox:
|
|
error = _check_code_safety(code)
|
|
if error:
|
|
return error
|
|
elif not _harden_parent_against_proc_env_leak():
|
|
# Close the /proc/<parent>/environ secret-recovery path first; if it
|
|
# cannot be applied, fail closed rather than leak the parent environ.
|
|
return (
|
|
"Execution error: could not harden the Studio process against "
|
|
"/proc environment reads; refusing bypass execution."
|
|
)
|
|
|
|
tmp_path = None
|
|
workdir = _get_workdir(session_id)
|
|
# Snapshot image mtimes to detect new and overwritten files.
|
|
_before: dict[str, int] = {}
|
|
if os.path.isdir(workdir):
|
|
for _name in os.listdir(workdir):
|
|
if os.path.splitext(_name)[1].lower() in _IMAGE_EXTS:
|
|
_p = os.path.join(workdir, _name)
|
|
if os.path.isfile(_p):
|
|
try:
|
|
_before[_name] = os.stat(_p).st_mtime_ns
|
|
except OSError:
|
|
pass
|
|
try:
|
|
fd, tmp_path = tempfile.mkstemp(suffix = ".py", prefix = "studio_exec_", dir = workdir)
|
|
# utf-8 so non-ASCII in model-written code survives the OS default codec
|
|
# (Windows cp1252 would otherwise raise UnicodeEncodeError).
|
|
# Sandboxed runs get the realpath backstop prepended (Stage 5); bypass
|
|
# runs execute the code verbatim.
|
|
file_body = (
|
|
code
|
|
if disable_sandbox
|
|
else _inject_sandbox_guard(code, _sandbox_runtime_prelude(workdir))
|
|
)
|
|
with os.fdopen(fd, "w", encoding = "utf-8") as f:
|
|
f.write(file_body)
|
|
|
|
safe_env = _build_bypass_env(workdir) if disable_sandbox else _build_safe_env(workdir)
|
|
if disable_sandbox:
|
|
# Match the sandboxed Python path without changing bypass shell I/O.
|
|
safe_env = dict(safe_env)
|
|
safe_env["PYTHONIOENCODING"] = "utf-8"
|
|
popen_kwargs = dict(
|
|
stdout = subprocess.PIPE,
|
|
stderr = subprocess.STDOUT,
|
|
text = True,
|
|
# Decode child output as utf-8 (it emits utf-8 via PYTHONIOENCODING);
|
|
# replace so non-ASCII output never crashes the read on Windows.
|
|
encoding = "utf-8",
|
|
errors = "replace",
|
|
cwd = workdir,
|
|
env = safe_env,
|
|
)
|
|
if sys.platform != "win32":
|
|
popen_kwargs["preexec_fn"] = _bypass_preexec if disable_sandbox else _sandbox_preexec
|
|
else:
|
|
popen_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
|
|
|
|
# -s disables the per-user site directory (~/.local/.../site-packages): the guard is
|
|
# injected into the SCRIPT body and runs after site initialization, so a prior run that
|
|
# dropped .local/.../site-packages/usercustomize.py (HOME points at the workdir) would
|
|
# otherwise import it during startup and run writers with unpatched stdlib. Bypass keeps
|
|
# the host default. The real site-packages stays available for user imports.
|
|
_py_argv = (
|
|
[sys.executable, tmp_path] if disable_sandbox else [sys.executable, "-s", tmp_path]
|
|
)
|
|
proc = subprocess.Popen(_py_argv, **popen_kwargs)
|
|
|
|
# Spawn cancel watcher if we have a cancel event
|
|
if cancel_event is not None:
|
|
watcher = threading.Thread(
|
|
target = _cancel_watcher, args = (proc, cancel_event), daemon = True
|
|
)
|
|
watcher.start()
|
|
|
|
try:
|
|
output, _ = proc.communicate(timeout = timeout)
|
|
except subprocess.TimeoutExpired:
|
|
_kill_process_tree(proc)
|
|
try:
|
|
proc.communicate(timeout = 5)
|
|
except subprocess.TimeoutExpired:
|
|
pass
|
|
return _truncate(f"Execution timed out after {timeout} seconds.")
|
|
|
|
if cancel_event is not None and cancel_event.is_set():
|
|
return "Execution cancelled."
|
|
|
|
result = output or ""
|
|
if proc.returncode != 0:
|
|
result = f"Exit code {proc.returncode}:\n{result}"
|
|
result = _truncate(result) if result.strip() else "(no output)"
|
|
|
|
# Detect new/overwritten images and append sentinel for the frontend
|
|
if session_id and os.path.isdir(workdir):
|
|
new_images = []
|
|
for _name in os.listdir(workdir):
|
|
if os.path.splitext(_name)[1].lower() not in _IMAGE_EXTS:
|
|
continue
|
|
_p = os.path.join(workdir, _name)
|
|
if not os.path.isfile(_p):
|
|
continue
|
|
try:
|
|
_mtime = os.stat(_p).st_mtime_ns
|
|
except OSError:
|
|
continue
|
|
if _name not in _before or _mtime != _before[_name]:
|
|
new_images.append(_name)
|
|
if new_images:
|
|
import json as _json
|
|
result += f"\n__IMAGES__:{_json.dumps(sorted(new_images))}"
|
|
|
|
return result
|
|
|
|
except Exception as e:
|
|
return f"Execution error: {e}"
|
|
finally:
|
|
if tmp_path and os.path.exists(tmp_path):
|
|
try:
|
|
os.unlink(tmp_path)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _bash_exec(
|
|
command: str,
|
|
cancel_event = None,
|
|
timeout: int = _EXEC_TIMEOUT,
|
|
session_id: str | None = None,
|
|
disable_sandbox: bool = False,
|
|
) -> str:
|
|
"""Execute a bash command in a subprocess sandbox.
|
|
|
|
disable_sandbox (Bypass Permissions): skip the command blocklist and rlimit
|
|
pre-exec, and use the host env minus secrets.
|
|
"""
|
|
if not command or not command.strip():
|
|
return "No command provided."
|
|
|
|
# Block dangerous commands (skipped when the sandbox is disabled)
|
|
if not disable_sandbox:
|
|
blocked = _find_blocked_commands(command)
|
|
if blocked:
|
|
return f"Blocked command(s) for safety: {', '.join(sorted(blocked))}"
|
|
# The command runs in an unguarded shell child, so the Python-tool open() backstop
|
|
# does not confine its reads; refuse an embedded host-secret read the same way an
|
|
# os.system('cat /etc/passwd') shell string is refused in the Python tool.
|
|
_read = _command_reads_sensitive(command)
|
|
if _read is not None:
|
|
return f"Blocked command for safety: sensitive file read ({_read})"
|
|
elif not _harden_parent_against_proc_env_leak():
|
|
# Close the /proc/<parent>/environ secret-recovery path first; if it
|
|
# cannot be applied, fail closed rather than leak the parent environ.
|
|
return (
|
|
"Execution error: could not harden the Studio process against "
|
|
"/proc environment reads; refusing bypass execution."
|
|
)
|
|
|
|
try:
|
|
workdir = _get_workdir(session_id)
|
|
safe_env = _build_bypass_env(workdir) if disable_sandbox else _build_safe_env(workdir)
|
|
popen_kwargs = dict(
|
|
stdout = subprocess.PIPE,
|
|
stderr = subprocess.STDOUT,
|
|
text = True,
|
|
cwd = workdir,
|
|
env = safe_env,
|
|
)
|
|
if sys.platform != "win32":
|
|
popen_kwargs["preexec_fn"] = _bypass_preexec if disable_sandbox else _sandbox_preexec
|
|
else:
|
|
popen_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
|
|
|
|
proc = subprocess.Popen(_get_shell_cmd(command), **popen_kwargs)
|
|
|
|
if cancel_event is not None:
|
|
watcher = threading.Thread(
|
|
target = _cancel_watcher, args = (proc, cancel_event), daemon = True
|
|
)
|
|
watcher.start()
|
|
|
|
try:
|
|
output, _ = proc.communicate(timeout = timeout)
|
|
except subprocess.TimeoutExpired:
|
|
_kill_process_tree(proc)
|
|
try:
|
|
proc.communicate(timeout = 5)
|
|
except subprocess.TimeoutExpired:
|
|
pass
|
|
return _truncate(f"Execution timed out after {timeout} seconds.")
|
|
|
|
if cancel_event is not None and cancel_event.is_set():
|
|
return "Execution cancelled."
|
|
|
|
result = output or ""
|
|
if proc.returncode != 0:
|
|
result = f"Exit code {proc.returncode}:\n{result}"
|
|
return _truncate(result) if result.strip() else "(no output)"
|
|
|
|
except Exception as e:
|
|
return f"Execution error: {e}"
|