ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Close five ways a sed program hid its shell payload

Browse source 
Review found five shapes the first pass missed. All five execute on GNU
sed 4.9, checked by running them rather than reading the manual.

A payload line ending in a backslash continues onto the next line, so the
scan now ends an `e` at an unescaped newline and unescapes the text the way
sed's read_text does. That is what resolves `r''m` back to `rm` for the
blocklist.

A sed comment ends at a real newline, but the terminal scan had already
replaced every newline with `;`, including newlines inside quotes, so
`# comment` swallowed the rest of the program. The sed arm now also sees a
variant where only unquoted newlines become separators, built on a
character-by-character quote scanner rather than a regex: an apostrophe in
a double-quoted word mis-pairs under a regex and inverts the state, which
opened a bypass while this was being written.

Everything attached to `-i` is a backup suffix, so reading `-ifoo` as an
attached `-f` lost the real script. Replaced the shared short-flag helper
with sed's own option grammar, which also fixes `-l 5` and
`--line-length 5` eating the script as their operand.

A sed child of `find -exec` was never recorded, so the blocklist skipped
its payload.

Substituted text splices straight into the program, and an address is as
good a place as any to open `;e CMD`, so a command substitution anywhere
in the program is treated as unresolvable. Scoped to the program: a
substitution in a file operand still runs, a `$(` or backtick inside single
quotes is literal, and parameter and arithmetic expansion are untouched.
The cost is that a substitution used to build a program now asks.

Bounding the -exec walk keeps the blocklist linear; without it a repeated
`-exec sed` line went quadratic.

Verified against real GNU sed across 103 commands run for real, no
mismatch in either direction.
This commit is contained in:
danielhanchen 2026-07-27 06:34:58 +00:00
commit 9a5cfddb4e
3 changed files with 340 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

255
studio/backend/core/inference/tools.py
View file

@ -274,6 +274,22 @@ _SED_COMMANDS = frozenset({"sed", "gsed", "ssed"})
# `s///` flags that may precede `e`. `w` is absent: it takes the rest of the
# line as a filename, so the e in `s/a/b/w report.txt` is part of that name.
_SED_SUBST_FLAGS = frozenset("0123456789gpiImMe")
# sed short options that consume text, so the rest of their token belongs to
# them and no later letter in the cluster is a flag: -e/-f take a script and
# -l a line length (attached or as the next token), while -i's backup suffix
# is ATTACHED ONLY. Reading the suffix as more flags hid the real script --
# `-ifoo` looked like an attached `-f oo`, and `-l 5` ate the script as the
# length operand.
_SED_VALUE_FLAGS = "efl"
_SED_ATTACHED_VALUE_FLAGS = "i"
# A backslash in a sed text argument escapes the next character, newline
# included, so it is stripped before the payload is read as a shell command.
_SED_TEXT_ESCAPE_RE = re.compile(r"\\([\s\S])")
# A sed script always sits among the leading options or as the first positional;
# everything past that is input files. Bounding the walk keeps a command padded
# with hundreds of `-exec sed` words linear, since each one would otherwise
# rescan the whole token list.
_MAX_SED_ARG_SCAN = 128
_WIN_CONDITIONAL_KEYWORDS = frozenset({"exist", "defined", "errorlevel", "not"})
_FIND_EXEC_FLAGS = frozenset({"-exec", "-execdir", "-ok", "-okdir"})
@ -297,6 +313,36 @@ def _blocked_matching_glob(base: str) -> "set[str]":
return {name for name in _BLOCKED_COMMANDS if fnmatch.fnmatchcase(name, base)}
def _sed_short_flag(token: str) -> "tuple[str, str] | None":
"""The first value-taking short option in a sed flag cluster, as
``(letter, text glued after it)``. The scan stops there because everything
left in the token is that option's value: `-ifoo` is -i with backup suffix
"foo", not an attached -f. ``None`` for a long option or a plain cluster."""
if not token.startswith("-") or token.startswith("--"):
return None
for index, ch in enumerate(token[1:]):
if ch in _SED_VALUE_FLAGS or ch in _SED_ATTACHED_VALUE_FLAGS:
return ch, token[index + 2 :]
return None
def _sed_long_flag(name: str) -> str:
"""Which value-taking sed long option ``--name`` is: "e" for --expression,
"f" for --file, "l" for --line-length, "" for anything else. getopt allows
unambiguous abbreviations, so --e/--ex/--expr are all --expression and --fi
upwards is --file (--f alone is ambiguous with --follow-symlinks, which sed
rejects). --in-place is absent: its suffix is attached, never a token."""
if len(name) <= 2:
return ""
if "--expression".startswith(name):
return "e"
if len(name) > 3 and "--file".startswith(name):
return "f"
if "--line-length".startswith(name):
return "l"
return ""
def _sed_program(tokens: "list[str]", start: int) -> str:
"""The sed script of the invocation whose command word sits at ``start``.
@ -309,8 +355,8 @@ def _sed_program(tokens: "list[str]", start: int) -> str:
programs: "list[str]" = []
first_positional = ""
has_program_flag = False
value_pending = "" # "e" or "f": the next token is that flag's value
for token in tokens[start + 1 :]:
value_pending = "" # "e", "f" or "l": the next token is that flag's value
for token in tokens[start + 1 : start + 1 + _MAX_SED_ARG_SCAN]:
if token in _SHELL_SEPARATORS:
break
if value_pending:
@ -320,25 +366,27 @@ def _sed_program(tokens: "list[str]", start: int) -> str:
continue
if token.startswith("--"):
name, sep, value = token.partition("=")
# getopt allows unambiguous abbreviations: --e/--ex/--expr are all
# --expression, and --fi upwards is --file (--f alone is ambiguous
# with --follow-symlinks, which sed rejects).
is_expression = len(name) > 2 and "--expression".startswith(name)
is_file = len(name) > 3 and "--file".startswith(name)
if is_expression or is_file:
has_program_flag = True
if not sep:
value_pending = "e" if is_expression else "f"
elif is_expression:
programs.append(value)
letter = _sed_long_flag(name)
if not letter:
continue
# -l only matters so its operand is not mistaken for the script.
has_program_flag = has_program_flag or letter in "ef"
if not sep:
value_pending = letter
elif letter == "e":
programs.append(value)
continue
if token.startswith("-"):
# A cluster glues the value on (-ne'1p') or takes the next (-ne '1p').
attached = _short_flag_arg(token, "ef")
if attached is None:
found = _sed_short_flag(token)
if found is None:
continue
has_program_flag = True
letter = next(ch for ch in token[1:] if ch in "ef")
letter, attached = found
if letter in _SED_ATTACHED_VALUE_FLAGS:
# -i's suffix is the rest of the token; it never takes the next
# one, so the script is still the positional ahead.
continue
has_program_flag = has_program_flag or letter in "ef"
if not attached:
value_pending = letter
elif letter == "e":
@ -351,6 +399,13 @@ def _sed_program(tokens: "list[str]", start: int) -> str:
return "\n".join(programs)
def _sed_text(text: str) -> str:
"""Unescape one sed text argument the way read_text does: every backslash
drops away and the character behind it stays, so `e touch MARK\\ER` runs
MARKER and `e\\` + newline runs the next line as its own command."""
return _SED_TEXT_ESCAPE_RE.sub(r"\1", text).strip()
def _sed_exec_payloads(program: str) -> "list[str]":
"""Shell payloads a sed program executes, in order.
@ -369,6 +424,14 @@ def _sed_exec_payloads(program: str) -> "list[str]":
end = program.find("\n", pos)
return n if end < 0 else end
def _end_of_text(pos: int) -> int:
# read_text, which collects `e`/`a`/`i`/`c` text: a backslash escapes
# the next character, so a line ending in one carries the text onto the
# NEXT line instead of stopping there.
while pos < n and program[pos] != "\n":
pos += 2 if program[pos] == "\\" else 1
return min(pos, n)
def _skip_bracket(pos: int) -> int:
# A bracket expression, where the delimiter is data (`s/[/]/x/` really
# substitutes a slash). A leading `]` is literal; [:class:] nests.
@ -444,9 +507,11 @@ def _sed_exec_payloads(program: str) -> "list[str]":
break
cmd, i = program[i], i + 1
if cmd == "e":
# The payload ends at the NEWLINE, so a `;` inside it is shell text.
end = _end_of_line(i)
payloads.append(program[i:end].strip())
# The payload ends at an UNESCAPED newline, so a `;` inside it is
# shell text and `e\` + newline hands the next line to the same
# shell (`1e\` / `rm -f victim` really runs rm).
end = _end_of_text(i)
payloads.append(_sed_text(program[i:end]))
i = end
elif cmd in "sy" and i < n:
delim, i = program[i], i + 1
@ -463,8 +528,7 @@ def _sed_exec_payloads(program: str) -> "list[str]":
i = _end_of_line(i)
elif cmd in "aic":
# Literal text; the `a\` + newline form continues on a trailing "\".
while i < n and program[i] != "\n":
i += 2 if program[i] == "\\" else 1
i = _end_of_text(i)
elif cmd in "rRwW":
i = _end_of_line(i) # the filename runs to the end of the line
elif cmd in "btT:v":
@ -591,6 +655,11 @@ def _find_blocked_commands(command: str) -> set[str]:
blocked |= _blocked_matching_glob(attached_base)
if tok in _FIND_EXEC_FLAGS and i + 1 < len(tokens):
base = _token_basename(tokens[i + 1])
if base in _SED_COMMANDS:
# find runs its -exec child directly, but the walk above only
# reaches `find`, so a sed there never got its program screened
# (`find . -exec sed '1e rm -f victim' {} +`).
sed_indexes.append(i + 1)
if base in _BLOCKED_COMMANDS:
blocked.add(base)
else:
@ -3784,6 +3853,103 @@ def _short_flag_arg(token: str, letters: str) -> "str | None":
return None
def _shell_quote_states(command: str) -> "list[str]":
"""The quote context of every character: ``""`` outside quoting, ``"'"``
(or ``"$'"`` for ANSI-C, which honours backslash escapes) inside single
quoting, and ``'"'`` inside double quoting. A quote mark itself reports the
context it opens from, so a character is text bash expands exactly when its
state is ``""`` or ``'"'``.
Tracked character by character rather than paired off with a regex, because
a regex matches the apostrophe in `echo "it's"` against the next quote,
inverting the state for everything after it.
"""
states: "list[str]" = []
quote = ""
i, n = 0, len(command)
while i < n:
ch = command[i]
if quote in ("'", "$'"):
# A plain single quote protects even backslashes; ANSI-C does not,
# so `\'` there is a quote character rather than the end of the word.
if quote == "$'" and ch == "\\" and i + 1 < n:
states += [quote, quote]
i += 2
continue
states.append(quote)
if ch == "'":
quote = ""
i += 1
continue
if ch == "\\" and i + 1 < n:
states += [quote, quote] # the next character is data, never syntax
i += 2
continue
states.append(quote)
if quote == '"':
# Only the closing quote ends it; an apostrophe here is text.
if ch == '"':
quote = ""
elif ch == "'":
quote = "$'" if i and command[i - 1] == "$" else "'"
elif ch == '"':
quote = '"'
i += 1
return states
def _live_command_substitutions(command: str) -> "list[str]":
"""Each `$(...)` / backtick substitution the shell actually RUNS, as the
exact text it occupies. Single-quoted ones are literal, so
`sed 's/`//g' NOTES.md` yields nothing. `$((` is arithmetic, not a
substitution, matching _HAS_COMMAND_SUBST_RE."""
found: "list[str]" = []
states = _shell_quote_states(command)
i, n = 0, len(command)
while i < n:
if states[i] not in ("", '"'):
i += 1
continue
if command[i] == "`":
end = command.find("`", i + 1)
end = n if end < 0 else end + 1
found.append(command[i:end])
i = end
continue
if command.startswith("$(", i) and not command.startswith("$((", i):
depth, end = 0, i + 1
while end < n:
if command[end] == "(":
depth += 1
elif command[end] == ")":
depth -= 1
if depth == 0:
end += 1
break
end += 1
found.append(command[i:end])
i = end
continue
i += 1
return found
def _separate_unquoted_newlines(text: str) -> str:
"""``text`` with each UNQUOTED newline replaced by `;`, which shlex reads as
a command boundary. A newline inside quotes is DATA -- a sed comment ends at
one -- so it survives, unlike a blanket replacement."""
states = _shell_quote_states(text)
out = []
for i, ch in enumerate(text):
if ch in "\r\n" and states[i] == "":
# \r\n is one boundary, not two.
if not (ch == "\n" and i and text[i - 1] == "\r"):
out.append(";")
else:
out.append(ch)
return "".join(out)
# git subcommands that discard or overwrite work: `clean` deletes untracked files,
# `restore` overwrites the worktree from the index/HEAD, `rm` deletes tracked
# files, and the plumbing entries delete refs/reflogs/objects or rewrite history.
@ -4109,12 +4275,23 @@ def _terminal_is_high_risk(command: str, _depth: int = 0) -> bool:
return True
# Newlines separate commands in a shell but read as whitespace to shlex, and
# ANSI-C quoting ($'rm') hides the real command name.
normalized = (
_decode_ansi_c(command, keep_one_word = True)
.replace("\r\n", ";")
.replace("\n", ";")
.replace("\r", ";")
decoded = _decode_ansi_c(command, keep_one_word = True)
normalized = decoded.replace("\r\n", ";").replace("\n", ";").replace("\r", ";")
# Identical to the blanket form unless a newline is actually present, so the
# usual single-line command never pays for the quote walk.
quoted_newlines_kept = (
_separate_unquoted_newlines(decoded) if "\n" in decoded or "\r" in decoded else normalized
)
# Matched against a sed program below to tell a script the shell generated
# from one that merely quotes a `$(`. Held in both newline forms so the
# match works whichever pass produced the tokens.
live_substitutions: "set[str]" = set()
if _HAS_COMMAND_SUBST_RE.search(command):
live_substitutions = {
form
for sub in _live_command_substitutions(command)
for form in (sub, sub.replace("\r\n", ";").replace("\n", ";").replace("\r", ";"))
}
# A verb hidden behind an assignment (c=rm; $c x) or a default parameter
# (${c:-rm}) is expanded so the resolved token is scanned too.
expanded = _expand_shell_assignments(_expand_param_defaults(normalized))
@ -4138,7 +4315,15 @@ def _terminal_is_high_risk(command: str, _depth: int = 0) -> bool:
# the check above misses it. A benign array print is untouched.
if _ARRAY_EXPANSION_RE.search(command) and _VAR_EXECUTED_AS_COMMAND_RE.search(command):
return True
for text in {normalized, expanded}:
# A newline inside a QUOTED argument is data, not a separator, and turning
# it into `;` rewrites that data: a sed comment ends at a real newline, so
# `sed '# note<newline>e CMD'` reads as one long comment once the newline is
# gone. So a pass that only separates the UNQUOTED ones is scanned too. It
# keeps every command boundary the blanket form has, so the token stream is
# the same and only quoted content differs: the pass adds detections without
# merging two commands into one segment. The set collapses to a single scan
# for the usual single-line command.
for text in {normalized, expanded, quoted_newlines_kept}:
try:
lexer = shlex.shlex(text, posix = True, punctuation_chars = ";&|()")
lexer.whitespace_split = True
@ -4521,10 +4706,20 @@ def _terminal_is_high_risk(command: str, _depth: int = 0) -> bool:
chdir_pending = True
if base in _AWK_COMMANDS:
awk_program_pending = True
if base in _SED_COMMANDS and _sed_exec_payloads(_sed_program(tokens, _tok_idx)):
if base in _SED_COMMANDS:
# `e` / `s///e` shell out from inside the script, which may
# ride on -e/--expression rather than the next positional.
return True
sed_program = _sed_program(tokens, _tok_idx)
if _sed_exec_payloads(sed_program):
return True
# A script the shell GENERATES is not knowable here: sed
# splices the output straight into the program text, where
# it can open `;e CMD` from any position, so it asks. The
# substitution has to land in the PROGRAM: one that only
# feeds file operands (sed -n 1p $(ls)) still runs, as does
# a `$(` the program merely quotes (sed 's/$(CC)/gcc/').
if any(sub in sed_program for sub in live_substitutions):
return True
elif current_command == "git" and not git_subcommand:
# The first positional after `git` is its subcommand.
git_subcommand = base

76
studio/backend/tests/test_permission_mode.py
View file

@ -929,6 +929,82 @@ def test_terminal_classifier(command, unsafe):
("sed -e '1a\\' -e 'echo appended' f", False), # a\ continues into -e
("echo \"sed '1e rm -f victim'\"", False),
("printf '%s' sed '1e rm -f victim'", False),
# --- prompt: an `e` payload ending in a backslash continues onto the
# NEXT line, which sed hands to the same shell ---
("sed -n '1e\\\nrm -f victim' f", True),
("sed -n '1e touch a\\\nrm -f victim' f", True),
("sed 'e r\\m -f victim' f", True), # the backslash drops, rm still runs
("sed -e 'e\\' -e 'rm -f victim' f", True),
# --- prompt: a sed comment ends at a real NEWLINE, not at a `;`, so an
# `e` on the line after one is a command, not comment text ---
("sed '# harmless\ne rm -f victim' input", True),
("sed '#c1\n#c2\ne rm -f victim' input", True),
("sed 's/a/b/w out.txt\ne rm -f victim' input", True), # w name ends too
("sed '1r notes.txt\ne rm -f victim' input", True),
("sed '1a hello\ne rm -f victim' input", True),
("sed '# harmless;e rm -f victim' input", False), # one long comment
("sed '# harmless\np' input", False),
# --- prompt: everything glued to -i is the backup SUFFIX, so the script
# is still the positional ahead; likewise -l/--line-length take an
# operand that is not the script ---
("sed -ifoo '1e rm -f victim' input", True),
("sed -itemp '1e rm -f victim' input", True),
("sed -ni.bak '1e rm -f victim' input", True),
("sed -ieBAK -e 'e rm -f victim' input", True),
("sed -l 5 '1e rm -f victim' input", True),
("sed -l5 '1e rm -f victim' input", True),
("sed -le 'e rm -f victim' input", True),
("sed --line-length 5 '1e rm -f victim' input", True),
("sed --l 5 '1e rm -f victim' input", True),
("sed --in-place=foo '1e rm -f victim' input", True),
("sed -i.bak 's/x/y/' f", False),
("sed -ifoo 's/x/y/' f", False),
("sed -l 80 's/x/y/' f", False),
("sed --line-length=80 -n '1,20p' f", False),
# --- prompt: sed under find -exec / xargs runs for real ---
("find . -exec sed '1e rm -f victim' {} +", True),
("find . -execdir sed '1e rm -f victim' {} \\;", True),
("xargs sed '1e rm -f victim'", True),
("find . -exec sed -n '1,3p' {} +", False),
("find . -exec sed -i.bak 's/a/b/' {} +", False),
# --- prompt: a program the SHELL generates is not knowable here, since
# sed splices the output into the script text ---
("sed \"$(printf 'e rm -f victim')\" input", True),
('sed "$(cat prog.sed)" input', True),
('sed -n "1,$(wc -l < f)p" f', True), # bounded cost of failing closed
# a substitution outside the program, and a literal `$(`/backtick inside
# single quotes, are not a generated program
("sed -n '1,3p' $(ls)", False),
("sed 's/`//g' NOTES.md", False),
("sed 's/$(x)/y/' f", False),
# an apostrophe inside a DOUBLE-quoted word must not be paired with the
# next quote: doing so hid a real generated program, and mis-read a
# single-quoted one as generated
('echo "it\'s"; sed "$(printf \'e rm -f victim\')" f', True),
('echo "it\'s"; sed "$(printf \'e rm -f x\')" f; echo "that\'s"', True),
("echo \"don't\" && sed 's/$(x)/y/' f", False),
("echo \"don't\" && sed 's/`//g' NOTES.md", False),
# `\'` inside ANSI-C quoting is a quote character, not the end of the
# word, so the tracker must not invert from there on
("sed -e $'s/\\'\\'/X/' -e \"$(cat prog.sed)\" f", True),
# the substitution has to reach the PROGRAM: one that only builds file
# operands leaves a program the scan can still read in full
("sed -i 's/$(CC)/gcc/' $(git ls-files '*.mk')", False),
("sed 's/`//g' $(ls *.md)", False),
# --- run: a newline BETWEEN commands still separates them, so the
# segment-scoped checks must not read the next line's words as
# arguments of this one ---
("git checkout main\nls", False),
("git checkout main\nnpm test", False),
("git checkout -b feature\ngit status", False),
("git checkout v1.0\npython3 setup.py build", False),
("export PATH=/usr/local/bin:$PATH\nmake", False),
("IFS=,\nread a b c", False),
("cd build\nmake -j4", False),
("git checkout HEAD notes.txt\nls", True), # still a real pathspec
('sed "s/$old/$new/g" f', False),
('sed -n "1,${n}p" f', False),
('sed -n "1,$((n + 1))p" f', False), # arithmetic, not a substitution
# --- prompt: setpriv execs what follows, after changing privilege ---
("setpriv --nnp rm -f victim", True),
("setpriv --reuid=1000 rm -rf build", True),

39
studio/backend/tests/test_sandbox_tools.py
View file

@ -645,6 +645,43 @@ class TestBashBlocklistPosition:
assert "rm" in self._find()("sed -ne '$e rm -rf build' input")
assert "wget" in self._find()("sed '1,2e wget https://bad' input")
def test_sed_exec_payload_continues_past_backslash(self):
# An `e` payload whose line ends in a backslash carries onto the NEXT
# line, which reaches the same shell, so the scan must not stop at the
# newline. Quote splitting (r''m) hides the name from the raw-text
# fallback, leaving the parsed payload as the only place rm shows up.
assert "rm" in self._find()("sed -n '1e\\\nrm -f victim' f")
assert "rm" in self._find()("sed -n '1e\\\nr''m -f victim' f")
assert "rm" in self._find()("sed -n '1e touch a\\\nrm -f victim' f")
# A backslash before an ordinary character drops away: r\m runs rm.
assert "rm" in self._find()("sed 'e r\\m -f victim' f")
def test_sed_comment_ends_at_newline(self):
# A sed comment runs to a real newline, so an `e` on the line after one
# is a command; with a literal `;` it is still all comment.
assert "rm" in self._find()("sed '# harmless\ne rm -f victim' input")
assert "curl" in self._find()("sed 's/a/b/w out.txt\ne curl https://x' input")
assert self._find()("sed '# harmless;e rm -f victim' input") == set()
def test_sed_attached_i_suffix_does_not_hide_the_script(self):
# Everything glued to -i is the backup suffix, so `-ifoo` is not an
# attached -f and the script is still the positional ahead. -l and
# --line-length take an operand that is likewise not the script.
assert "rm" in self._find()("sed -ifoo '1e rm -f victim' input")
assert "rm" in self._find()("sed -itemp '1e rm -f victim' input")
assert "curl" in self._find()("sed -ni.bak '1e curl https://x' input")
assert "rm" in self._find()("sed -l 5 '1e rm -f victim' input")
assert "rm" in self._find()("sed --line-length 5 '1e rm -f victim' input")
assert self._find()("sed -ifoo 's/old/new/g' input") == set()
assert self._find()("sed -l 80 -n '1,20p' input") == set()
def test_sed_under_find_exec_blocked(self):
# find runs its -exec child directly, but the command-position walk only
# reaches `find`, so the nested sed needs its script read explicitly.
assert "rm" in self._find()("find . -exec sed '1e rm -f victim' {} +")
assert "curl" in self._find()("find . -execdir sed '1e curl https://x' {} \\;")
assert self._find()("find . -exec sed -n '1,3p' {} +") == set()
def test_ordinary_sed_program_allowed(self):
# Plain stream editing runs nothing, and a mention of sed in argument
# position is text: only a command-position sed has its script read.
@ -652,6 +689,8 @@ class TestBashBlocklistPosition:
assert self._find()("sed -n '1,20p' input") == set()
assert self._find()("sed 's/rm/RM/g' input") == set()
assert self._find()("printf '%s' sed '1e rm -rf victim'") == set()
assert self._find()("sed 's/a/b/we out.txt' input") == set()
assert self._find()("sed -e '1a\\' -e 'e rm -rf x' input") == set()
def test_subshell_command_blocked(self):
assert "rm" in self._find()("echo $(rm -rf /tmp)")