ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Match web-fetch MIME subtypes exactly and detect control-char binary

Browse source
This commit is contained in:
oobabooga 2026-07-14 19:25:15 -03:00
commit 75dfba2eae
2 changed files with 55 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

50
studio/backend/core/inference/tools.py
View file

@ -1381,12 +1381,14 @@ _MAX_PAGE_CHARS = 16000 # cap fetched page text (after HTML-to-MD conversion)
# sections stripped during conversion; 512 KB reaches article content even
# where <head> alone is ~200 KB.
_MAX_FETCH_BYTES = 512 * 1024
# A decoded page is treated as binary when more than 1/_BINARY_REPLACEMENT_DIVISOR
# (12.5%) of its chars are U+FFFD, but always tolerating up to
# _MIN_REPLACEMENT_CHARS stray bad bytes so minor encoding glitches don't drop a
# real page.
_MIN_REPLACEMENT_CHARS = 16
_BINARY_REPLACEMENT_DIVISOR = 8
# Chars that don't occur in real text: undecodable bytes (U+FFFD) plus control
# chars (C0 minus tab/newline/CR and ESC, DEL, C1). ESC is excluded so a text
# page of ANSI-colored terminal output isn't mistaken for binary. A decoded page
# is treated as binary when more than 1/_BINARY_CHAR_DIVISOR (12.5%) of its chars
# are these, tolerating up to _MIN_BINARY_CHARS so minor glitches don't drop it.
_BINARY_CHAR_RE = re.compile("[\\x00-\\x08\\x0b\\x0c\\x0e-\\x1a\\x1c-\\x1f\\x7f-\\x9f\\ufffd]")
_MIN_BINARY_CHARS = 16
_BINARY_CHAR_DIVISOR = 8
_USER_AGENTS = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
@ -1488,23 +1490,33 @@ def _validate_and_resolve_host(hostname: str, port: int) -> tuple[bool, str, str
return True, "", first_ip
def _is_texty_content_type(content_type: str) -> bool:
# Bare application/* subtypes that are text (the leading x- and the +json/+xml
# structured-syntax suffixes are handled in the check).
_TEXT_APPLICATION_SUBTYPES = frozenset(
{"json", "xml", "javascript", "ecmascript", "csv", "yaml", "ndjson", "jsonl"}
)
def _is_texty_content_type(content_type: str | None) -> bool:
"""True for MIME types safe to decode as text (HTML pages, plain text,
JSON/XML feeds). Binary types (PDF, images, archives, octet-stream) return
False so the fetcher never decodes them into a flood of U+FFFD replacement
chars that poison the model context (unslothai/unsloth#7084).
JSON/XML/YAML feeds). Binary types (PDF, images, archives, Office/ZIP,
octet-stream) return False so the fetcher never decodes them into a flood of
replacement chars that poison the model context (unslothai/unsloth#7084).
A missing Content-Type coerces to ``text/plain`` upstream, so unlabeled
bodies pass here and are caught instead by the replacement-char fallback.
bodies pass here and are caught instead by the binary-char fallback.
"""
ct = (content_type or "").lower()
if not ct:
return True # unlabeled: let the replacement-char fallback decide
return True # unlabeled: let the binary-char fallback decide
if ct.startswith("text/"):
return True
if ct.startswith("application/"):
# "xml" also covers xhtml+xml / *+xml; "json" covers *+json.
return any(marker in ct for marker in ("json", "xml", "javascript", "csv"))
# Match the exact subtype, not a loose substring, so a .docx labeled
# application/vnd.openxmlformats-... isn't read as xml. RFC 6839
# +json/+xml suffixes (ld+json, xhtml+xml, ...) pass too.
subtype = ct[len("application/") :].removeprefix("x-")
return subtype in _TEXT_APPLICATION_SUBTYPES or subtype.endswith(("+json", "+xml"))
return False
@ -1604,11 +1616,11 @@ def _fetch_page_text(
charset = resp.headers.get_content_charset() or "utf-8"
raw_html = raw_bytes.decode(charset, errors = "replace")
# Fallback for binary mislabeled as text/* or sent with no Content-Type:
# a real text page has only a few replacement chars, if any.
if raw_html.count("\ufffd") > max(
_MIN_REPLACEMENT_CHARS, len(raw_html) // _BINARY_REPLACEMENT_DIVISOR
):
# Fallback for binary mislabeled as text/* or sent with no Content-Type,
# including valid-UTF-8 binary (NUL/control-heavy payloads) that decodes
# without replacement chars: a real text page has few binary chars.
binary_chars = len(_BINARY_CHAR_RE.findall(raw_html))
if binary_chars > max(_MIN_BINARY_CHARS, len(raw_html) // _BINARY_CHAR_DIVISOR):
return f"(binary content, {len(raw_bytes)} bytes; not readable as text)"
except _HTTPError as e:
return f"Failed to fetch URL: HTTP {e.code} {getattr(e, 'reason', '')}"

26
studio/backend/tests/test_web_fetch_binary_guard.py
View file

@ -71,12 +71,18 @@ def _fetch_with(monkeypatch, body: bytes, content_type: str | None) -> str:
("application/xml", True),
("application/xhtml+xml", True),
("application/ld+json", True),
("application/yaml", True), # raw configs / API specs
("application/x-yaml", True),
("application/x-ndjson", True), # newline-delimited JSON is text
("application/ndjson", True),
("application/pdf", False),
("image/png", False),
("image/svg+xml", False), # SVG source isn't extracted downstream; reject
("application/octet-stream", False),
("application/zip", False),
("", True), # unlabeled: defer to the replacement-char fallback
# A .docx ZIP must not pass just because "xml" is inside "openxmlformats".
("application/vnd.openxmlformats-officedocument.wordprocessingml.document", False),
("", True), # unlabeled: defer to the binary-char fallback
(None, True),
],
)
@ -101,12 +107,28 @@ def test_image_rejected_by_content_type(monkeypatch):
def test_binary_mislabeled_as_text_caught_by_fallback(monkeypatch):
# A server sends binary but labels it text/plain -> the type check passes,
# so the replacement-char fallback must catch it.
# so the binary-char fallback must catch it.
out = _fetch_with(monkeypatch, bytes(range(256)) * 20, "text/plain")
assert "<EFBFBD>" not in out
assert "binary content" in out
def test_valid_utf8_binary_caught_by_control_chars(monkeypatch):
# NUL/control-heavy binary is valid UTF-8, so it decodes with no U+FFFD;
# the fallback must still catch it via control-char density (codex #2).
body = bytes([0, 1, 2, 3, 4, 5, 6, 7]) * 400 # all valid UTF-8, 0 replacement chars
out = _fetch_with(monkeypatch, body, "text/plain")
assert "binary content" in out
def test_ansi_colored_text_log_kept(monkeypatch):
# A text log with per-token ANSI color codes is text; ESC is excluded from
# the binary-char set so it isn't dropped as binary.
line = "".join(f"\x1b[32m+{i}\x1b[0m\n" for i in range(300)).encode()
out = _fetch_with(monkeypatch, line, "text/plain")
assert "binary content" not in out
def test_binary_unlabeled_caught_by_fallback(monkeypatch):
# No Content-Type coerces to text/plain upstream; the fallback still catches it.
out = _fetch_with(monkeypatch, bytes(range(256)) * 20, None)