diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 11e7328b6f..16dce26b55 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -1381,12 +1381,14 @@ _MAX_PAGE_CHARS = 16000 # cap fetched page text (after HTML-to-MD conversion) # sections stripped during conversion; 512 KB reaches article content even # where alone is ~200 KB. _MAX_FETCH_BYTES = 512 * 1024 -# A decoded page is treated as binary when more than 1/_BINARY_REPLACEMENT_DIVISOR -# (12.5%) of its chars are U+FFFD, but always tolerating up to -# _MIN_REPLACEMENT_CHARS stray bad bytes so minor encoding glitches don't drop a -# real page. -_MIN_REPLACEMENT_CHARS = 16 -_BINARY_REPLACEMENT_DIVISOR = 8 +# Chars that don't occur in real text: undecodable bytes (U+FFFD) plus control +# chars (C0 minus tab/newline/CR and ESC, DEL, C1). ESC is excluded so a text +# page of ANSI-colored terminal output isn't mistaken for binary. A decoded page +# is treated as binary when more than 1/_BINARY_CHAR_DIVISOR (12.5%) of its chars +# are these, tolerating up to _MIN_BINARY_CHARS so minor glitches don't drop it. +_BINARY_CHAR_RE = re.compile("[\\x00-\\x08\\x0b\\x0c\\x0e-\\x1a\\x1c-\\x1f\\x7f-\\x9f\\ufffd]") +_MIN_BINARY_CHARS = 16 +_BINARY_CHAR_DIVISOR = 8 _USER_AGENTS = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", @@ -1488,23 +1490,33 @@ def _validate_and_resolve_host(hostname: str, port: int) -> tuple[bool, str, str return True, "", first_ip -def _is_texty_content_type(content_type: str) -> bool: +# Bare application/* subtypes that are text (the leading x- and the +json/+xml +# structured-syntax suffixes are handled in the check). +_TEXT_APPLICATION_SUBTYPES = frozenset( + {"json", "xml", "javascript", "ecmascript", "csv", "yaml", "ndjson", "jsonl"} +) + + +def _is_texty_content_type(content_type: str | None) -> bool: """True for MIME types safe to decode as text (HTML pages, plain text, - JSON/XML feeds). Binary types (PDF, images, archives, octet-stream) return - False so the fetcher never decodes them into a flood of U+FFFD replacement - chars that poison the model context (unslothai/unsloth#7084). + JSON/XML/YAML feeds). Binary types (PDF, images, archives, Office/ZIP, + octet-stream) return False so the fetcher never decodes them into a flood of + replacement chars that poison the model context (unslothai/unsloth#7084). A missing Content-Type coerces to ``text/plain`` upstream, so unlabeled - bodies pass here and are caught instead by the replacement-char fallback. + bodies pass here and are caught instead by the binary-char fallback. """ ct = (content_type or "").lower() if not ct: - return True # unlabeled: let the replacement-char fallback decide + return True # unlabeled: let the binary-char fallback decide if ct.startswith("text/"): return True if ct.startswith("application/"): - # "xml" also covers xhtml+xml / *+xml; "json" covers *+json. - return any(marker in ct for marker in ("json", "xml", "javascript", "csv")) + # Match the exact subtype, not a loose substring, so a .docx labeled + # application/vnd.openxmlformats-... isn't read as xml. RFC 6839 + # +json/+xml suffixes (ld+json, xhtml+xml, ...) pass too. + subtype = ct[len("application/") :].removeprefix("x-") + return subtype in _TEXT_APPLICATION_SUBTYPES or subtype.endswith(("+json", "+xml")) return False @@ -1604,11 +1616,11 @@ def _fetch_page_text( charset = resp.headers.get_content_charset() or "utf-8" raw_html = raw_bytes.decode(charset, errors = "replace") - # Fallback for binary mislabeled as text/* or sent with no Content-Type: - # a real text page has only a few replacement chars, if any. - if raw_html.count("\ufffd") > max( - _MIN_REPLACEMENT_CHARS, len(raw_html) // _BINARY_REPLACEMENT_DIVISOR - ): + # Fallback for binary mislabeled as text/* or sent with no Content-Type, + # including valid-UTF-8 binary (NUL/control-heavy payloads) that decodes + # without replacement chars: a real text page has few binary chars. + binary_chars = len(_BINARY_CHAR_RE.findall(raw_html)) + if binary_chars > max(_MIN_BINARY_CHARS, len(raw_html) // _BINARY_CHAR_DIVISOR): return f"(binary content, {len(raw_bytes)} bytes; not readable as text)" except _HTTPError as e: return f"Failed to fetch URL: HTTP {e.code} {getattr(e, 'reason', '')}" diff --git a/studio/backend/tests/test_web_fetch_binary_guard.py b/studio/backend/tests/test_web_fetch_binary_guard.py index cf9e0ce493..e5a1870de0 100644 --- a/studio/backend/tests/test_web_fetch_binary_guard.py +++ b/studio/backend/tests/test_web_fetch_binary_guard.py @@ -71,12 +71,18 @@ def _fetch_with(monkeypatch, body: bytes, content_type: str | None) -> str: ("application/xml", True), ("application/xhtml+xml", True), ("application/ld+json", True), + ("application/yaml", True), # raw configs / API specs + ("application/x-yaml", True), + ("application/x-ndjson", True), # newline-delimited JSON is text + ("application/ndjson", True), ("application/pdf", False), ("image/png", False), ("image/svg+xml", False), # SVG source isn't extracted downstream; reject ("application/octet-stream", False), ("application/zip", False), - ("", True), # unlabeled: defer to the replacement-char fallback + # A .docx ZIP must not pass just because "xml" is inside "openxmlformats". + ("application/vnd.openxmlformats-officedocument.wordprocessingml.document", False), + ("", True), # unlabeled: defer to the binary-char fallback (None, True), ], ) @@ -101,12 +107,28 @@ def test_image_rejected_by_content_type(monkeypatch): def test_binary_mislabeled_as_text_caught_by_fallback(monkeypatch): # A server sends binary but labels it text/plain -> the type check passes, - # so the replacement-char fallback must catch it. + # so the binary-char fallback must catch it. out = _fetch_with(monkeypatch, bytes(range(256)) * 20, "text/plain") assert "�" not in out assert "binary content" in out +def test_valid_utf8_binary_caught_by_control_chars(monkeypatch): + # NUL/control-heavy binary is valid UTF-8, so it decodes with no U+FFFD; + # the fallback must still catch it via control-char density (codex #2). + body = bytes([0, 1, 2, 3, 4, 5, 6, 7]) * 400 # all valid UTF-8, 0 replacement chars + out = _fetch_with(monkeypatch, body, "text/plain") + assert "binary content" in out + + +def test_ansi_colored_text_log_kept(monkeypatch): + # A text log with per-token ANSI color codes is text; ESC is excluded from + # the binary-char set so it isn't dropped as binary. + line = "".join(f"\x1b[32m+{i}\x1b[0m\n" for i in range(300)).encode() + out = _fetch_with(monkeypatch, line, "text/plain") + assert "binary content" not in out + + def test_binary_unlabeled_caught_by_fallback(monkeypatch): # No Content-Type coerces to text/plain upstream; the fallback still catches it. out = _fetch_with(monkeypatch, bytes(range(256)) * 20, None)