diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index de772fd965..5f9f1bf53d 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -1391,6 +1391,7 @@ _BINARY_CHAR_DIVISOR = 8 _BINARY_MAGIC = ( b"%PDF-", # PDF b"PK\x03\x04", # zip / docx / xlsx / pptx / epub / jar + b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1", # OLE / legacy Office b"\x89PNG\r\n\x1a\n", # PNG b"\xff\xd8\xff", # JPEG b"GIF87a", @@ -1413,6 +1414,12 @@ def _looks_binary(text: str) -> bool: ) +def _has_binary_magic(data: bytes) -> bool: + """Whether a common binary signature follows optional BOM or whitespace.""" + head = data[:1024].lstrip().removeprefix(b"\xef\xbb\xbf").lstrip() + return head.startswith(_BINARY_MAGIC) + + def _has_single_byte_text_evidence(data: bytes) -> bool: """True when *data* has enough ASCII structure for a cp1252 text retry.""" if not data: @@ -1528,11 +1535,8 @@ _BINARY_APPLICATION_SUBTYPES = frozenset( "epub+zip", "gzip", "java-archive", - "msword", "pdf", "vnd.apple.installer+xml", - "vnd.ms-excel", - "vnd.ms-powerpoint", "wasm", "x-7z-compressed", "x-bzip2", @@ -1544,10 +1548,6 @@ _BINARY_APPLICATION_SUBTYPES = frozenset( "zstd", } ) -_BINARY_APPLICATION_PREFIXES = ( - "vnd.oasis.opendocument.", - "vnd.openxmlformats-officedocument.", -) def _is_text_candidate_content_type(content_type: str | None) -> bool: @@ -1560,9 +1560,7 @@ def _is_text_candidate_content_type(content_type: str | None) -> bool: return True if ct.startswith("application/"): subtype = ct[len("application/") :] - return subtype not in _BINARY_APPLICATION_SUBTYPES and not subtype.startswith( - _BINARY_APPLICATION_PREFIXES - ) + return subtype not in _BINARY_APPLICATION_SUBTYPES return False @@ -1655,7 +1653,7 @@ def _fetch_page_text( return f"(non-text content: {safe_type}, {len(raw_bytes)} bytes; not readable as text)" # Catch text-labeled binary whose header and first chunk look textual. - if raw_bytes.startswith(_BINARY_MAGIC): + if _has_binary_magic(raw_bytes): return f"(binary content, {len(raw_bytes)} bytes; not readable as text)" declared = resp.headers.get_content_charset() diff --git a/studio/backend/tests/test_web_fetch_binary_guard.py b/studio/backend/tests/test_web_fetch_binary_guard.py index 9e767b64c1..411db2276f 100644 --- a/studio/backend/tests/test_web_fetch_binary_guard.py +++ b/studio/backend/tests/test_web_fetch_binary_guard.py @@ -75,7 +75,8 @@ def _fetch_with(monkeypatch, body: bytes, content_type: str | None) -> str: ("image/svg+xml", False), ("application/octet-stream", True), ("application/zip", False), - ("application/vnd.openxmlformats-officedocument.wordprocessingml.document", False), + ("application/vnd.ms-excel", True), + ("application/vnd.openxmlformats-officedocument.wordprocessingml.document", True), ("", True), (None, True), ], @@ -114,6 +115,13 @@ def test_unknown_application_text_kept_after_sniffing(monkeypatch, content_type) assert "non-text content" not in out and "binary content" not in out +def test_excel_labeled_csv_kept_after_sniffing(monkeypatch): + body = b"name,value\nreadable,42\n" * 100 + out = _fetch_with(monkeypatch, body, "application/vnd.ms-excel") + assert "readable" in out + assert "binary content" not in out + + def test_valid_utf8_binary_caught_by_control_chars(monkeypatch): # These controls are valid UTF-8 and therefore produce no replacement chars. body = bytes([0, 1, 2, 3, 4, 5, 6, 7]) * 400 @@ -123,13 +131,43 @@ def test_valid_utf8_binary_caught_by_control_chars(monkeypatch): @pytest.mark.parametrize( "magic", - [b"%PDF-", b"PK\x03\x04", b"\x1f\x8b", b"BZh", b"\xfd7zXZ\x00", b"\x28\xb5\x2f\xfd"], + [ + b"%PDF-", + b"PK\x03\x04", + b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1", + b"\x1f\x8b", + b"BZh", + b"\xfd7zXZ\x00", + b"\x28\xb5\x2f\xfd", + ], ) def test_text_labeled_binary_caught_by_magic(monkeypatch, magic): out = _fetch_with(monkeypatch, magic + b" printable text-heavy body" * 100, "text/plain") assert "binary content" in out +@pytest.mark.parametrize("prefix", [b"\xef\xbb\xbf", b" \r\n", b"\t\xef\xbb\xbf "]) +def test_pdf_magic_after_harmless_prefix(monkeypatch, prefix): + body = prefix + b"%PDF-1.7\n" + b"1 0 obj<>endobj\n" * 100 + out = _fetch_with(monkeypatch, body, "text/plain") + assert "binary content" in out + + +@pytest.mark.parametrize( + "content_type,magic", + [ + ("application/vnd.ms-excel", b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"), + ( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + b"PK\x03\x04", + ), + ], +) +def test_office_labeled_binary_caught_by_magic(monkeypatch, content_type, magic): + out = _fetch_with(monkeypatch, magic + b" printable text-heavy body" * 100, content_type) + assert "binary content" in out + + def test_latin1_text_without_charset_kept(monkeypatch): # The cp1252 retry should rescue accent-heavy text with ASCII structure. body = (