Sniff ambiguous Office MIME and prefixed magic

studio/backend/core/inference/tools.py

@@ -1391,6 +1391,7 @@ _BINARY_CHAR_DIVISOR = 8
 _BINARY_MAGIC = (
     b"%PDF-",  # PDF
     b"PK\x03\x04",  # zip / docx / xlsx / pptx / epub / jar
+    b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1",  # OLE / legacy Office
     b"\x89PNG\r\n\x1a\n",  # PNG
     b"\xff\xd8\xff",  # JPEG
     b"GIF87a",
@@ -1413,6 +1414,12 @@ def _looks_binary(text: str) -> bool:
     )
 
 
+def _has_binary_magic(data: bytes) -> bool:
+    """Whether a common binary signature follows optional BOM or whitespace."""
+    head = data[:1024].lstrip().removeprefix(b"\xef\xbb\xbf").lstrip()
+    return head.startswith(_BINARY_MAGIC)
+
+
 def _has_single_byte_text_evidence(data: bytes) -> bool:
     """True when *data* has enough ASCII structure for a cp1252 text retry."""
     if not data:
@@ -1528,11 +1535,8 @@ _BINARY_APPLICATION_SUBTYPES = frozenset(
         "epub+zip",
         "gzip",
         "java-archive",
-        "msword",
         "pdf",
         "vnd.apple.installer+xml",
-        "vnd.ms-excel",
-        "vnd.ms-powerpoint",
         "wasm",
         "x-7z-compressed",
         "x-bzip2",
@@ -1544,10 +1548,6 @@ _BINARY_APPLICATION_SUBTYPES = frozenset(
         "zstd",
     }
 )
-_BINARY_APPLICATION_PREFIXES = (
-    "vnd.oasis.opendocument.",
-    "vnd.openxmlformats-officedocument.",
-)
 
 
 def _is_text_candidate_content_type(content_type: str | None) -> bool:
@@ -1560,9 +1560,7 @@ def _is_text_candidate_content_type(content_type: str | None) -> bool:
         return True
     if ct.startswith("application/"):
         subtype = ct[len("application/") :]
-        return subtype not in _BINARY_APPLICATION_SUBTYPES and not subtype.startswith(
-            _BINARY_APPLICATION_PREFIXES
-        )
+        return subtype not in _BINARY_APPLICATION_SUBTYPES
     return False
 
 
@@ -1655,7 +1653,7 @@ def _fetch_page_text(
             return f"(non-text content: {safe_type}, {len(raw_bytes)} bytes; not readable as text)"
 
         # Catch text-labeled binary whose header and first chunk look textual.
-        if raw_bytes.startswith(_BINARY_MAGIC):
+        if _has_binary_magic(raw_bytes):
             return f"(binary content, {len(raw_bytes)} bytes; not readable as text)"
 
         declared = resp.headers.get_content_charset()

studio/backend/tests/test_web_fetch_binary_guard.py

@@ -75,7 +75,8 @@ def _fetch_with(monkeypatch, body: bytes, content_type: str | None) -> str:
         ("image/svg+xml", False),
         ("application/octet-stream", True),
         ("application/zip", False),
-        ("application/vnd.openxmlformats-officedocument.wordprocessingml.document", False),
+        ("application/vnd.ms-excel", True),
+        ("application/vnd.openxmlformats-officedocument.wordprocessingml.document", True),
         ("", True),
         (None, True),
     ],
@@ -114,6 +115,13 @@ def test_unknown_application_text_kept_after_sniffing(monkeypatch, content_type)
     assert "non-text content" not in out and "binary content" not in out
 
 
+def test_excel_labeled_csv_kept_after_sniffing(monkeypatch):
+    body = b"name,value\nreadable,42\n" * 100
+    out = _fetch_with(monkeypatch, body, "application/vnd.ms-excel")
+    assert "readable" in out
+    assert "binary content" not in out
+
+
 def test_valid_utf8_binary_caught_by_control_chars(monkeypatch):
     # These controls are valid UTF-8 and therefore produce no replacement chars.
     body = bytes([0, 1, 2, 3, 4, 5, 6, 7]) * 400
@@ -123,13 +131,43 @@ def test_valid_utf8_binary_caught_by_control_chars(monkeypatch):
 
 @pytest.mark.parametrize(
     "magic",
-    [b"%PDF-", b"PK\x03\x04", b"\x1f\x8b", b"BZh", b"\xfd7zXZ\x00", b"\x28\xb5\x2f\xfd"],
+    [
+        b"%PDF-",
+        b"PK\x03\x04",
+        b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1",
+        b"\x1f\x8b",
+        b"BZh",
+        b"\xfd7zXZ\x00",
+        b"\x28\xb5\x2f\xfd",
+    ],
 )
 def test_text_labeled_binary_caught_by_magic(monkeypatch, magic):
     out = _fetch_with(monkeypatch, magic + b" printable text-heavy body" * 100, "text/plain")
     assert "binary content" in out
 
 
+@pytest.mark.parametrize("prefix", [b"\xef\xbb\xbf", b" \r\n", b"\t\xef\xbb\xbf "])
+def test_pdf_magic_after_harmless_prefix(monkeypatch, prefix):
+    body = prefix + b"%PDF-1.7\n" + b"1 0 obj<</Type/Catalog>>endobj\n" * 100
+    out = _fetch_with(monkeypatch, body, "text/plain")
+    assert "binary content" in out
+
+
+@pytest.mark.parametrize(
+    "content_type,magic",
+    [
+        ("application/vnd.ms-excel", b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"),
+        (
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+            b"PK\x03\x04",
+        ),
+    ],
+)
+def test_office_labeled_binary_caught_by_magic(monkeypatch, content_type, magic):
+    out = _fetch_with(monkeypatch, magic + b" printable text-heavy body" * 100, content_type)
+    assert "binary content" in out
+
+
 def test_latin1_text_without_charset_kept(monkeypatch):
     # The cp1252 retry should rescue accent-heavy text with ASCII structure.
     body = (