Sniff unknown MIME types and handle Latin-1

studio/backend/core/inference/tools.py

@@ -5,6 +5,7 @@
 (DuckDuckGo), Python code execution, and terminal commands."""
 
 import ast
+import codecs
 import http.client
 import os
 import signal
@@ -1520,28 +1521,47 @@ def _validate_and_resolve_host(hostname: str, port: int) -> tuple[bool, str, str
     return True, "", first_ip
 
 
-# Bare application/* subtypes that are text (the leading x- and the +json/+xml
-# structured-syntax suffixes are handled in the check).
-_TEXT_APPLICATION_SUBTYPES = frozenset(
-    {"json", "xml", "javascript", "ecmascript", "csv", "yaml", "ndjson", "jsonl"}
+# Known binary application subtypes are rejected by MIME type. Unknown
+# application types are sniffed so textual artifacts such as SQL remain usable.
+_BINARY_APPLICATION_SUBTYPES = frozenset(
+    {
+        "epub+zip",
+        "gzip",
+        "java-archive",
+        "msword",
+        "pdf",
+        "vnd.apple.installer+xml",
+        "vnd.ms-excel",
+        "vnd.ms-powerpoint",
+        "wasm",
+        "x-7z-compressed",
+        "x-bzip2",
+        "x-gzip",
+        "x-rar-compressed",
+        "x-tar",
+        "x-xz",
+        "zip",
+        "zstd",
+    }
+)
+_BINARY_APPLICATION_PREFIXES = (
+    "vnd.oasis.opendocument.",
+    "vnd.openxmlformats-officedocument.",
 )
 
 
 def _is_text_candidate_content_type(content_type: str | None) -> bool:
     """Whether a MIME type is textual or ambiguous enough for byte sniffing."""
-    ct = (content_type or "").partition(";")[0].strip().lower()
-    if not ct:
+    match = re.match(r"[\w.+-]+/[\w.+-]+", content_type or "")
+    if not match:
         return True
+    ct = match.group(0).lower()
     if ct.startswith("text/"):
         return True
     if ct.startswith("application/"):
-        # Exact matches avoid treating "openxmlformats" as XML. Structured
-        # +json/+xml suffixes remain valid text candidates.
-        subtype = ct[len("application/") :].removeprefix("x-")
-        return (
-            subtype == "octet-stream"
-            or subtype in _TEXT_APPLICATION_SUBTYPES
-            or subtype.endswith(("+json", "+xml"))
+        subtype = ct[len("application/") :]
+        return subtype not in _BINARY_APPLICATION_SUBTYPES and not subtype.startswith(
+            _BINARY_APPLICATION_PREFIXES
         )
     return False
 
@@ -1639,6 +1659,7 @@ def _fetch_page_text(
             return f"(binary content, {len(raw_bytes)} bytes; not readable as text)"
 
         declared = resp.headers.get_content_charset()
+        declared_codec = codecs.lookup(declared).name if declared else None
         raw_html = raw_bytes.decode(declared or "utf-8", errors = "replace")
 
         # Catch mislabeled or unlabeled binary, including valid UTF-8 controls.
@@ -1646,7 +1667,8 @@ def _fetch_page_text(
             # Rescue undeclared cp1252 only when the bytes have text structure.
             alt = (
                 raw_bytes.decode("cp1252", "replace")
-                if declared is None and _has_single_byte_text_evidence(raw_bytes)
+                if declared_codec in (None, "iso8859-1")
+                and _has_single_byte_text_evidence(raw_bytes)
                 else None
             )
             if alt is not None and not _looks_binary(alt):

studio/backend/tests/test_web_fetch_binary_guard.py

@@ -68,6 +68,8 @@ def _fetch_with(monkeypatch, body: bytes, content_type: str | None) -> str:
         ("application/x-yaml", True),
         ("application/x-ndjson", True),
         ("application/ndjson", True),
+        ("application/sql", True),
+        ("application/x-www-form-urlencoded", True),
         ("application/pdf", False),
         ("image/png", False),
         ("image/svg+xml", False),
@@ -95,13 +97,23 @@ def test_text_octet_stream_kept_after_sniffing(monkeypatch):
     assert "non-text content" not in out and "binary content" not in out
 
 
-@pytest.mark.parametrize("content_type", ["application/octet-stream", "text/plain", None])
+@pytest.mark.parametrize(
+    "content_type",
+    ["application/octet-stream", "application/x-custom-binary", "text/plain", None],
+)
 def test_binary_candidates_rejected_after_sniffing(monkeypatch, content_type):
     out = _fetch_with(monkeypatch, bytes(range(256)) * 20, content_type)
     assert "<EFBFBD>" not in out
     assert "binary content" in out
 
 
+@pytest.mark.parametrize("content_type", ["application/sql", "application/x-www-form-urlencoded"])
+def test_unknown_application_text_kept_after_sniffing(monkeypatch, content_type):
+    out = _fetch_with(monkeypatch, b"select readable_text from artifacts;\n" * 100, content_type)
+    assert "readable_text" in out
+    assert "non-text content" not in out and "binary content" not in out
+
+
 def test_valid_utf8_binary_caught_by_control_chars(monkeypatch):
     # These controls are valid UTF-8 and therefore produce no replacement chars.
     body = bytes([0, 1, 2, 3, 4, 5, 6, 7]) * 400
@@ -129,6 +141,14 @@ def test_latin1_text_without_charset_kept(monkeypatch):
     assert "MARKERWORD" in out
 
 
+@pytest.mark.parametrize("charset", ["iso-8859-1", "latin-1", "latin1"])
+def test_declared_latin1_cp1252_punctuation_kept(monkeypatch, charset):
+    body = ("“quoted” " * 100).encode("cp1252")
+    out = _fetch_with(monkeypatch, body, f"text/plain; charset={charset}")
+    assert "quoted" in out
+    assert "binary content" not in out
+
+
 def test_high_byte_binary_not_rescued_as_cp1252(monkeypatch):
     # cp1252 maps these bytes to printable characters, but they lack ASCII structure.
     body = bytes(range(0xA0, 0x100)) * 40