diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index d655c89397..de772fd965 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -5,6 +5,7 @@ (DuckDuckGo), Python code execution, and terminal commands.""" import ast +import codecs import http.client import os import signal @@ -1520,28 +1521,47 @@ def _validate_and_resolve_host(hostname: str, port: int) -> tuple[bool, str, str return True, "", first_ip -# Bare application/* subtypes that are text (the leading x- and the +json/+xml -# structured-syntax suffixes are handled in the check). -_TEXT_APPLICATION_SUBTYPES = frozenset( - {"json", "xml", "javascript", "ecmascript", "csv", "yaml", "ndjson", "jsonl"} +# Known binary application subtypes are rejected by MIME type. Unknown +# application types are sniffed so textual artifacts such as SQL remain usable. +_BINARY_APPLICATION_SUBTYPES = frozenset( + { + "epub+zip", + "gzip", + "java-archive", + "msword", + "pdf", + "vnd.apple.installer+xml", + "vnd.ms-excel", + "vnd.ms-powerpoint", + "wasm", + "x-7z-compressed", + "x-bzip2", + "x-gzip", + "x-rar-compressed", + "x-tar", + "x-xz", + "zip", + "zstd", + } +) +_BINARY_APPLICATION_PREFIXES = ( + "vnd.oasis.opendocument.", + "vnd.openxmlformats-officedocument.", ) def _is_text_candidate_content_type(content_type: str | None) -> bool: """Whether a MIME type is textual or ambiguous enough for byte sniffing.""" - ct = (content_type or "").partition(";")[0].strip().lower() - if not ct: + match = re.match(r"[\w.+-]+/[\w.+-]+", content_type or "") + if not match: return True + ct = match.group(0).lower() if ct.startswith("text/"): return True if ct.startswith("application/"): - # Exact matches avoid treating "openxmlformats" as XML. Structured - # +json/+xml suffixes remain valid text candidates. - subtype = ct[len("application/") :].removeprefix("x-") - return ( - subtype == "octet-stream" - or subtype in _TEXT_APPLICATION_SUBTYPES - or subtype.endswith(("+json", "+xml")) + subtype = ct[len("application/") :] + return subtype not in _BINARY_APPLICATION_SUBTYPES and not subtype.startswith( + _BINARY_APPLICATION_PREFIXES ) return False @@ -1639,6 +1659,7 @@ def _fetch_page_text( return f"(binary content, {len(raw_bytes)} bytes; not readable as text)" declared = resp.headers.get_content_charset() + declared_codec = codecs.lookup(declared).name if declared else None raw_html = raw_bytes.decode(declared or "utf-8", errors = "replace") # Catch mislabeled or unlabeled binary, including valid UTF-8 controls. @@ -1646,7 +1667,8 @@ def _fetch_page_text( # Rescue undeclared cp1252 only when the bytes have text structure. alt = ( raw_bytes.decode("cp1252", "replace") - if declared is None and _has_single_byte_text_evidence(raw_bytes) + if declared_codec in (None, "iso8859-1") + and _has_single_byte_text_evidence(raw_bytes) else None ) if alt is not None and not _looks_binary(alt): diff --git a/studio/backend/tests/test_web_fetch_binary_guard.py b/studio/backend/tests/test_web_fetch_binary_guard.py index 1feaf066b2..9e767b64c1 100644 --- a/studio/backend/tests/test_web_fetch_binary_guard.py +++ b/studio/backend/tests/test_web_fetch_binary_guard.py @@ -68,6 +68,8 @@ def _fetch_with(monkeypatch, body: bytes, content_type: str | None) -> str: ("application/x-yaml", True), ("application/x-ndjson", True), ("application/ndjson", True), + ("application/sql", True), + ("application/x-www-form-urlencoded", True), ("application/pdf", False), ("image/png", False), ("image/svg+xml", False), @@ -95,13 +97,23 @@ def test_text_octet_stream_kept_after_sniffing(monkeypatch): assert "non-text content" not in out and "binary content" not in out -@pytest.mark.parametrize("content_type", ["application/octet-stream", "text/plain", None]) +@pytest.mark.parametrize( + "content_type", + ["application/octet-stream", "application/x-custom-binary", "text/plain", None], +) def test_binary_candidates_rejected_after_sniffing(monkeypatch, content_type): out = _fetch_with(monkeypatch, bytes(range(256)) * 20, content_type) assert "�" not in out assert "binary content" in out +@pytest.mark.parametrize("content_type", ["application/sql", "application/x-www-form-urlencoded"]) +def test_unknown_application_text_kept_after_sniffing(monkeypatch, content_type): + out = _fetch_with(monkeypatch, b"select readable_text from artifacts;\n" * 100, content_type) + assert "readable_text" in out + assert "non-text content" not in out and "binary content" not in out + + def test_valid_utf8_binary_caught_by_control_chars(monkeypatch): # These controls are valid UTF-8 and therefore produce no replacement chars. body = bytes([0, 1, 2, 3, 4, 5, 6, 7]) * 400 @@ -129,6 +141,14 @@ def test_latin1_text_without_charset_kept(monkeypatch): assert "MARKERWORD" in out +@pytest.mark.parametrize("charset", ["iso-8859-1", "latin-1", "latin1"]) +def test_declared_latin1_cp1252_punctuation_kept(monkeypatch, charset): + body = ("“quoted” " * 100).encode("cp1252") + out = _fetch_with(monkeypatch, body, f"text/plain; charset={charset}") + assert "quoted" in out + assert "binary content" not in out + + def test_high_byte_binary_not_rescued_as_cp1252(monkeypatch): # cp1252 maps these bytes to printable characters, but they lack ASCII structure. body = bytes(range(0xA0, 0x100)) * 40