Studio: require ASCII evidence for declared Latin-1/cp1252 web fetches

Latin-1 and cp1252 decode every byte to a printable character, so a high-byte
binary body declared as iso-8859-1/windows-1252 decoded cleanly and slipped
past the control-character binary check. Apply the existing ASCII-structure gate
to those declared decodes as well. Scoped to the Latin family so legitimate
non-Latin single-byte pages (Cyrillic, Greek) are not rejected.
This commit is contained in:
danielhanchen 2026-07-15 15:06:58 +00:00
commit c7fbec216c
2 changed files with 43 additions and 0 deletions

View file

@ -3890,6 +3890,14 @@ def _fetch_page_text(
raw_html = alt
else:
return f"(binary content, {len(raw_bytes)} bytes; not readable as text)"
elif declared_codec in ("iso8859-1", "cp1252") and not (
_has_single_byte_text_evidence(raw_bytes)
):
# Latin-1/cp1252 map every byte to a printable character, so high-byte
# binary decodes cleanly and slips past _looks_binary; require ASCII text
# structure. Other single-byte charsets are left alone so legitimate
# non-Latin pages (Cyrillic, Greek, ...) are not rejected.
return f"(binary content, {len(raw_bytes)} bytes; not readable as text)"
except _HTTPError as e:
return f"Failed to fetch URL: HTTP {e.code} {getattr(e, 'reason', '')}"
except Exception as e:

View file

@ -264,3 +264,38 @@ def test_text_with_a_few_stray_replacement_chars_kept(monkeypatch):
out = _fetch_with(monkeypatch, body, "text/html")
assert "Real article text." in out
assert "binary content" not in out
@pytest.mark.parametrize("charset", ["iso-8859-1", "latin-1", "windows-1252", "cp1252"])
def test_declared_latin1_high_byte_binary_rejected(monkeypatch, charset):
# Latin-1/cp1252 decode high bytes to printable chars, so binary evades the
# control-char check; the ASCII-structure gate must still reject it.
body = bytes(range(0xA0, 0x100)) * 40
out = _fetch_with(monkeypatch, body, f"text/plain; charset={charset}")
assert "binary content" in out
@pytest.mark.parametrize("charset", ["iso-8859-1", "windows-1252"])
def test_declared_latin1_real_text_still_kept(monkeypatch, charset):
# Genuine accented Western text is ASCII-dominated and must not be rejected.
body = ("Cafe un tres bon eleve a l ecole. MARKERWORD ".replace("e", "é") + "voila ").encode(
"cp1252"
) * 40
out = _fetch_with(monkeypatch, body, f"text/plain; charset={charset}")
assert "MARKERWORD" in out
assert "binary content" not in out
@pytest.mark.parametrize(
"charset,codec,sample",
[
("windows-1251", "cp1251", "Это настоящая русская статья. Привет. "),
("koi8-r", "koi8-r", "Это настоящая русская статья. Привет. "),
("iso-8859-7", "iso-8859-7", "Αυτό είναι ένα ελληνικό άρθρο. "),
],
)
def test_declared_non_latin_single_byte_text_kept(monkeypatch, charset, codec, sample):
# Cyrillic/Greek single-byte pages are high-byte dense; the Latin-1 ASCII
# gate must not touch them or legitimate non-Western text would be dropped.
out = _fetch_with(monkeypatch, (sample * 40).encode(codec), f"text/plain; charset={charset}")
assert "binary content" not in out