Studio: require ASCII evidence for declared Latin-1/cp1252 web fetches
Latin-1 and cp1252 decode every byte to a printable character, so a high-byte binary body declared as iso-8859-1/windows-1252 decoded cleanly and slipped past the control-character binary check. Apply the existing ASCII-structure gate to those declared decodes as well. Scoped to the Latin family so legitimate non-Latin single-byte pages (Cyrillic, Greek) are not rejected.
This commit is contained in:
parent
4eb12832c4
commit
c7fbec216c
2 changed files with 43 additions and 0 deletions
|
|
@ -3890,6 +3890,14 @@ def _fetch_page_text(
|
|||
raw_html = alt
|
||||
else:
|
||||
return f"(binary content, {len(raw_bytes)} bytes; not readable as text)"
|
||||
elif declared_codec in ("iso8859-1", "cp1252") and not (
|
||||
_has_single_byte_text_evidence(raw_bytes)
|
||||
):
|
||||
# Latin-1/cp1252 map every byte to a printable character, so high-byte
|
||||
# binary decodes cleanly and slips past _looks_binary; require ASCII text
|
||||
# structure. Other single-byte charsets are left alone so legitimate
|
||||
# non-Latin pages (Cyrillic, Greek, ...) are not rejected.
|
||||
return f"(binary content, {len(raw_bytes)} bytes; not readable as text)"
|
||||
except _HTTPError as e:
|
||||
return f"Failed to fetch URL: HTTP {e.code} {getattr(e, 'reason', '')}"
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -264,3 +264,38 @@ def test_text_with_a_few_stray_replacement_chars_kept(monkeypatch):
|
|||
out = _fetch_with(monkeypatch, body, "text/html")
|
||||
assert "Real article text." in out
|
||||
assert "binary content" not in out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("charset", ["iso-8859-1", "latin-1", "windows-1252", "cp1252"])
|
||||
def test_declared_latin1_high_byte_binary_rejected(monkeypatch, charset):
|
||||
# Latin-1/cp1252 decode high bytes to printable chars, so binary evades the
|
||||
# control-char check; the ASCII-structure gate must still reject it.
|
||||
body = bytes(range(0xA0, 0x100)) * 40
|
||||
out = _fetch_with(monkeypatch, body, f"text/plain; charset={charset}")
|
||||
assert "binary content" in out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("charset", ["iso-8859-1", "windows-1252"])
|
||||
def test_declared_latin1_real_text_still_kept(monkeypatch, charset):
|
||||
# Genuine accented Western text is ASCII-dominated and must not be rejected.
|
||||
body = ("Cafe un tres bon eleve a l ecole. MARKERWORD ".replace("e", "é") + "voila ").encode(
|
||||
"cp1252"
|
||||
) * 40
|
||||
out = _fetch_with(monkeypatch, body, f"text/plain; charset={charset}")
|
||||
assert "MARKERWORD" in out
|
||||
assert "binary content" not in out
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"charset,codec,sample",
|
||||
[
|
||||
("windows-1251", "cp1251", "Это настоящая русская статья. Привет. "),
|
||||
("koi8-r", "koi8-r", "Это настоящая русская статья. Привет. "),
|
||||
("iso-8859-7", "iso-8859-7", "Αυτό είναι ένα ελληνικό άρθρο. "),
|
||||
],
|
||||
)
|
||||
def test_declared_non_latin_single_byte_text_kept(monkeypatch, charset, codec, sample):
|
||||
# Cyrillic/Greek single-byte pages are high-byte dense; the Latin-1 ASCII
|
||||
# gate must not touch them or legitimate non-Western text would be dropped.
|
||||
out = _fetch_with(monkeypatch, (sample * 40).encode(codec), f"text/plain; charset={charset}")
|
||||
assert "binary content" not in out
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue