ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Tighten comments in the header extraction path

Browse source
This commit is contained in:
danielhanchen 2026-07-29 09:01:40 +00:00
commit 2a2ca40016
2 changed files with 46 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

77
studio/backend/core/inference/_html_to_md.py
View file

@ -246,8 +246,8 @@ class _HeaderFrame:
# Tallied on emit; re-cleaning each parent's buffer would be quadratic.
self.rendered_chars: int = 0
self.heading_chars: int = 0
# Buffers open now enclose the frame, later ones nest. Links and cells are keyed by sequence
# number, not a flag: an inner one replaces the outer in the renderer's single slot.
# Buffers open now enclose the frame, later ones nest. Link/cell sequence numbers, not
# flags: an inner one replaces the outer in the renderer's single slot.
self.outer_list_depth = list_depth
self.outer_link_seq = link_seq
self.outer_cell_seq = cell_seq
@ -267,8 +267,8 @@ class _HeaderFrame:
if not closed_by_own_tag:
return "".join(self.parts)
headings = "".join(self.heading_parts)
# Droppable visible chars only: headings are kept anyway, and blank structure _cleanup
# collapses (500 empty <div>s) must not make a tiny header look huge.
# Droppable chars only; headings survive, and blank structure _cleanup collapses must not
# inflate a tiny header.
droppable = self.rendered_chars - self.heading_chars
big_enough = self.text_chars >= _HEADER_MIN_CHARS or droppable >= _HEADER_MAX_RENDERED_CHARS
if big_enough and self.link_chars >= _HEADER_LINK_DENSITY * self.text_chars:
@ -299,13 +299,12 @@ class _MarkdownRenderer(HTMLParser):
self._scope_tags = scope_tags
self._scope_depth: int = 0
# One boundary per top-level scope element, so each candidate is sized alone and a swarm
# of tiny sibling cards cannot clear the threshold together.
# One boundary per top-level scope element, so a swarm of tiny cards cannot clear the gate.
self.scope_segments: list[str] = []
self._scope_seg_start: int | None = None
# Open non-void tags plus the indices where a hidden element started. End tags pop to the
# matching tag, so an omitted </p>/</li> cannot leave the renderer stuck hidden.
# Open non-void tags plus hidden-start indices, so an omitted </p>/</li> cannot leave the
# renderer stuck hidden.
self._open_tags: list[str] = []
# Open tags that can be closed implicitly; zero lets _close_implicit skip the scan.
self._closable_open: int = 0
@ -318,8 +317,8 @@ class _MarkdownRenderer(HTMLParser):
self._dropped_chars: int = 0
self._seg_dropped_start: int = 0
self.scope_dropped: list[int] = []
# Heading text per segment: role="heading", <hgroup> and linked <h1> render as ordinary
# prose, so ATX reparsing alone cannot keep them out of the gate.
# Heading text per segment: role="heading", hgroup and linked h1 render as prose, so ATX
# reparsing alone cannot keep them out of the gate.
self._seg_heading_texts: list[str] = []
self.scope_heading_prose: list[int] = []
# Open-tag indices of headings, unwound with _hidden_marks.
@ -330,8 +329,7 @@ class _MarkdownRenderer(HTMLParser):
self._link_text_parts: list[str] = []
self._in_link: bool = False
self._link_seq: int = 0
# A link wrapping a heading emits after the heading mark is gone, so the tee is told to
# treat that one emit as heading output.
# A link wrapping a heading emits after the mark pops, so the tee is told to treat it so.
self._link_had_heading: bool = False
self._link_heading_parts: list[str] = []
self._emit_as_heading: bool = False
@ -356,8 +354,7 @@ class _MarkdownRenderer(HTMLParser):
# Pre/code state
self._in_pre: bool = False
self._pre_parts: list[str] = []
# Depth, not a flag: <code><code>x</code></code> opens two spans and each
# end tag owes a backtick, else the delimiters stop pairing.
# Depth, not a flag: nested <code> opens two spans and each </code> owes a backtick.
self._inline_code_depth: int = 0
# Blockquote state: stack of buffers so nested blockquotes get the right ">" depth.
@ -369,8 +366,8 @@ class _MarkdownRenderer(HTMLParser):
Such a buffer emits into the frame when it closes; an enclosing one
(already open at ``<header>``) must not capture it."""
# Only the buffer _emit would pick matters, in its order: a blockquote inside a
# cell-enclosed header still routes to the cell, so OR-ing them would call that nested.
# Only the buffer _emit would pick matters, in its order; OR-ing them calls an enclosing
# one nested.
if self._in_link:
return self._link_seq != frame.outer_link_seq
if self._in_cell:
@ -382,28 +379,24 @@ class _MarkdownRenderer(HTMLParser):
def _emit(self, text: str) -> None:
frame = self._header_stack[-1] if self._header_stack else None
# Tee wherever the text routes, so a heading in a nested buffer is captured. A link opened
# inside the frame delivers twice (raw, then formatted by _finish_link), so only the
# formatted form is teed; an enclosing link never re-delivers and is teed as it comes.
# inside the frame delivers twice, so tee only the formatted form; an enclosing link, once.
in_nested_link = (
self._in_link and self._link_seq != frame.outer_link_seq if frame else False
)
# A flushed buffer re-delivers text already teed on the way in, so replays never tee;
# _finish_link is the exception and re-arms the tee itself.
# Replays never tee: the text was teed on the way in. _finish_link re-arms the tee itself.
as_heading = (
self._heading_marks and not in_nested_link and not self._replaying
) or self._emit_as_heading
if frame is not None and as_heading:
frame.heading_parts.append(text)
frame.heading_chars += len(text.strip())
# Tallied for the eligibility gate, independent of any frame. Text inside a link waits for
# _finish_link so the formatted form counts once.
# For the eligibility gate, frame or not. Link text waits for _finish_link to count once.
if not self._replaying and (
(self._heading_marks and not self._in_link) or self._emit_as_heading
):
self._seg_heading_texts.append(text)
nested_open = self._nested_buffer_open(frame) if frame is not None else False
# Tally once, on the emit that reaches the frame: counting text going INTO a nested buffer
# and again when it flushes doubled it, so a kept header stripped once quoted.
# Tally once, on the emit reaching the frame; counting again on flush doubled it.
if frame is not None and not nested_open:
frame.rendered_chars += len(text.strip())
frame.parts.append(text)
@ -499,8 +492,7 @@ class _MarkdownRenderer(HTMLParser):
self._in_link = False
self._link_text_parts = []
self._link_heading_parts = []
# An anchor wrapping a heading AND other content is furniture carrying a title: tee the
# title alone, or the whole nav rides out as a heading.
# An anchor wrapping a heading AND other content tees the title alone, else the nav rides.
partial = bool(heading_text) and heading_text != text
self._emit_as_heading = self._link_had_heading and not partial
self._link_had_heading = False
@ -515,8 +507,7 @@ class _MarkdownRenderer(HTMLParser):
frame = self._header_stack[-1]
frame.heading_parts.append(heading_text + "\n\n")
frame.heading_chars += len(heading_text)
# Preserved by hand, so the gate needs telling too or a title-only card
# reads as body prose.
# Preserved by hand, so tell the gate too or a title-only card reads as body prose.
self._seg_heading_texts.append(heading_text)
# ------------------------------------------------------------------
@ -592,13 +583,11 @@ class _MarkdownRenderer(HTMLParser):
# The header boundary proves this anchor did not adopt the body: its text is furniture.
frame.link_chars += self._link_header_chars
self._finish_link()
# Inline code opened OUTSIDE the header is the page's; closing it here would leave the real
# </code> to emit an unpaired backtick.
# Code opened OUTSIDE the header is the page's; closing it here leaves </code> unpaired.
while self._inline_code_depth > frame.outer_in_code:
self._inline_code_depth -= 1
self._emit("`")
# Before the cell: _finish_row emits, and an open <pre> would swallow the
# row into the code block as CODE| | instead of a cell holding the code.
# Before the cell: _finish_row emits, and an open <pre> would swallow the row as CODE| |.
if self._in_pre and not frame.outer_in_pre:
self._drain_pre()
if self._in_cell and self._cell_seq != frame.outer_cell_seq:
@ -691,8 +680,8 @@ class _MarkdownRenderer(HTMLParser):
suppressed = bool(self._hidden_marks) or (
self._scope_tags is not None and self._scope_depth == 0
)
# An element that IS the heading (e.g. <a role="heading">) emits when its buffer closes,
# after the mark below is popped, so flush it here while the tee still recognises it.
# An element that IS the heading emits when its buffer closes, after the mark pops; flush
# it here while the tee still recognises it.
if self._in_link and tag == "a" and self._heading_marks:
self._finish_link()
if tag not in _VOID_TAGS:
@ -866,8 +855,7 @@ class _MarkdownRenderer(HTMLParser):
elif tag == "pre" and self._in_pre:
self._drain_pre()
# Already closed means a header frame recovered it; a second backtick here would leave the
# rest of the page formatted as code.
# Already closed means a frame recovered it; a second backtick codes the rest of the page.
elif tag == "code" and not self._in_pre and self._inline_code_depth:
self._inline_code_depth -= 1
self._emit("`")
@ -908,8 +896,8 @@ class _MarkdownRenderer(HTMLParser):
return
# Collapse all whitespace (including newlines) per HTML rules.
text = re.sub(r"\s+", " ", data)
# Sized after collapsing, as the reader sees it: a run of source spaces in a link otherwise
# clears the floor at ~100% density and drops the byline.
# Sized after collapsing, as the reader sees it: raw spaces in a link cleared the floor at
# ~100% density and dropped the byline.
self._count_header_text(text)
# Suppress whitespace-only nodes between table elements (source indentation).
if self._in_table and not self._in_cell and not text.strip():
@ -933,8 +921,8 @@ class _MarkdownRenderer(HTMLParser):
# Flush pending buffers (handles truncated HTML from capped fetches)
def flush_pending(self) -> None:
"""Flush open side-buffers into ``_out`` after close(), recovering truncated HTML."""
# Headers first: each frame finalizes the buffers opened inside it, then emits into what
# encloses it, so an enclosing link or cell must still be open here; it is finalized below.
# Headers first: a frame finalizes its inner buffers, then emits into the enclosing link or
# cell, which must still be open here; it is finalized below.
self._flush_header_frames()
# Flush innermost buffers first so their content propagates outward.
@ -1166,11 +1154,9 @@ def _visible_len(line: str) -> int:
continue
if line[i] == "[":
open_bracket = True
# Without a bracket that opened it, "](" is literal text and the parens after
# it are prose: skipping them dropped visible characters from the score.
# With no opening bracket, "](" is literal and the parens after it are prose.
if open_bracket and line[i] == "]" and i + 1 < n and line[i + 1] == "(":
# Destinations may hold balanced or escaped parens (/card(foo)?q=..), so stopping at
# the first ) leaves the rest scored as prose.
# Destinations may hold balanced or escaped parens, so the first ) does not end them.
j, depth = i + 2, 1
while j < n and depth:
char = line[j]
@ -1180,8 +1166,7 @@ def _visible_len(line: str) -> int:
depth += (char == "(") - (char == ")")
j += 1
if depth:
# Never balances, so this is not a link a reader would resolve; the bytes show as
# text, so count them rather than dropping the prose on the rest of the line.
# Never balances, so it is not a link; the bytes show as text, so count them.
total += 1
i += 1
continue

35
studio/backend/tests/test_web_fetch_extraction.py
View file

@ -1626,8 +1626,8 @@ def test_header_survives_every_buffer_combination(
assert "Article body sentence." in out, "body lost"
assert out.strip(), "empty output"
assert out.index("Article body sentence.") < 16000, "body pushed past the fetch cap"
# The title holds only for closed markup with no <pre>: in <pre> a heading is verbatim text,
# and unclosed shapes get best-effort recovery from code predating this pass.
# The title holds only for closed markup with no <pre>: there a heading is verbatim text, and
# unclosed shapes get best-effort recovery predating this pass.
well_formed = close_header and close_nested and "pre" not in (wrapper, nested)
if _GRID_HEADINGS[heading] and well_formed:
assert out.count("Page Title") == 1, "title duplicated or lost"
@ -1760,8 +1760,8 @@ _FENCE = "`" * 3
def test_dropped_furniture_cannot_dominate_sibling_ranking():
# Credit keeps a stripped article competitive but must not decide the match:
# a teaser with a 1000 link header outranked five times its own real text.
# Credit must not decide the match: a teaser with a 1000 link header outranked five times its
# own real text.
teaser = "<article><header>%s</header><p>%s</p></article>" % (
"".join('<a href="/l%d">Lang%d</a>' % (i, i) for i in range(1000)),
"Teaser words here. " * 20,
@ -1773,8 +1773,7 @@ def test_dropped_furniture_cannot_dominate_sibling_ranking():
def test_literal_bracket_paren_is_prose_not_a_destination():
# No [ opened it, so "](" is literal text and the parens hold visible prose.
# Skipping them scored 192 of 295 and dropped the article under the gate.
# No [ opened it, so "](" is literal and the parens hold prose; skipping them scored 192 of 295.
article = "<article><p>%s](%s) %s</p></article>" % (
"Real article prose that the reader wants to see. " * 3,
"y" * 100,
@ -1787,8 +1786,8 @@ def test_literal_bracket_paren_is_prose_not_a_destination():
def test_hand_preserved_heading_reaches_the_eligibility_tally():
# The partial branch writes the title straight into heading_parts, so the
# gate has to be told as well or a title-only card reads as body prose.
# The partial branch writes the title straight into heading_parts, so the gate needs telling
# too or a title-only card reads as body prose.
card = (
'<article><header><a href="/h"><div role="heading">%s</div>%s</a><ul>%s</ul></header></article>'
% (
@ -1803,8 +1802,7 @@ def test_hand_preserved_heading_reaches_the_eligibility_tally():
def test_pre_inside_a_table_cell_is_drained_before_the_row():
# The row is emitted, so an open <pre> swallowed it and produced a fenced
# block holding CODEMARKER| | instead of a cell holding the code.
# The row is emitted, so an open <pre> swallowed it into a fence as CODEMARKER| |.
body = "<main><header><h1>T</h1><table><tr><td><pre>CODEMARKER</header><p>%s</p></main>" % (
"Article body. " * 30,
)
@ -1814,8 +1812,8 @@ def test_pre_inside_a_table_cell_is_drained_before_the_row():
def test_post_processing_respects_the_widened_fence():
# _cleanup and _strip_boilerplate_lines toggled on any ``` line, so the literal one
# closed the block and its code was cleaned and de-boilerplated.
# Both passes toggled on any ``` line, so the literal one closed the block and its code was
# cleaned and de-boilerplated.
code = "<pre>%s\nskip to content\n\nreal code line \nmore code</pre>" % _FENCE
body = "<article>%s<p>%s</p></article>" % (code, "Body text here. " * 20)
out = html_to_markdown(f"<body><main>{body}</main></body>", main_content = True)
@ -1836,8 +1834,8 @@ def test_unbalanced_destination_keeps_scoring_the_rest_of_the_line():
def test_structural_headings_do_not_satisfy_the_eligibility_gate():
# role="heading" renders as plain prose, so ATX reparsing missed it and a header-only card
# cleared the gate on its title plus dropped-list credit.
# role="heading" renders as prose, so ATX reparsing missed it and a header-only card cleared
# the gate on its title plus dropped-list credit.
card = '<article><header><div role="heading">%s</div><ul>%s</ul></header></article>' % (
"Card Title Words " * 14,
_interlanguage_list(300),
@ -1908,8 +1906,7 @@ def test_heading_through_a_nested_buffer_is_emitted_once():
def test_late_code_end_tag_after_a_recovered_header_is_a_no_op():
# </code> arrives after </header>; the frame already closed the span, so a second emit is
# unpaired across the rest of the page.
# </code> arrives after </header>; the frame already closed the span, so a second emit is odd.
body = "<main><header><h1>T</h1><code>navcode<ul>%s</ul></header><p>%s</p></code></main>" % (
_interlanguage_list(300),
"Article body. " * 30,
@ -1987,8 +1984,7 @@ def test_aria_heading_accepts_a_fallback_role_token_list():
],
)
def test_heading_survives_a_stripped_header(heading_markup, marker):
# However the title is expressed, reducing a link-only header keeps it and
# nothing else: through a cell, an ARIA role, a link, or an hgroup subtitle.
# However the title is expressed, reducing a link-only header keeps it and nothing else.
body = "<main><header>%s<ul>%s</ul></header><p>%s</p></main>" % (
heading_markup,
_interlanguage_list(300),
@ -2052,8 +2048,7 @@ def test_header_size_is_independent_of_the_buffer_it_renders_through():
def test_nested_inline_code_closes_every_span_it_opened():
# Two <code> elements owe two closing backticks. Tracking open/closed as a
# flag let the first </code> answer for both and left the delimiters odd.
# Two <code> elements owe two backticks; as a flag the first </code> answered for both.
body = "<main><article><p><code><code>x</code></code></p><p>%s</p></article></main>" % (
"Body text here. " * 20,
)