%s
T
CODEMARKER |
diff --git a/studio/backend/core/inference/_html_to_md.py b/studio/backend/core/inference/_html_to_md.py index a73b931969..5f207176e6 100644 --- a/studio/backend/core/inference/_html_to_md.py +++ b/studio/backend/core/inference/_html_to_md.py @@ -246,8 +246,8 @@ class _HeaderFrame: # Tallied on emit; re-cleaning each parent's buffer would be quadratic. self.rendered_chars: int = 0 self.heading_chars: int = 0 - # Buffers open now enclose the frame, later ones nest. Links and cells are keyed by sequence - # number, not a flag: an inner one replaces the outer in the renderer's single slot. + # Buffers open now enclose the frame, later ones nest. Link/cell sequence numbers, not + # flags: an inner one replaces the outer in the renderer's single slot. self.outer_list_depth = list_depth self.outer_link_seq = link_seq self.outer_cell_seq = cell_seq @@ -267,8 +267,8 @@ class _HeaderFrame: if not closed_by_own_tag: return "".join(self.parts) headings = "".join(self.heading_parts) - # Droppable visible chars only: headings are kept anyway, and blank structure _cleanup - # collapses (500 empty
x opens two spans and each
- # end tag owes a backtick, else the delimiters stop pairing.
+ # Depth, not a flag: nested opens two spans and each owes a backtick.
self._inline_code_depth: int = 0
# Blockquote state: stack of buffers so nested blockquotes get the right ">" depth.
@@ -369,8 +366,8 @@ class _MarkdownRenderer(HTMLParser):
Such a buffer emits into the frame when it closes; an enclosing one
(already open at `` would swallow the
- # row into the code block as CODE| | instead of a cell holding the code.
+ # Before the cell: _finish_row emits, and an open would swallow the row as CODE| |.
if self._in_pre and not frame.outer_in_pre:
self._drain_pre()
if self._in_cell and self._cell_seq != frame.outer_cell_seq:
@@ -691,8 +680,8 @@ class _MarkdownRenderer(HTMLParser):
suppressed = bool(self._hidden_marks) or (
self._scope_tags is not None and self._scope_depth == 0
)
- # An element that IS the heading (e.g. ) emits when its buffer closes,
- # after the mark below is popped, so flush it here while the tee still recognises it.
+ # An element that IS the heading emits when its buffer closes, after the mark pops; flush
+ # it here while the tee still recognises it.
if self._in_link and tag == "a" and self._heading_marks:
self._finish_link()
if tag not in _VOID_TAGS:
@@ -866,8 +855,7 @@ class _MarkdownRenderer(HTMLParser):
elif tag == "pre" and self._in_pre:
self._drain_pre()
- # Already closed means a header frame recovered it; a second backtick here would leave the
- # rest of the page formatted as code.
+ # Already closed means a frame recovered it; a second backtick codes the rest of the page.
elif tag == "code" and not self._in_pre and self._inline_code_depth:
self._inline_code_depth -= 1
self._emit("`")
@@ -908,8 +896,8 @@ class _MarkdownRenderer(HTMLParser):
return
# Collapse all whitespace (including newlines) per HTML rules.
text = re.sub(r"\s+", " ", data)
- # Sized after collapsing, as the reader sees it: a run of source spaces in a link otherwise
- # clears the floor at ~100% density and drops the byline.
+ # Sized after collapsing, as the reader sees it: raw spaces in a link cleared the floor at
+ # ~100% density and dropped the byline.
self._count_header_text(text)
# Suppress whitespace-only nodes between table elements (source indentation).
if self._in_table and not self._in_cell and not text.strip():
@@ -933,8 +921,8 @@ class _MarkdownRenderer(HTMLParser):
# Flush pending buffers (handles truncated HTML from capped fetches)
def flush_pending(self) -> None:
"""Flush open side-buffers into ``_out`` after close(), recovering truncated HTML."""
- # Headers first: each frame finalizes the buffers opened inside it, then emits into what
- # encloses it, so an enclosing link or cell must still be open here; it is finalized below.
+ # Headers first: a frame finalizes its inner buffers, then emits into the enclosing link or
+ # cell, which must still be open here; it is finalized below.
self._flush_header_frames()
# Flush innermost buffers first so their content propagates outward.
@@ -1166,11 +1154,9 @@ def _visible_len(line: str) -> int:
continue
if line[i] == "[":
open_bracket = True
- # Without a bracket that opened it, "](" is literal text and the parens after
- # it are prose: skipping them dropped visible characters from the score.
+ # With no opening bracket, "](" is literal and the parens after it are prose.
if open_bracket and line[i] == "]" and i + 1 < n and line[i + 1] == "(":
- # Destinations may hold balanced or escaped parens (/card(foo)?q=..), so stopping at
- # the first ) leaves the rest scored as prose.
+ # Destinations may hold balanced or escaped parens, so the first ) does not end them.
j, depth = i + 2, 1
while j < n and depth:
char = line[j]
@@ -1180,8 +1166,7 @@ def _visible_len(line: str) -> int:
depth += (char == "(") - (char == ")")
j += 1
if depth:
- # Never balances, so this is not a link a reader would resolve; the bytes show as
- # text, so count them rather than dropping the prose on the rest of the line.
+ # Never balances, so it is not a link; the bytes show as text, so count them.
total += 1
i += 1
continue
diff --git a/studio/backend/tests/test_web_fetch_extraction.py b/studio/backend/tests/test_web_fetch_extraction.py
index 016eb29458..0074242e2e 100644
--- a/studio/backend/tests/test_web_fetch_extraction.py
+++ b/studio/backend/tests/test_web_fetch_extraction.py
@@ -1626,8 +1626,8 @@ def test_header_survives_every_buffer_combination(
assert "Article body sentence." in out, "body lost"
assert out.strip(), "empty output"
assert out.index("Article body sentence.") < 16000, "body pushed past the fetch cap"
- # The title holds only for closed markup with no : in a heading is verbatim text,
- # and unclosed shapes get best-effort recovery from code predating this pass.
+ # The title holds only for closed markup with no : there a heading is verbatim text, and
+ # unclosed shapes get best-effort recovery predating this pass.
well_formed = close_header and close_nested and "pre" not in (wrapper, nested)
if _GRID_HEADINGS[heading] and well_formed:
assert out.count("Page Title") == 1, "title duplicated or lost"
@@ -1760,8 +1760,8 @@ _FENCE = "`" * 3
def test_dropped_furniture_cannot_dominate_sibling_ranking():
- # Credit keeps a stripped article competitive but must not decide the match:
- # a teaser with a 1000 link header outranked five times its own real text.
+ # Credit must not decide the match: a teaser with a 1000 link header outranked five times its
+ # own real text.
teaser = "%s %s
" % (
"".join('Lang%d' % (i, i) for i in range(1000)),
"Teaser words here. " * 20,
@@ -1773,8 +1773,7 @@ def test_dropped_furniture_cannot_dominate_sibling_ranking():
def test_literal_bracket_paren_is_prose_not_a_destination():
- # No [ opened it, so "](" is literal text and the parens hold visible prose.
- # Skipping them scored 192 of 295 and dropped the article under the gate.
+ # No [ opened it, so "](" is literal and the parens hold prose; skipping them scored 192 of 295.
article = "%s](%s) %s
" % (
"Real article prose that the reader wants to see. " * 3,
"y" * 100,
@@ -1787,8 +1786,8 @@ def test_literal_bracket_paren_is_prose_not_a_destination():
def test_hand_preserved_heading_reaches_the_eligibility_tally():
- # The partial branch writes the title straight into heading_parts, so the
- # gate has to be told as well or a title-only card reads as body prose.
+ # The partial branch writes the title straight into heading_parts, so the gate needs telling
+ # too or a title-only card reads as body prose.
card = (
'%s%s%s
'
% (
@@ -1803,8 +1802,7 @@ def test_hand_preserved_heading_reaches_the_eligibility_tally():
def test_pre_inside_a_table_cell_is_drained_before_the_row():
- # The row is emitted, so an open swallowed it and produced a fenced
- # block holding CODEMARKER| | instead of a cell holding the code.
+ # The row is emitted, so an open swallowed it into a fence as CODEMARKER| |.
body = "T
CODEMARKER%s
" % (
"Article body. " * 30,
)
@@ -1814,8 +1812,8 @@ def test_pre_inside_a_table_cell_is_drained_before_the_row():
def test_post_processing_respects_the_widened_fence():
- # _cleanup and _strip_boilerplate_lines toggled on any ``` line, so the literal one
- # closed the block and its code was cleaned and de-boilerplated.
+ # Both passes toggled on any ``` line, so the literal one closed the block and its code was
+ # cleaned and de-boilerplated.
code = "%s\nskip to content\n\nreal code line \nmore code
" % _FENCE
body = "%s%s
" % (code, "Body text here. " * 20)
out = html_to_markdown(f"{body} ", main_content = True)
@@ -1836,8 +1834,8 @@ def test_unbalanced_destination_keeps_scoring_the_rest_of_the_line():
def test_structural_headings_do_not_satisfy_the_eligibility_gate():
- # role="heading" renders as plain prose, so ATX reparsing missed it and a header-only card
- # cleared the gate on its title plus dropped-list credit.
+ # role="heading" renders as prose, so ATX reparsing missed it and a header-only card cleared
+ # the gate on its title plus dropped-list credit.
card = '%s%s
' % (
"Card Title Words " * 14,
_interlanguage_list(300),
@@ -1908,8 +1906,7 @@ def test_heading_through_a_nested_buffer_is_emitted_once():
def test_late_code_end_tag_after_a_recovered_header_is_a_no_op():
- # arrives after ; the frame already closed the span, so a second emit is
- # unpaired across the rest of the page.
+ # arrives after ; the frame already closed the span, so a second emit is odd.
body = "T
navcode%s
%s
" % (
_interlanguage_list(300),
"Article body. " * 30,
@@ -1987,8 +1984,7 @@ def test_aria_heading_accepts_a_fallback_role_token_list():
],
)
def test_heading_survives_a_stripped_header(heading_markup, marker):
- # However the title is expressed, reducing a link-only header keeps it and
- # nothing else: through a cell, an ARIA role, a link, or an hgroup subtitle.
+ # However the title is expressed, reducing a link-only header keeps it and nothing else.
body = "%s%s
%s
" % (
heading_markup,
_interlanguage_list(300),
@@ -2052,8 +2048,7 @@ def test_header_size_is_independent_of_the_buffer_it_renders_through():
def test_nested_inline_code_closes_every_span_it_opened():
- # Two elements owe two closing backticks. Tracking open/closed as a
- # flag let the first answer for both and left the delimiters odd.
+ # Two elements owe two backticks; as a flag the first answered for both.
body = "x
%s
" % (
"Body text here. " * 20,
)