diff --git a/studio/backend/core/inference/_html_to_md.py b/studio/backend/core/inference/_html_to_md.py
index e7de4a5312..92471b9866 100644
--- a/studio/backend/core/inference/_html_to_md.py
+++ b/studio/backend/core/inference/_html_to_md.py
@@ -7,6 +7,11 @@ Minimal HTML-to-Markdown converter using only the standard library.
Replaces the external ``html2text`` (GPL-3.0) dependency with a ~250-line
``html.parser.HTMLParser`` subclass. Covers headings, links, bold/italic,
lists, tables, blockquotes, code blocks, and entity decoding.
+
+``main_content=True`` also applies a readability-style heuristic: scope
+conversion to the page's ```` (else ````) subtree when it
+carries substantial text, and strip known boilerplate fragments (skip-links,
+error placeholders, session banners, cookie prompts) from the result.
"""
from __future__ import annotations
@@ -27,8 +32,138 @@ _SKIP_TAGS = frozenset(
"math",
"nav",
"footer",
+ # Never-rendered / form-chrome elements, not page content.
+ "template",
+ "dialog",
+ "button",
+ "select",
+ "datalist",
}
)
+# is NOT skipped: docs use it for admonition callouts (real content);
+# page-furniture asides are excluded by the main-content scoping pass instead.
+
+# Void elements never produce an end tag, so they must not join the
+# open-element stack used to bound hidden subtrees.
+_VOID_TAGS = frozenset(
+ {
+ "area",
+ "base",
+ "br",
+ "col",
+ "embed",
+ "hr",
+ "img",
+ "input",
+ "link",
+ "meta",
+ "param",
+ "source",
+ "track",
+ "wbr",
+ }
+)
+
+
+def _style_hides_element(style: str) -> bool:
+ """True when an inline ``style`` sets ``display:none`` / ``visibility:hidden``.
+
+ Parsed per property so an unrelated value that merely contains ``none`` is
+ not misread as hidden."""
+ lowered = style.lower()
+ if "none" not in lowered and "hidden" not in lowered:
+ return False
+ for declaration in style.split(";"):
+ prop, sep, value = declaration.partition(":")
+ if not sep:
+ continue
+ prop = prop.strip().lower()
+ # Drop any !important flag and keep the first token of the value.
+ value = value.split("!", 1)[0].strip().lower()
+ if prop == "display" and value == "none":
+ return True
+ if prop == "visibility" and value == "hidden":
+ return True
+ return False
+
+
+def _is_hidden_element(attr_dict: dict) -> bool:
+ """True when the element is not rendered: ``hidden`` attribute,
+ ``aria-hidden="true"``, or an inline ``style`` hiding it. Such JS-only
+ placeholders ship in the HTML but must not reach the output. ``hidden`` is
+ enumerated: any present value (even ``hidden="false"``) means not rendered."""
+ if "hidden" in attr_dict:
+ return True
+ if (attr_dict.get("aria-hidden") or "").strip().lower() == "true":
+ return True
+ return _style_hides_element(attr_dict.get("style") or "")
+
+
+# HTML5 optional end tags: a listed start tag implicitly closes an open element
+# of the key type (as browsers do), else an unclosed ````/``
``
+# swallows every following sibling. Keys: closable elements; values: closers.
+_P_CLOSING_TAGS = frozenset(
+ {
+ "address",
+ "article",
+ "aside",
+ "blockquote",
+ "details",
+ "div",
+ "dl",
+ "fieldset",
+ "figcaption",
+ "figure",
+ "footer",
+ "form",
+ "h1",
+ "h2",
+ "h3",
+ "h4",
+ "h5",
+ "h6",
+ "header",
+ "hgroup",
+ "hr",
+ "main",
+ "menu",
+ "nav",
+ "ol",
+ "p",
+ "pre",
+ "section",
+ "table",
+ "ul",
+ }
+)
+_IMPLICIT_CLOSERS: dict = {
+ "p": _P_CLOSING_TAGS,
+ "li": frozenset({"li"}),
+ "dt": frozenset({"dt", "dd"}),
+ "dd": frozenset({"dt", "dd"}),
+ "tr": frozenset({"tr"}),
+ "td": frozenset({"td", "th", "tr"}),
+ "th": frozenset({"td", "th", "tr"}),
+ "option": frozenset({"option", "optgroup"}),
+ "optgroup": frozenset({"optgroup"}),
+}
+
+
+# Item tag -> container tags that re-scope it: a nested container makes an inner
+# item a descendant, not an optional-close sibling, so recovery must stop there
+# rather than close (and un-hide) the outer item and leak its nested content.
+_CLOSE_BARRIERS: dict = {
+ "li": frozenset({"ul", "ol", "menu"}),
+ "dt": frozenset({"dl"}),
+ "dd": frozenset({"dl"}),
+ "tr": frozenset({"table"}),
+ "td": frozenset({"table"}),
+ "th": frozenset({"table"}),
+ "option": frozenset({"select", "datalist"}),
+ "optgroup": frozenset({"select", "datalist"}),
+}
+
+
_BLOCK_TAGS = frozenset(
{
"p",
@@ -51,13 +186,33 @@ _INLINE_EMPHASIS = {"strong": "**", "b": "**", "em": "*", "i": "*"}
class _MarkdownRenderer(HTMLParser):
- """HTMLParser subclass that emits Markdown tokens into a list."""
+ """HTMLParser subclass that emits Markdown tokens into a list.
- def __init__(self):
+ ``scope_tags`` restricts emission to the subtree(s) of the given tags
+ (e.g. ``{"article"}``): outside them every handler is a no-op, which is
+ how the readability-style main-content pass drops page furniture.
+ """
+
+ def __init__(self, scope_tags: frozenset[str] | None = None):
super().__init__(convert_charrefs = False)
self._out: list[str] = []
self._skip_depth: int = 0
+ # Main-content scoping: emit only while inside a scope tag.
+ self._scope_tags = scope_tags
+ self._scope_depth: int = 0
+
+ # Output boundaries per top-level scope element, so a caller can size each
+ # candidate alone and a swarm of tiny sibling cards can't clear the threshold.
+ self.scope_segments: list[str] = []
+ self._scope_seg_start: int | None = None
+
+ # Hidden-subtree tracking: stack of open non-void tags plus the indices
+ # where a hidden element started. End tags pop to the matching tag, so
+ # an omitted
/ close cannot leave the renderer stuck hidden.
+ self._open_tags: list[str] = []
+ self._hidden_marks: list[int] = []
+
# Link state
self._link_href: str | None = None
self._link_text_parts: list[str] = []
@@ -150,16 +305,95 @@ class _MarkdownRenderer(HTMLParser):
# ------------------------------------------------------------------
# Tag handlers
# ------------------------------------------------------------------
+ # Structural bookkeeping shared by every start tag (skip/hidden/scope).
+ def _close_implicit(self, tag: str) -> None:
+ """HTML5 optional-end-tag recovery for a start tag about to open.
+
+ Pops each implicitly-closed ancestor (and its hidden marks), scanning the
+ whole stack so an open ````/``
`` still closes under an unclosed inline
+ ````. Stops at a ``_CLOSE_BARRIERS`` container so recovery never crosses
+ a nested list/table/dl and leaks the outer item's hidden content. Runs even
+ for skipped ````/````, which also close ````."""
+ barriers = _CLOSE_BARRIERS.get(tag, ())
+ while True:
+ close_at = None
+ for i in range(len(self._open_tags) - 1, -1, -1):
+ name = self._open_tags[i]
+ if tag in _IMPLICIT_CLOSERS.get(name, ()):
+ close_at = i
+ break
+ # A barrier container re-scopes the item; stop before it.
+ if name in barriers:
+ break
+ if close_at is None:
+ break
+ del self._open_tags[close_at:]
+ while self._hidden_marks and self._hidden_marks[-1] >= close_at:
+ self._hidden_marks.pop()
+
+ def _enter_tag(self, tag: str, attr_dict: dict) -> bool:
+ """Track open/hidden/scope state; return True when the tag's content
+ should be rendered (False = suppressed). Caller runs ``_close_implicit``
+ first so recovery also fires for skipped tags."""
+ if tag not in _VOID_TAGS:
+ self._open_tags.append(tag)
+ if _is_hidden_element(attr_dict):
+ self._hidden_marks.append(len(self._open_tags) - 1)
+ elif _is_hidden_element(attr_dict):
+ # Void elements never join the stack, so suppress a hidden one inline.
+ return False
+ if self._scope_tags is not None and tag in self._scope_tags:
+ if self._scope_depth == 0:
+ self._scope_seg_start = len(self._out)
+ self._scope_depth += 1
+ if self._hidden_marks:
+ return False
+ if self._scope_tags is not None and self._scope_depth == 0:
+ return False
+ return True
+
+ def _exit_tag(self, tag: str) -> bool:
+ """Pop to the matching open tag; return True when the end tag should
+ be rendered (False = it closed inside a hidden / out-of-scope region)."""
+ suppressed = bool(self._hidden_marks) or (
+ self._scope_tags is not None and self._scope_depth == 0
+ )
+ if tag not in _VOID_TAGS:
+ # Pop to the innermost matching open tag (recovers omitted closes).
+ for i in range(len(self._open_tags) - 1, -1, -1):
+ if self._open_tags[i] == tag:
+ del self._open_tags[i:]
+ while self._hidden_marks and self._hidden_marks[-1] >= i:
+ self._hidden_marks.pop()
+ break
+ if self._scope_tags is not None and tag in self._scope_tags and self._scope_depth > 0:
+ self._scope_depth -= 1
+ if self._scope_depth == 0 and self._scope_seg_start is not None:
+ self.scope_segments.append("".join(self._out[self._scope_seg_start :]))
+ self._scope_seg_start = None
+ return not suppressed
+
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
tag = tag.lower()
+ if self._skip_depth:
+ # Inside a skipped subtree: only track nested skip depth.
+ if tag in _SKIP_TAGS:
+ self._skip_depth += 1
+ return
+
+ # Recover optional end tags before the skip decision: a skipped
+ # / still implicitly closes an open , releasing its
+ # hidden mark so following siblings render.
+ self._close_implicit(tag)
+
if tag in _SKIP_TAGS:
self._skip_depth += 1
return
- if self._skip_depth:
- return
attr_dict = dict(attrs)
+ if not self._enter_tag(tag, attr_dict):
+ return
if tag in _HEADING_TAGS:
level = int(tag[1])
@@ -250,6 +484,9 @@ class _MarkdownRenderer(HTMLParser):
if self._skip_depth:
return
+ if not self._exit_tag(tag):
+ return
+
if tag in _HEADING_TAGS:
self._emit("\n\n")
@@ -308,8 +545,13 @@ class _MarkdownRenderer(HTMLParser):
# ------------------------------------------------------------------
# Text / entity handlers
# ------------------------------------------------------------------
+ def _text_suppressed(self) -> bool:
+ if self._skip_depth or self._hidden_marks:
+ return True
+ return self._scope_tags is not None and self._scope_depth == 0
+
def handle_data(self, data: str) -> None:
- if self._skip_depth:
+ if self._text_suppressed():
return
if self._in_pre:
self._pre_parts.append(data)
@@ -326,12 +568,12 @@ class _MarkdownRenderer(HTMLParser):
self._emit(text)
def handle_entityref(self, name: str) -> None:
- if self._skip_depth:
+ if self._text_suppressed():
return
self._emit(html.unescape(f"&{name};"))
def handle_charref(self, name: str) -> None:
- if self._skip_depth:
+ if self._text_suppressed():
return
self._emit(html.unescape(f"{name};"))
@@ -366,6 +608,14 @@ class _MarkdownRenderer(HTMLParser):
else:
self._out.append("\n\n" + prefixed + "\n\n")
+ # A scope left open by truncated HTML never reached _exit_tag, so its output
+ # never joined scope_segments and would score 0. Flush the still-open segment
+ # here (after the side-buffers) so a truncated main-content page is scored.
+ if self._scope_seg_start is not None:
+ self.scope_segments.append("".join(self._out[self._scope_seg_start :]))
+ self._scope_seg_start = None
+ self._scope_depth = 0
+
# Post-processing
def _cleanup(text: str) -> str:
@@ -399,17 +649,124 @@ def _cleanup(text: str) -> str:
return "\n".join(out).strip()
-# Public API
-def html_to_markdown(source_html: str) -> str:
- """Convert HTML to Markdown (headings, links, emphasis, lists, tables, blockquotes, code, entities).
+# Known boilerplate fragments stripped from main-content conversions, matched
+# only against short lines. Sources: GitHub page furniture / client-side error
+# placeholders, skip-links, cookie banners.
+_BOILERPLATE_FRAGMENTS = (
+ "skip to content",
+ "skip to main content",
+ "there was an error while loading",
+ "please reload this page",
+ "you can't perform that action at this time",
+ "you signed in with another tab or window",
+ "you signed out in another tab or window",
+ "you switched accounts on another tab or window",
+ "reload to refresh your session",
+ "you must be signed in to change notification settings",
+ "uh oh!",
+ "{{ message }}",
+ "this website uses cookies",
+ "we use cookies",
+ "accept all cookies",
+ "manage cookie preferences",
+)
+# Only shorter lines are eligible for boilerplate dropping; real content
+# sentences quoting a fragment run longer.
+_BOILERPLATE_MAX_LINE_CHARS = 300
- ``