# SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 """Main-content extraction and boilerplate stripping for the web fetch tool. The HTML fixtures below snapshot the relevant fragments of a real GitHub repo page (github.com/unslothai/unsloth, fetched 2026-07): the ``hidden`` client-side error placeholders ("Uh oh! There was an error while loading."), the skip-link / nav / footer furniture, and the README rendered inside ``
``. No network access is required. """ from __future__ import annotations import sys from pathlib import Path import pytest _BACKEND_DIR = str(Path(__file__).resolve().parent.parent) if _BACKEND_DIR not in sys.path: sys.path.insert(0, _BACKEND_DIR) from core.inference._html_to_md import html_to_markdown from core.inference.tools import ( _fetch_page_text, _fetch_url_raw, _github_repo_readme_api_url, _looks_like_html, ) # ── Fixtures: snapshot of GitHub repo page fragments ───────────── # GitHub ships client-side error placeholders behind the `hidden` attribute (JS # reveals them on a failed fetch); a text converter must not surface them. _GITHUB_HIDDEN_ERROR_BLOCK = """ """ _GITHUB_PAGE = f""" unslothai/unsloth Skip to content
Sign in Sign up
{_GITHUB_HIDDEN_ERROR_BLOCK}
{_GITHUB_HIDDEN_ERROR_BLOCK}
unslothai / unsloth Notifications You must be signed in to change notification settings
NameLast commit message
unsloth

Unsloth Studio

Unsloth Studio lets you run and train models locally. Fine-tune and run LLMs on Windows, Linux and macOS with a single install command, then export to GGUF, Ollama, vLLM or Hugging Face when you are done.

Install

curl -fsSL https://unsloth.ai/install.sh | sh

See the documentation for quickstarts, notebooks, and fine-tuning guides for every major model family including Llama, Gemma, Qwen and DeepSeek.

Languages

""" # ── html_to_markdown: hidden elements ──────────────────────────── def test_hidden_attribute_subtree_is_dropped(): html = "

visible

after

" out = html_to_markdown(html) assert "visible" in out assert "after" in out assert "secret error text" not in out def test_aria_hidden_true_subtree_is_dropped(): html = '

keep

' out = html_to_markdown(html) assert "keep" in out assert "decoration" not in out def test_aria_hidden_false_subtree_is_kept(): html = 'still here' assert "still here" in html_to_markdown(html) def test_inline_style_display_none_subtree_is_dropped(): # Error/loading blocks are often hidden with inline CSS rather than the # ``hidden`` attribute; browsers do not render them, so they must not leak. html = ( "

visible

" '
secret loading block
' "

after

" ) out = html_to_markdown(html) assert "visible" in out assert "after" in out assert "secret loading block" not in out def test_inline_style_visibility_hidden_subtree_is_dropped(): html = '

keep

ghost' out = html_to_markdown(html) assert "keep" in out assert "ghost" not in out def test_inline_style_display_none_important_is_dropped(): # The !important flag must not defeat the display:none detection. html = '

keep

gone
' out = html_to_markdown(html) assert "keep" in out assert "gone" not in out def test_inline_style_display_none_among_other_declarations(): html = ( "

keep

" '
gone
' ) out = html_to_markdown(html) assert "keep" in out assert "gone" not in out def test_inline_style_visible_display_is_kept(): # Over-strip guard: display:block / visibility:visible render, and a value or # URL merely containing the substring "none" must not trigger the hidden path. html = ( "" '
block kept
' '
visible kept
' 'link kept' "" ) out = html_to_markdown(html) assert "block kept" in out assert "visible kept" in out assert "link kept" in out def test_hidden_recovers_from_omitted_close_tags(): #

kept

" out = html_to_markdown(html) assert "gone" not in out assert "kept" in out def test_nested_hidden_regions(): html = "

ok

" out = html_to_markdown(html) assert "inner" not in out assert "outer" not in out assert "ok" in out def test_hidden_false_is_still_hidden(): # ``hidden`` is enumerated: the spec maps invalid/empty values to the Hidden # state, so hidden="false" is NOT rendered and must not reach the Markdown. html = '

keep

' out = html_to_markdown(html) assert "keep" in out assert "not rendered" not in out def test_hidden_paragraph_omitted_close_does_not_swallow_siblings(): # HTML5 optional end tags: a sibling

start tag implicitly closes an open #

visible one

visible two

after

" ) out = html_to_markdown(html) assert "secret" not in out assert "visible one" in out assert "visible two" in out assert "after" in out def test_hidden_list_item_omitted_close_keeps_following_items(): # is implicitly closed by the next
  • . html = "" out = html_to_markdown(html) assert "secret" not in out assert "shown A" in out assert "shown B" in out def test_hr_implicitly_closes_hidden_paragraph(): # Void elements also imply closes:
    ends an open
    kept text" out = html_to_markdown(html) assert "secret" not in out assert "kept text" in out def test_skipped_tag_implicitly_closes_hidden_paragraph(): # A skipped block (