ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[pre-commit.ci] auto fixes from pre-commit.com hooks

Browse source 
for more information, see https://pre-commit.ci
This commit is contained in:
pre-commit-ci[bot] 2026-03-27 13:35:01 +00:00
commit 1e66c83649
1 changed files with 9 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

11
studio/backend/core/inference/tools.py
View file

@ -147,18 +147,25 @@ def _fetch_page_text(url: str, max_chars: int = 4000, timeout: int = 10) -> str:
"""Fetch a URL and extract plain text content (best-effort)."""
import urllib.request
import urllib.error
try:
req = urllib.request.Request(url, headers = {"User-Agent": "Mozilla/5.0"})
with urllib.request.urlopen(req, timeout = timeout) as resp:
raw = resp.read(200_000).decode("utf-8", errors = "replace")
# Strip HTML tags (lightweight, no extra deps)
import re
text = re.sub(r"<script[^>]*>.*?</script>", "", raw, flags = re.DOTALL | re.IGNORECASE)
text = re.sub(r"<style[^>]*>.*?</style>", "", text, flags = re.DOTALL | re.IGNORECASE)
text = re.sub(
r"<script[^>]*>.*?</script>", "", raw, flags = re.DOTALL | re.IGNORECASE
)
text = re.sub(
r"<style[^>]*>.*?</style>", "", text, flags = re.DOTALL | re.IGNORECASE
)
text = re.sub(r"<[^>]+>", " ", text)
text = re.sub(r"\s+", " ", text).strip()
# Decode HTML entities
import html
text = html.unescape(text)
if len(text) > max_chars:
text = text[:max_chars] + "..."