From cab3ed41e94b26f4dcb7491c8c99898e665e16ca Mon Sep 17 00:00:00 2001
From: Daniel Han <danielhanchen@gmail.com>
Date: Fri, 27 Mar 2026 13:34:41 +0000
Subject: [PATCH] fix: fetch top search result page content for web_search tool

The web_search tool previously returned only titles, URLs, and short
snippets from DuckDuckGo.  Small models would loop repeatedly trying
different search queries because the snippets never contained enough
data to answer the question.

Now fetches the actual page content for the top 1-2 search results
(best-effort, capped at 6000 chars and 10s timeout per fetch) and
includes it in the tool response.  This gives the model real data to
work with on the first search instead of endlessly retrying.

Uses only stdlib (urllib + re) so no new dependencies are needed.
---
 studio/backend/core/inference/tools.py | 47 ++++++++++++++++++++++++--
 1 file changed, 44 insertions(+), 3 deletions(-)

diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py
index 55bfa095f9..04fdae54cc 100644
--- a/studio/backend/core/inference/tools.py
+++ b/studio/backend/core/inference/tools.py
@@ -143,8 +143,36 @@ def execute_tool(
     return f"Unknown tool: {name}"
 
 
+def _fetch_page_text(url: str, max_chars: int = 4000, timeout: int = 10) -> str:
+    """Fetch a URL and extract plain text content (best-effort)."""
+    import urllib.request
+    import urllib.error
+    try:
+        req = urllib.request.Request(url, headers = {"User-Agent": "Mozilla/5.0"})
+        with urllib.request.urlopen(req, timeout = timeout) as resp:
+            raw = resp.read(200_000).decode("utf-8", errors = "replace")
+        # Strip HTML tags (lightweight, no extra deps)
+        import re
+        text = re.sub(r"<script[^>]*>.*?</script>", "", raw, flags = re.DOTALL | re.IGNORECASE)
+        text = re.sub(r"<style[^>]*>.*?</style>", "", text, flags = re.DOTALL | re.IGNORECASE)
+        text = re.sub(r"<[^>]+>", " ", text)
+        text = re.sub(r"\s+", " ", text).strip()
+        # Decode HTML entities
+        import html
+        text = html.unescape(text)
+        if len(text) > max_chars:
+            text = text[:max_chars] + "..."
+        return text
+    except Exception:
+        return ""
+
+
 def _web_search(query: str, max_results: int = 5, timeout: int = _EXEC_TIMEOUT) -> str:
-    """Search the web using DuckDuckGo and return formatted results."""
+    """Search the web using DuckDuckGo and return formatted results.
+
+    For the top result, also fetches the actual page content so the
+    model has real data to work with instead of just snippets.
+    """
     if not query.strip():
         return "No query provided."
     try:
@@ -153,13 +181,26 @@ def _web_search(query: str, max_results: int = 5, timeout: int = _EXEC_TIMEOUT)
         results = DDGS(timeout = timeout).text(query, max_results = max_results)
         if not results:
             return "No results found."
+
+        # Fetch full page content for the top result (best-effort,
+        # capped at 10s so it does not block the agentic loop).
+        top_page = ""
+        for r in results[:2]:
+            href = r.get("href", "")
+            if href:
+                top_page = _fetch_page_text(href, max_chars = 6000, timeout = 10)
+                if len(top_page) > 200:
+                    break
         parts = []
-        for r in results:
-            parts.append(
+        for i, r in enumerate(results):
+            entry = (
                 f"Title: {r.get('title', '')}\n"
                 f"URL: {r.get('href', '')}\n"
                 f"Snippet: {r.get('body', '')}"
             )
+            if i == 0 and top_page:
+                entry += f"\n\nPage content:\n{top_page}"
+            parts.append(entry)
         return "\n\n---\n\n".join(parts)
     except Exception as e:
         return f"Search failed: {e}"