From 9807cdf039ec63ad638125b20dcf573eb2a18246 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 26 May 2026 12:53:39 +0000 Subject: [PATCH] Studio: surface per-call web_search sources, full shell command, exit_code, transparent image overlay Backend (external_provider.py): - Request `include=[web_search_call.action.sources, web_search_call.results]` so each web_search card has its consulted URLs and (for reasoning models) the search-result snippets attached. Format them as the Title/URL/Snippet blocks the frontend's source-pill parser already understands, so every search card now surfaces real sources instead of repeating the trigger label. - _format_shell_output now always emits exit_code (not just non-zero), reads stdout from `stdout`/`text`/`content` for tolerance across shell_call_output revisions, and dumps the raw entry dict as a fallback when no recognised text fields are present so the user still sees what OpenAI sent. Drops the literal "(no output)" string in favour of an empty result; the frontend renders a friendlier placeholder. Frontend: - tool-ui-code-execution.tsx: expanded card now renders the full untruncated command in its own labelled block above the output, so long heredocs clipped in the trigger are reachable. Empty output is rendered as italic "Command completed with no output." instead of showing nothing. - tool-ui-web-search.tsx: removed the 
 fallback that just echoed
  the trigger label back; non-search cards now link to the page URL
  when no sources are present.
- thread.tsx: image overlay container is now transparent (no muted
  background, no border ring); the image itself carries the rounded
  corners and the buttons float over its top-right corner so no
  yellow frame appears around narrow images.

Tests: 2 new web_search source-formatting tests; 66/66 pass.
---
 .../core/inference/external_provider.py       | 103 ++++++++++++++----
 .../test_openai_tool_result_fallbacks.py      |  55 ++++++++++
 .../src/components/assistant-ui/thread.tsx    |  18 ++-
 .../assistant-ui/tool-ui-code-execution.tsx   |  44 ++++++--
 .../assistant-ui/tool-ui-web-search.tsx       |  17 +--
 5 files changed, 191 insertions(+), 46 deletions(-)

diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py
index a28bced444..2e0c5e6256 100644
--- a/studio/backend/core/inference/external_provider.py
+++ b/studio/backend/core/inference/external_provider.py
@@ -276,6 +276,41 @@ def _extract_web_search_action(item: dict[str, Any]) -> dict[str, Any]:
     return out
 
 
+def _format_web_search_per_call_sources(
+    results: Any, sources: Any
+) -> str:
+    """Format per-call sources as the Title/URL/Snippet block the
+    frontend's parseSearchResults expects. Prefers `results` (snippet
+    bearing, reasoning models) then `action.sources` (urls only).
+    Returns "" when neither is populated.
+    """
+    blocks: list[str] = []
+    if isinstance(results, list):
+        for r in results:
+            if not isinstance(r, dict):
+                continue
+            url = r.get("url") if isinstance(r.get("url"), str) else ""
+            if not url:
+                continue
+            title = r.get("title") if isinstance(r.get("title"), str) else url
+            snippet = r.get("snippet") or r.get("text") or ""
+            entry = f"Title: {title}\nURL: {url}"
+            if isinstance(snippet, str) and snippet:
+                entry += f"\nSnippet: {snippet}"
+            blocks.append(entry)
+    if not blocks and isinstance(sources, list):
+        for s in sources:
+            url = ""
+            if isinstance(s, str):
+                url = s
+            elif isinstance(s, dict):
+                url = s.get("url") if isinstance(s.get("url"), str) else ""
+            if not url:
+                continue
+            blocks.append(f"Title: {url}\nURL: {url}")
+    return "\n---\n".join(blocks)
+
+
 def _web_search_card_text(args: dict[str, Any]) -> str:
     """Single-line summary for the per-card tool_end result."""
     if not args:
@@ -2235,7 +2270,7 @@ class ExternalProviderClient:
                             parts.append(f"--- stderr ---\n{stderr}")
                         if isinstance(return_code, int) and return_code != 0:
                             parts.append(f"return_code: {return_code}")
-                        return "\n".join(parts) if parts else "(no output)"
+                        return "\n".join(parts) if parts else ""
                     if inner_type == "text_editor_code_execution_result":
                         # view: file content; create: is_file_update flag;
                         # str_replace: diff `lines` list. The matching
@@ -3283,6 +3318,14 @@ class ExternalProviderClient:
                 tools_array.append(_openai_image_generation_tool())
             if tools_array:
                 body["tools"] = tools_array
+            # Opt into the per-call source list (action.sources) and the
+            # per-call snippet list (results, reasoning models only) so
+            # each web_search card can surface what was actually consulted.
+            if "web_search" in enabled_tools:
+                body["include"] = [
+                    "web_search_call.action.sources",
+                    "web_search_call.results",
+                ]
 
         url = f"{self.base_url}/responses"
         completion_id = f"chatcmpl-openai-{model.replace('/', '-')}"
@@ -3317,6 +3360,13 @@ class ExternalProviderClient:
                     attempt_body["tools"] = tools_array_attempt
                 else:
                     attempt_body.pop("tools", None)
+                if "web_search" in enabled_tools:
+                    attempt_body["include"] = [
+                        "web_search_call.action.sources",
+                        "web_search_call.results",
+                    ]
+                else:
+                    attempt_body.pop("include", None)
             return attempt_body
 
         def _is_openai_container_expired_error(error_text: str) -> bool:
@@ -3518,43 +3568,47 @@ class ExternalProviderClient:
                         return f"data: {_json.dumps(chunk)}"
 
                     def _format_shell_output(output: Any) -> str:
-                        """Render an OpenAI `shell_call_output.output` list
-                        as the preformatted text payload the frontend's
-                        CodeExecutionToolUI displays inside a 
. Each
-                        entry has stdout/stderr/outcome — concatenate them
-                        with a separator block per entry and append
-                        `return_code` / `(timeout)` annotations only when
-                        they convey information beyond "succeeded".
+                        """Render OpenAI `shell_call_output.output` for the
+                        CodeExecutionToolUI 
. Each entry has
+                        stdout/stderr/outcome (canonical) or `text`/`content`
+                        (older shapes). When the entry contains no
+                        recognised text fields at all, dump the raw JSON so
+                        the user can see what OpenAI actually returned.
                         """
                         if not isinstance(output, list):
                             return ""
                         parts: list[str] = []
                         for entry in output:
                             if not isinstance(entry, dict):
+                                if isinstance(entry, str) and entry:
+                                    parts.append(entry)
                                 continue
-                            stdout = entry.get("stdout") or ""
+                            stdout = entry.get("stdout") or entry.get("text") or entry.get("content") or ""
                             stderr = entry.get("stderr") or ""
                             outcome = entry.get("outcome") or {}
                             chunk_parts: list[str] = []
-                            if stdout:
+                            if isinstance(stdout, str) and stdout:
                                 chunk_parts.append(stdout)
-                            if stderr:
+                            if isinstance(stderr, str) and stderr:
                                 chunk_parts.append(f"--- stderr ---\n{stderr}")
                             if isinstance(outcome, dict):
                                 outcome_type = outcome.get("type")
                                 if outcome_type == "exit":
                                     exit_code = outcome.get("exit_code")
-                                    if isinstance(exit_code, int) and exit_code != 0:
-                                        chunk_parts.append(f"return_code: {exit_code}")
+                                    if isinstance(exit_code, int):
+                                        chunk_parts.append(f"exit_code: {exit_code}")
                                 elif outcome_type == "timeout":
                                     chunk_parts.append("(timeout)")
+                            if not chunk_parts:
+                                # Unknown shape: surface the raw dict so the
+                                # user can see what OpenAI actually returned.
+                                try:
+                                    chunk_parts.append(_json.dumps(entry, indent=2))
+                                except (TypeError, ValueError):
+                                    pass
                             if chunk_parts:
                                 parts.append("\n".join(chunk_parts))
-                        return (
-                            "\n--- next command ---\n".join(parts)
-                            if parts
-                            else "(no output)"
-                        )
+                        return "\n--- next command ---\n".join(parts)
 
                     def _record_url_citation(payload: dict[str, Any]) -> None:
                         """Append a url_citation onto the shared all_url_citations
@@ -3922,11 +3976,22 @@ class ExternalProviderClient:
                                             "arguments": args,
                                         }
                                     )
+                                    # Per-call sources: prefer the snippet-bearing
+                                    # `results` list (reasoning models only), then
+                                    # fall back to `action.sources` URLs. Formatted
+                                    # as the Title/URL/Snippet block the frontend
+                                    # parser already understands.
+                                    action_obj = (
+                                        item.get("action") if isinstance(item.get("action"), dict) else {}
+                                    ) or {}
+                                    per_call_sources = _format_web_search_per_call_sources(
+                                        item.get("results"), action_obj.get("sources")
+                                    )
                                     yield _emit_tool_event(
                                         {
                                             "type": "tool_end",
                                             "tool_call_id": item_id,
-                                            "result": _web_search_card_text(args),
+                                            "result": per_call_sources or _web_search_card_text(args),
                                         }
                                     )
                                 elif item.get("type") == "shell_call":
diff --git a/studio/backend/tests/test_openai_tool_result_fallbacks.py b/studio/backend/tests/test_openai_tool_result_fallbacks.py
index 5d8fa947bd..e9f8f2a308 100644
--- a/studio/backend/tests/test_openai_tool_result_fallbacks.py
+++ b/studio/backend/tests/test_openai_tool_result_fallbacks.py
@@ -274,6 +274,61 @@ def test_web_search_action_queries_plural_falls_back(monkeypatch):
     assert ends[0]["result"] == "Searching: renewable energy 2026"
 
 
+def test_web_search_per_call_results_formatted_as_source_blocks(monkeypatch):
+    """`results` array (reasoning models) is formatted into Title/URL/Snippet blocks."""
+    sse_events = [
+        {
+            "type": "response.output_item.done",
+            "item": {
+                "type": "web_search_call",
+                "id": "ws_r",
+                "action": {"type": "search", "query": "tiger ranking"},
+                "results": [
+                    {"url": "https://a.example/1", "title": "Tigers", "snippet": "Big cats"},
+                    {"url": "https://b.example/2", "title": "Lion stats"},
+                ],
+            },
+        },
+        {"type": "response.completed", "response": {}},
+    ]
+    lines = _drive_stream(sse_events, ["web_search"], monkeypatch)
+    events = _tool_events(lines)
+    ends = [e for e in events if e["type"] == "tool_end"]
+    body = ends[0]["result"]
+    assert "Title: Tigers" in body
+    assert "URL: https://a.example/1" in body
+    assert "Snippet: Big cats" in body
+    assert "Title: Lion stats" in body
+
+
+def test_web_search_action_sources_url_only_falls_back(monkeypatch):
+    """`action.sources` URLs are surfaced when `results` is absent."""
+    sse_events = [
+        {
+            "type": "response.output_item.done",
+            "item": {
+                "type": "web_search_call",
+                "id": "ws_s",
+                "action": {
+                    "type": "search",
+                    "query": "X",
+                    "sources": [
+                        {"type": "url", "url": "https://x.example/1"},
+                        "https://x.example/2",
+                    ],
+                },
+            },
+        },
+        {"type": "response.completed", "response": {}},
+    ]
+    lines = _drive_stream(sse_events, ["web_search"], monkeypatch)
+    events = _tool_events(lines)
+    ends = [e for e in events if e["type"] == "tool_end"]
+    body = ends[0]["result"]
+    assert "https://x.example/1" in body
+    assert "https://x.example/2" in body
+
+
 # ── shell_call output fallbacks ────────────────────────────────────────
 
 
diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsx
index 9c15b7fddd..1951922b74 100644
--- a/studio/frontend/src/components/assistant-ui/thread.tsx
+++ b/studio/frontend/src/components/assistant-ui/thread.tsx
@@ -230,9 +230,14 @@ const GeneratedImageViewportOverlay: FC<{ hideComposer?: boolean }> = ({
         )}
         aria-label="Generated image preview"
       >
-        

-          

-            

+        

+          

+            {overlay.title}
+            

               
             

           

-          

-            {overlay.title}
-          

           

       
             {runningLabel}
           

-        ) : resultText ? (
-          

-            

-              
-            

-            
-              {displayedResult}
-            

+        ) : (
+          

+            {showFullCommand ? (
+              

+                

+                  Command
+                  
+                

+                
+                  {command}
+                

+              

+            ) : null}
+            {resultText ? (
+              

+                

+                  Output
+                  
+                

+                
+                  {displayedResult}
+                

+              

+            ) : emptyOutput ? (
+              

+                Command completed with no output.
+              

+            ) : null}
           

-        ) : null}
+        )}
       
     
   );
diff --git a/studio/frontend/src/components/assistant-ui/tool-ui-web-search.tsx b/studio/frontend/src/components/assistant-ui/tool-ui-web-search.tsx
index f23cb8dc18..c65db443d5 100644
--- a/studio/frontend/src/components/assistant-ui/tool-ui-web-search.tsx
+++ b/studio/frontend/src/components/assistant-ui/tool-ui-web-search.tsx
@@ -154,14 +154,15 @@ const WebSearchToolUIImpl: ToolCallMessagePartComponent = ({
               
             ))}
           

-        ) : result ? (
-          

-            
-              {typeof result === "string"
-                ? result
-                : JSON.stringify(result, null, 2)}
-            

-          

+        ) : url ? (
+          
+            {url}
+          
         ) : null}