From 42159249ea34c7484b4a37ee70229275426b2ec1 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 26 May 2026 12:35:52 +0000 Subject: [PATCH] Studio: agentic web_search action variants + image overlay polish (OpenAI) Backend: dispatch OpenAI Responses web_search_call on action.type ({search, open_page, find_in_page}) so agentic gpt-5.x calls render meaningful per-card labels (`Read `, `Find "" in `) instead of empty quotes. Probe action.queries[0]/item.query as fallbacks for older shapes. Backfill query from any prior event for the same id. Frontend: web-search card renders all three action variants in both collapsed trigger and running states; falls back to `Searching...` when nothing is known. Defensive scrubber in MarkdownText strips any leftover U+E200/U+E201/U+E202 citation markers that survive the backend rewriter (e.g. SSE dropped before the end-of-stream flush). Image overlay: download + close buttons enlarged to size-10 with green primary styling so they match the "Type edits below, then send" pill. Image generation Show more/less now uses ChevronRight/Down icons to match the other tool cards. Tests: 3 new web_search action-variant tests; 55/55 pass. --- .../core/inference/external_provider.py | 86 +++++++++++++------ .../test_openai_tool_result_fallbacks.py | 74 ++++++++++++++++ .../components/assistant-ui/markdown-text.tsx | 15 +++- .../src/components/assistant-ui/thread.tsx | 14 +-- .../assistant-ui/tool-ui-image-generation.tsx | 15 +++- .../assistant-ui/tool-ui-web-search.tsx | 32 +++++-- 6 files changed, 192 insertions(+), 44 deletions(-) diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index 8f34bb23fc..6968225475 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -243,6 +243,52 @@ def _split_pending_citation_tail(text: str) -> tuple[str, str]: return text[:last_open], text[last_open:] +def _extract_web_search_action(item: dict[str, Any]) -> dict[str, Any]: + """Normalise web_search_call.action across variants. gpt-5.x agentic + search emits action.type in {search, open_page, find_in_page}; older + variants may put query at item.query or use action.queries[0]. + Returns only non-empty keys among query/url/pattern/action_type. + """ + if not isinstance(item, dict): + return {} + action = item.get("action") if isinstance(item.get("action"), dict) else {} + atype = action.get("type") if isinstance(action.get("type"), str) else "" + query = action.get("query") if isinstance(action.get("query"), str) else "" + if not query: + # Fallbacks: action.queries[0], item.query, item.queries[0]. + for src in (action.get("queries"), item.get("queries")): + if isinstance(src, list) and src and isinstance(src[0], str) and src[0]: + query = src[0] + break + if not query and isinstance(item.get("query"), str): + query = item["query"] + url = action.get("url") if isinstance(action.get("url"), str) else "" + pattern = action.get("pattern") if isinstance(action.get("pattern"), str) else "" + out: dict[str, Any] = {} + if query: out["query"] = query + if url: out["url"] = url + if pattern: out["pattern"] = pattern + if atype: out["action_type"] = atype + return out + + +def _web_search_card_text(args: dict[str, Any]) -> str: + """Single-line summary for the per-card tool_end result.""" + if not args: + return "" + q, u, p = args.get("query") or "", args.get("url") or "", args.get("pattern") or "" + atype = args.get("action_type") or "" + if atype == "open_page" and u: + return f"Read: {u}" + if atype == "find_in_page" and u: + return f"Find {p!r} in {u}" if p else f"Find in {u}" + if q: + return f"Searching: {q}" + if u: + return f"Read: {u}" + return "" + + class _AnthropicThinkingSpec(NamedTuple): prefixes: tuple[str, ...] kind: Literal["adaptive", "manual"] @@ -3844,45 +3890,35 @@ class ExternalProviderClient: yield _chunk_with_text(summary_text) reasoning_emitted = True elif item.get("type") == "web_search_call": - # done is the canonical place to read the - # query, so emit both tool_start and tool_end - # here. Frontend then renders a card per call - # with the proper "Searching: " label. - # Citations are aggregated separately and the - # *last* call's result is overwritten at - # response.completed with the citation list - # (so the source-pill extraction at message - # tail surfaces them once). + # Dispatch on action.type so open_page / + # find_in_page render url+pattern instead + # of an empty query card. Last call's + # result is overwritten with citations at + # response.completed. item_id = item.get("id", "") or ( f"ws_{len(web_search_calls)}" ) - action = item.get("action") - query = ( - action.get("query", "") - if isinstance(action, dict) - else "" - ) - web_search_calls[item_id] = {"query": query} + args = _extract_web_search_action(item) + # Backfill query from any prior event for this id. + if not args.get("query"): + prior = web_search_calls.get(item_id) or {} + prior_q = prior.get("query") if isinstance(prior, dict) else "" + if isinstance(prior_q, str) and prior_q: + args["query"] = prior_q + web_search_calls[item_id] = dict(args) yield _emit_tool_event( { "type": "tool_start", "tool_name": "web_search", "tool_call_id": item_id, - "arguments": ( - {"query": query} if query else {} - ), + "arguments": args, } ) - # Per-card text; last call gets overwritten - # with citations at response.completed. - per_call_result = ( - f"Searching: {query}" if query else "" - ) yield _emit_tool_event( { "type": "tool_end", "tool_call_id": item_id, - "result": per_call_result, + "result": _web_search_card_text(args), } ) elif item.get("type") == "shell_call": diff --git a/studio/backend/tests/test_openai_tool_result_fallbacks.py b/studio/backend/tests/test_openai_tool_result_fallbacks.py index 7c033bc348..5d8fa947bd 100644 --- a/studio/backend/tests/test_openai_tool_result_fallbacks.py +++ b/studio/backend/tests/test_openai_tool_result_fallbacks.py @@ -200,6 +200,80 @@ def test_web_search_empty_query_falls_back_to_empty_result(monkeypatch): assert ends[0]["result"] == "" +def test_web_search_open_page_action_renders_url(monkeypatch): + """gpt-5.x agentic search emits `open_page` with a url (no query).""" + sse_events = [ + { + "type": "response.output_item.done", + "item": { + "type": "web_search_call", + "id": "ws_open", + "action": { + "type": "open_page", + "url": "https://en.wikipedia.org/wiki/Tiger", + }, + }, + }, + {"type": "response.completed", "response": {}}, + ] + lines = _drive_stream(sse_events, ["web_search"], monkeypatch) + events = _tool_events(lines) + starts = [e for e in events if e["type"] == "tool_start"] + ends = [e for e in events if e["type"] == "tool_end"] + assert starts[0]["arguments"]["url"] == "https://en.wikipedia.org/wiki/Tiger" + assert starts[0]["arguments"]["action_type"] == "open_page" + assert "Read: https://en.wikipedia.org/wiki/Tiger" in ends[0]["result"] + + +def test_web_search_find_in_page_action_renders_url_and_pattern(monkeypatch): + """`find_in_page` actions surface both url and pattern to the card.""" + sse_events = [ + { + "type": "response.output_item.done", + "item": { + "type": "web_search_call", + "id": "ws_find", + "action": { + "type": "find_in_page", + "url": "https://en.wikipedia.org/wiki/Tiger", + "pattern": "population", + }, + }, + }, + {"type": "response.completed", "response": {}}, + ] + lines = _drive_stream(sse_events, ["web_search"], monkeypatch) + events = _tool_events(lines) + starts = [e for e in events if e["type"] == "tool_start"] + ends = [e for e in events if e["type"] == "tool_end"] + assert starts[0]["arguments"]["url"] == "https://en.wikipedia.org/wiki/Tiger" + assert starts[0]["arguments"]["pattern"] == "population" + assert starts[0]["arguments"]["action_type"] == "find_in_page" + assert "population" in ends[0]["result"] + assert "en.wikipedia.org" in ends[0]["result"] + + +def test_web_search_action_queries_plural_falls_back(monkeypatch): + """`action.queries[0]` is used when `action.query` is absent (older shape).""" + sse_events = [ + { + "type": "response.output_item.done", + "item": { + "type": "web_search_call", + "id": "ws_plural", + "action": {"queries": ["renewable energy 2026"]}, + }, + }, + {"type": "response.completed", "response": {}}, + ] + lines = _drive_stream(sse_events, ["web_search"], monkeypatch) + events = _tool_events(lines) + starts = [e for e in events if e["type"] == "tool_start"] + ends = [e for e in events if e["type"] == "tool_end"] + assert starts[0]["arguments"]["query"] == "renewable energy 2026" + assert ends[0]["result"] == "Searching: renewable energy 2026" + + # ── shell_call output fallbacks ──────────────────────────────────────── diff --git a/studio/frontend/src/components/assistant-ui/markdown-text.tsx b/studio/frontend/src/components/assistant-ui/markdown-text.tsx index d2c6208fda..a4beff925c 100644 --- a/studio/frontend/src/components/assistant-ui/markdown-text.tsx +++ b/studio/frontend/src/components/assistant-ui/markdown-text.tsx @@ -411,9 +411,22 @@ function StreamdownBlock(props: BlockProps) { } const AUDIO_PLAYER_RE = //; +// Defensive scrub: strip any OpenAI cite markers that escape the +// backend rewriter (e.g. SSE dropped before end-of-stream flush). +const OPENAI_CITE_MARKER_RE = /cite[^]*/g; +const OPENAI_PUA_ORPHAN_RE = /[]/g; +function scrubOpenAICitationMarkers(text: string): string { + if (!text) return text; + if (!text.includes("") && !text.includes("")) return text; + return text.replace(OPENAI_CITE_MARKER_RE, "").replace(OPENAI_PUA_ORPHAN_RE, ""); +} + const MarkdownTextImpl = () => { const { text, status } = useMessagePartText(); - const processedText = useMemo(() => preprocessLaTeX(text), [text]); + const processedText = useMemo( + () => preprocessLaTeX(scrubOpenAICitationMarkers(text)), + [text], + ); const audioMatch = text.match(AUDIO_PLAYER_RE); if (audioMatch) { diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsx index da243f37e1..9c15b7fddd 100644 --- a/studio/frontend/src/components/assistant-ui/thread.tsx +++ b/studio/frontend/src/components/assistant-ui/thread.tsx @@ -232,12 +232,12 @@ const GeneratedImageViewportOverlay: FC<{ hideComposer?: boolean }> = ({ >
-
+
diff --git a/studio/frontend/src/components/assistant-ui/tool-ui-image-generation.tsx b/studio/frontend/src/components/assistant-ui/tool-ui-image-generation.tsx index 7dfdd903fe..6c054b31c1 100644 --- a/studio/frontend/src/components/assistant-ui/tool-ui-image-generation.tsx +++ b/studio/frontend/src/components/assistant-ui/tool-ui-image-generation.tsx @@ -6,7 +6,13 @@ import { Button } from "@/components/ui/button"; import { cn } from "@/lib/utils"; import type { ToolCallMessagePartComponent } from "@assistant-ui/react"; -import { DownloadIcon, ImageIcon, PencilIcon } from "lucide-react"; +import { + ChevronDownIcon, + ChevronRightIcon, + DownloadIcon, + ImageIcon, + PencilIcon, +} from "lucide-react"; import type { CSSProperties, MouseEvent } from "react"; import { memo, useCallback, useEffect, useRef, useState } from "react"; import { useGeneratedImageOverlay } from "./generated-image-overlay-context"; @@ -348,7 +354,7 @@ const ImageGenerationToolUIImpl: ToolCallMessagePartComponent = ({ {promptCanExpand ? ( ) : null} diff --git a/studio/frontend/src/components/assistant-ui/tool-ui-web-search.tsx b/studio/frontend/src/components/assistant-ui/tool-ui-web-search.tsx index b9bda2e832..f23cb8dc18 100644 --- a/studio/frontend/src/components/assistant-ui/tool-ui-web-search.tsx +++ b/studio/frontend/src/components/assistant-ui/tool-ui-web-search.tsx @@ -70,7 +70,11 @@ const WebSearchToolUIImpl: ToolCallMessagePartComponent = ({ }) => { const query = (args as { query?: string })?.query ?? ""; const url = ((args as { url?: string })?.url ?? "").trim(); - const isUrlFetch = !!url; + const pattern = (args as { pattern?: string })?.pattern ?? ""; + const actionType = (args as { action_type?: string })?.action_type ?? ""; + // gpt-5.x agentic action.type: search | open_page | find_in_page. + const isFindInPage = actionType === "find_in_page" || (!!url && !!pattern); + const isUrlFetch = !!url && !isFindInPage; const displayDomain = (() => { if (!url) return ""; try { @@ -105,11 +109,15 @@ const WebSearchToolUIImpl: ToolCallMessagePartComponent = ({ - {isUrlFetch - ? <>Reading {displayDomain || "page"}… - : <>Searching for “{query}”… + {isFindInPage + ? pattern + ? <>Finding “{pattern}” in {displayDomain || "page"}… + : <>Searching {displayDomain || "page"}… + : isUrlFetch + ? <>Reading {displayDomain || "page"}… + : query + ? <>Searching for “{query}”… + : <>Searching… }