Studio: agentic web_search action variants + image overlay polish (OpenAI)
Backend: dispatch OpenAI Responses web_search_call on action.type
({search, open_page, find_in_page}) so agentic gpt-5.x calls render
meaningful per-card labels (`Read <url>`, `Find "<pattern>" in <url>`)
instead of empty quotes. Probe action.queries[0]/item.query as
fallbacks for older shapes. Backfill query from any prior event for the
same id.
Frontend: web-search card renders all three action variants in both
collapsed trigger and running states; falls back to `Searching...` when
nothing is known. Defensive scrubber in MarkdownText strips any
leftover U+E200/U+E201/U+E202 citation markers that survive the
backend rewriter (e.g. SSE dropped before the end-of-stream flush).
Image overlay: download + close buttons enlarged to size-10 with green
primary styling so they match the "Type edits below, then send" pill.
Image generation Show more/less now uses ChevronRight/Down icons to
match the other tool cards.
Tests: 3 new web_search action-variant tests; 55/55 pass.
This commit is contained in:
Daniel Han
parent
fb65fed3b0
commit
42159249ea
6 changed files with 192 additions and 44 deletions
|
|
@ -243,6 +243,52 @@ def _split_pending_citation_tail(text: str) -> tuple[str, str]:
|
|||
return text[:last_open], text[last_open:]
|
||||
|
||||
|
||||
def _extract_web_search_action(item: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Normalise web_search_call.action across variants. gpt-5.x agentic
|
||||
search emits action.type in {search, open_page, find_in_page}; older
|
||||
variants may put query at item.query or use action.queries[0].
|
||||
Returns only non-empty keys among query/url/pattern/action_type.
|
||||
"""
|
||||
if not isinstance(item, dict):
|
||||
return {}
|
||||
action = item.get("action") if isinstance(item.get("action"), dict) else {}
|
||||
atype = action.get("type") if isinstance(action.get("type"), str) else ""
|
||||
query = action.get("query") if isinstance(action.get("query"), str) else ""
|
||||
if not query:
|
||||
# Fallbacks: action.queries[0], item.query, item.queries[0].
|
||||
for src in (action.get("queries"), item.get("queries")):
|
||||
if isinstance(src, list) and src and isinstance(src[0], str) and src[0]:
|
||||
query = src[0]
|
||||
break
|
||||
if not query and isinstance(item.get("query"), str):
|
||||
query = item["query"]
|
||||
url = action.get("url") if isinstance(action.get("url"), str) else ""
|
||||
pattern = action.get("pattern") if isinstance(action.get("pattern"), str) else ""
|
||||
out: dict[str, Any] = {}
|
||||
if query: out["query"] = query
|
||||
if url: out["url"] = url
|
||||
if pattern: out["pattern"] = pattern
|
||||
if atype: out["action_type"] = atype
|
||||
return out
|
||||
|
||||
|
||||
def _web_search_card_text(args: dict[str, Any]) -> str:
|
||||
"""Single-line summary for the per-card tool_end result."""
|
||||
if not args:
|
||||
return ""
|
||||
q, u, p = args.get("query") or "", args.get("url") or "", args.get("pattern") or ""
|
||||
atype = args.get("action_type") or ""
|
||||
if atype == "open_page" and u:
|
||||
return f"Read: {u}"
|
||||
if atype == "find_in_page" and u:
|
||||
return f"Find {p!r} in {u}" if p else f"Find in {u}"
|
||||
if q:
|
||||
return f"Searching: {q}"
|
||||
if u:
|
||||
return f"Read: {u}"
|
||||
return ""
|
||||
|
||||
|
||||
class _AnthropicThinkingSpec(NamedTuple):
|
||||
prefixes: tuple[str, ...]
|
||||
kind: Literal["adaptive", "manual"]
|
||||
|
|
@ -3844,45 +3890,35 @@ class ExternalProviderClient:
|
|||
yield _chunk_with_text(summary_text)
|
||||
reasoning_emitted = True
|
||||
elif item.get("type") == "web_search_call":
|
||||
# done is the canonical place to read the
|
||||
# query, so emit both tool_start and tool_end
|
||||
# here. Frontend then renders a card per call
|
||||
# with the proper "Searching: <query>" label.
|
||||
# Citations are aggregated separately and the
|
||||
# *last* call's result is overwritten at
|
||||
# response.completed with the citation list
|
||||
# (so the source-pill extraction at message
|
||||
# tail surfaces them once).
|
||||
# Dispatch on action.type so open_page /
|
||||
# find_in_page render url+pattern instead
|
||||
# of an empty query card. Last call's
|
||||
# result is overwritten with citations at
|
||||
# response.completed.
|
||||
item_id = item.get("id", "") or (
|
||||
f"ws_{len(web_search_calls)}"
|
||||
)
|
||||
action = item.get("action")
|
||||
query = (
|
||||
action.get("query", "")
|
||||
if isinstance(action, dict)
|
||||
else ""
|
||||
)
|
||||
web_search_calls[item_id] = {"query": query}
|
||||
args = _extract_web_search_action(item)
|
||||
# Backfill query from any prior event for this id.
|
||||
if not args.get("query"):
|
||||
prior = web_search_calls.get(item_id) or {}
|
||||
prior_q = prior.get("query") if isinstance(prior, dict) else ""
|
||||
if isinstance(prior_q, str) and prior_q:
|
||||
args["query"] = prior_q
|
||||
web_search_calls[item_id] = dict(args)
|
||||
yield _emit_tool_event(
|
||||
{
|
||||
"type": "tool_start",
|
||||
"tool_name": "web_search",
|
||||
"tool_call_id": item_id,
|
||||
"arguments": (
|
||||
{"query": query} if query else {}
|
||||
),
|
||||
"arguments": args,
|
||||
}
|
||||
)
|
||||
# Per-card text; last call gets overwritten
|
||||
# with citations at response.completed.
|
||||
per_call_result = (
|
||||
f"Searching: {query}" if query else ""
|
||||
)
|
||||
yield _emit_tool_event(
|
||||
{
|
||||
"type": "tool_end",
|
||||
"tool_call_id": item_id,
|
||||
"result": per_call_result,
|
||||
"result": _web_search_card_text(args),
|
||||
}
|
||||
)
|
||||
elif item.get("type") == "shell_call":
|
||||
|
|
|
|||
|
|
@ -200,6 +200,80 @@ def test_web_search_empty_query_falls_back_to_empty_result(monkeypatch):
|
|||
assert ends[0]["result"] == ""
|
||||
|
||||
|
||||
def test_web_search_open_page_action_renders_url(monkeypatch):
|
||||
"""gpt-5.x agentic search emits `open_page` with a url (no query)."""
|
||||
sse_events = [
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"item": {
|
||||
"type": "web_search_call",
|
||||
"id": "ws_open",
|
||||
"action": {
|
||||
"type": "open_page",
|
||||
"url": "https://en.wikipedia.org/wiki/Tiger",
|
||||
},
|
||||
},
|
||||
},
|
||||
{"type": "response.completed", "response": {}},
|
||||
]
|
||||
lines = _drive_stream(sse_events, ["web_search"], monkeypatch)
|
||||
events = _tool_events(lines)
|
||||
starts = [e for e in events if e["type"] == "tool_start"]
|
||||
ends = [e for e in events if e["type"] == "tool_end"]
|
||||
assert starts[0]["arguments"]["url"] == "https://en.wikipedia.org/wiki/Tiger"
|
||||
assert starts[0]["arguments"]["action_type"] == "open_page"
|
||||
assert "Read: https://en.wikipedia.org/wiki/Tiger" in ends[0]["result"]
|
||||
|
||||
|
||||
def test_web_search_find_in_page_action_renders_url_and_pattern(monkeypatch):
|
||||
"""`find_in_page` actions surface both url and pattern to the card."""
|
||||
sse_events = [
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"item": {
|
||||
"type": "web_search_call",
|
||||
"id": "ws_find",
|
||||
"action": {
|
||||
"type": "find_in_page",
|
||||
"url": "https://en.wikipedia.org/wiki/Tiger",
|
||||
"pattern": "population",
|
||||
},
|
||||
},
|
||||
},
|
||||
{"type": "response.completed", "response": {}},
|
||||
]
|
||||
lines = _drive_stream(sse_events, ["web_search"], monkeypatch)
|
||||
events = _tool_events(lines)
|
||||
starts = [e for e in events if e["type"] == "tool_start"]
|
||||
ends = [e for e in events if e["type"] == "tool_end"]
|
||||
assert starts[0]["arguments"]["url"] == "https://en.wikipedia.org/wiki/Tiger"
|
||||
assert starts[0]["arguments"]["pattern"] == "population"
|
||||
assert starts[0]["arguments"]["action_type"] == "find_in_page"
|
||||
assert "population" in ends[0]["result"]
|
||||
assert "en.wikipedia.org" in ends[0]["result"]
|
||||
|
||||
|
||||
def test_web_search_action_queries_plural_falls_back(monkeypatch):
|
||||
"""`action.queries[0]` is used when `action.query` is absent (older shape)."""
|
||||
sse_events = [
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"item": {
|
||||
"type": "web_search_call",
|
||||
"id": "ws_plural",
|
||||
"action": {"queries": ["renewable energy 2026"]},
|
||||
},
|
||||
},
|
||||
{"type": "response.completed", "response": {}},
|
||||
]
|
||||
lines = _drive_stream(sse_events, ["web_search"], monkeypatch)
|
||||
events = _tool_events(lines)
|
||||
starts = [e for e in events if e["type"] == "tool_start"]
|
||||
ends = [e for e in events if e["type"] == "tool_end"]
|
||||
assert starts[0]["arguments"]["query"] == "renewable energy 2026"
|
||||
assert ends[0]["result"] == "Searching: renewable energy 2026"
|
||||
|
||||
|
||||
# ── shell_call output fallbacks ────────────────────────────────────────
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -411,9 +411,22 @@ function StreamdownBlock(props: BlockProps) {
|
|||
}
|
||||
const AUDIO_PLAYER_RE = /<audio-player\s+src="([^"]+)"\s*\/>/;
|
||||
|
||||
// Defensive scrub: strip any OpenAI cite markers that escape the
|
||||
// backend rewriter (e.g. SSE dropped before end-of-stream flush).
|
||||
const OPENAI_CITE_MARKER_RE = /cite[^]*/g;
|
||||
const OPENAI_PUA_ORPHAN_RE = /[]/g;
|
||||
function scrubOpenAICitationMarkers(text: string): string {
|
||||
if (!text) return text;
|
||||
if (!text.includes("") && !text.includes("")) return text;
|
||||
return text.replace(OPENAI_CITE_MARKER_RE, "").replace(OPENAI_PUA_ORPHAN_RE, "");
|
||||
}
|
||||
|
||||
const MarkdownTextImpl = () => {
|
||||
const { text, status } = useMessagePartText();
|
||||
const processedText = useMemo(() => preprocessLaTeX(text), [text]);
|
||||
const processedText = useMemo(
|
||||
() => preprocessLaTeX(scrubOpenAICitationMarkers(text)),
|
||||
[text],
|
||||
);
|
||||
|
||||
const audioMatch = text.match(AUDIO_PLAYER_RE);
|
||||
if (audioMatch) {
|
||||
|
|
|
|||
|
|
@ -232,12 +232,12 @@ const GeneratedImageViewportOverlay: FC<{ hideComposer?: boolean }> = ({
|
|||
>
|
||||
<div className="pointer-events-auto relative flex min-h-0 w-full max-w-[1100px] flex-1 flex-col items-center justify-center gap-3 rounded-3xl bg-muted/10 p-3 ring-1 ring-border/20">
|
||||
<div className="absolute inset-x-3 top-3 z-10 flex justify-end">
|
||||
<div className="flex shrink-0 items-center gap-1 rounded-full bg-background/70 p-1 ring-1 ring-border/20 backdrop-blur-sm">
|
||||
<div className="flex shrink-0 items-center gap-2 rounded-full bg-background/80 p-1.5 ring-1 ring-border/30 backdrop-blur-sm">
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
className="size-7 rounded-full"
|
||||
size="icon"
|
||||
className="size-10 rounded-full bg-primary/10 text-primary ring-1 ring-primary/30 hover:bg-primary/20 hover:text-primary"
|
||||
onClick={() =>
|
||||
downloadImagePart({
|
||||
image: overlay.image,
|
||||
|
|
@ -246,17 +246,17 @@ const GeneratedImageViewportOverlay: FC<{ hideComposer?: boolean }> = ({
|
|||
}
|
||||
aria-label="Download generated image"
|
||||
>
|
||||
<DownloadIcon className="size-3.5" />
|
||||
<DownloadIcon className="size-5" strokeWidth={2} />
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
className="size-7 rounded-full"
|
||||
size="icon"
|
||||
className="size-10 rounded-full bg-primary/10 text-primary ring-1 ring-primary/30 hover:bg-primary/20 hover:text-primary"
|
||||
onClick={closeOverlay}
|
||||
aria-label="Close generated image preview"
|
||||
>
|
||||
<XIcon className="size-3.5" />
|
||||
<XIcon className="size-5" strokeWidth={2} />
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -6,7 +6,13 @@
|
|||
import { Button } from "@/components/ui/button";
|
||||
import { cn } from "@/lib/utils";
|
||||
import type { ToolCallMessagePartComponent } from "@assistant-ui/react";
|
||||
import { DownloadIcon, ImageIcon, PencilIcon } from "lucide-react";
|
||||
import {
|
||||
ChevronDownIcon,
|
||||
ChevronRightIcon,
|
||||
DownloadIcon,
|
||||
ImageIcon,
|
||||
PencilIcon,
|
||||
} from "lucide-react";
|
||||
import type { CSSProperties, MouseEvent } from "react";
|
||||
import { memo, useCallback, useEffect, useRef, useState } from "react";
|
||||
import { useGeneratedImageOverlay } from "./generated-image-overlay-context";
|
||||
|
|
@ -348,7 +354,7 @@ const ImageGenerationToolUIImpl: ToolCallMessagePartComponent = ({
|
|||
{promptCanExpand ? (
|
||||
<button
|
||||
type="button"
|
||||
className="mt-2 inline-flex text-xs font-medium text-foreground/80 underline-offset-4 hover:text-foreground hover:underline focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-background"
|
||||
className="mt-2 inline-flex items-center gap-1 text-xs font-medium text-foreground/80 underline-offset-4 hover:text-foreground hover:underline focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-background"
|
||||
onClick={() =>
|
||||
setExpandedCaptionPrompt((value) =>
|
||||
value === captionPrompt ? null : captionPrompt,
|
||||
|
|
@ -356,6 +362,11 @@ const ImageGenerationToolUIImpl: ToolCallMessagePartComponent = ({
|
|||
}
|
||||
aria-expanded={promptExpanded}
|
||||
>
|
||||
{promptExpanded ? (
|
||||
<ChevronDownIcon className="size-3.5" />
|
||||
) : (
|
||||
<ChevronRightIcon className="size-3.5" />
|
||||
)}
|
||||
{promptExpanded ? "Show less" : "Show more"}
|
||||
</button>
|
||||
) : null}
|
||||
|
|
|
|||
|
|
@ -70,7 +70,11 @@ const WebSearchToolUIImpl: ToolCallMessagePartComponent = ({
|
|||
}) => {
|
||||
const query = (args as { query?: string })?.query ?? "";
|
||||
const url = ((args as { url?: string })?.url ?? "").trim();
|
||||
const isUrlFetch = !!url;
|
||||
const pattern = (args as { pattern?: string })?.pattern ?? "";
|
||||
const actionType = (args as { action_type?: string })?.action_type ?? "";
|
||||
// gpt-5.x agentic action.type: search | open_page | find_in_page.
|
||||
const isFindInPage = actionType === "find_in_page" || (!!url && !!pattern);
|
||||
const isUrlFetch = !!url && !isFindInPage;
|
||||
const displayDomain = (() => {
|
||||
if (!url) return "";
|
||||
try {
|
||||
|
|
@ -105,11 +109,15 @@ const WebSearchToolUIImpl: ToolCallMessagePartComponent = ({
|
|||
<ToolFallbackRoot open={open} onOpenChange={setOpen}>
|
||||
<ToolFallbackTrigger
|
||||
toolName={
|
||||
isUrlFetch
|
||||
? displayDomain ? `Read ${displayDomain}` : "Read page"
|
||||
: query
|
||||
? `Searched "${query}"`
|
||||
: "Web Search"
|
||||
isFindInPage
|
||||
? pattern
|
||||
? `Found "${pattern}" in ${displayDomain || "page"}`
|
||||
: `Find in ${displayDomain || "page"}`
|
||||
: isUrlFetch
|
||||
? displayDomain ? `Read ${displayDomain}` : "Read page"
|
||||
: query
|
||||
? `Searched "${query}"`
|
||||
: "Web Search"
|
||||
}
|
||||
status={status}
|
||||
icon={GlobeIcon}
|
||||
|
|
@ -119,9 +127,15 @@ const WebSearchToolUIImpl: ToolCallMessagePartComponent = ({
|
|||
<div className="flex items-center gap-2 text-sm text-muted-foreground">
|
||||
<LoaderIcon className="size-3.5 animate-spin" />
|
||||
<span>
|
||||
{isUrlFetch
|
||||
? <>Reading {displayDomain || "page"}…</>
|
||||
: <>Searching for “{query}”…</>
|
||||
{isFindInPage
|
||||
? pattern
|
||||
? <>Finding “{pattern}” in {displayDomain || "page"}…</>
|
||||
: <>Searching {displayDomain || "page"}…</>
|
||||
: isUrlFetch
|
||||
? <>Reading {displayDomain || "page"}…</>
|
||||
: query
|
||||
? <>Searching for “{query}”…</>
|
||||
: <>Searching…</>
|
||||
}
|
||||
</span>
|
||||
</div>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue