Support follow-up edits for generated images (#5712)
* Support follow-up edits for generated images * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix generated image edit references * Fix image generation tool guard * Replay OpenAI image reasoning refs * Capture streamed OpenAI reasoning refs * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Trigger Gather Town test * Use OpenAI response context for image edits * Studio: improve generated image card UI * Studio: refine generated image overlay and edit context * Studio: animate generated image loading state * Studio: bind generated-image edits to selected image * Studio: drop empty external assistant payloads * Studio: guard generated image clipboard MIME * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Simplify image generation edit wiring * Remove temporary image generation test changes * Preserve explicit image edit references * Scope image edit references to threads * Harden OpenAI image edit replay handling * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update image generation tool event test --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
parent
542d74370f
commit
f364b08b6c
13 changed files with 1842 additions and 286 deletions
|
|
@ -67,6 +67,50 @@ _ANTHROPIC_4_7_SAMPLING_REMOVED = re.compile(
|
|||
r"^claude-(?:opus|sonnet|haiku)-4-7(?:[-.]|$)"
|
||||
)
|
||||
_OPENAI_REASONING_SUMMARY_UNSUPPORTED = re.compile(r"^o3(?:[-.]|$)")
|
||||
_OPENAI_REASONING_STATUSES = {"in_progress", "completed", "incomplete"}
|
||||
|
||||
|
||||
def _openai_image_replay_requires_reasoning(model: str) -> bool:
|
||||
normalized = model.strip().lower()
|
||||
return normalized.startswith("gpt-5") or normalized.startswith("o")
|
||||
|
||||
|
||||
def _sanitize_openai_reasoning_replay_item(
|
||||
item: Any,
|
||||
) -> Optional[dict[str, Any]]:
|
||||
"""Return a Responses input-safe reasoning item, if ``item`` is one.
|
||||
|
||||
OpenAI's image-generation docs allow follow-up edits by sending the
|
||||
previous ``image_generation_call`` id. Reasoning models can additionally
|
||||
require the paired ``reasoning`` output item in manually managed context,
|
||||
so keep the public replay fields only and drop everything else.
|
||||
"""
|
||||
if not isinstance(item, dict) or item.get("type") != "reasoning":
|
||||
return None
|
||||
item_id = item.get("id")
|
||||
if not isinstance(item_id, str) or not item_id:
|
||||
return None
|
||||
summary_parts: list[dict[str, str]] = []
|
||||
summary = item.get("summary")
|
||||
if isinstance(summary, list):
|
||||
for part in summary:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
if part.get("type") != "summary_text":
|
||||
continue
|
||||
text = part.get("text")
|
||||
if isinstance(text, str):
|
||||
summary_parts.append({"type": "summary_text", "text": text})
|
||||
replay_item: dict[str, Any] = {
|
||||
"type": "reasoning",
|
||||
"id": item_id,
|
||||
"summary": summary_parts,
|
||||
}
|
||||
status = item.get("status")
|
||||
if isinstance(status, str) and status in _OPENAI_REASONING_STATUSES:
|
||||
replay_item["status"] = status
|
||||
return replay_item
|
||||
|
||||
|
||||
# OpenAI Responses inline citation markers: `citeSOURCE_ID[id2...][LOCATOR]`
|
||||
# using private-use codepoints (see
|
||||
|
|
@ -733,8 +777,7 @@ class ExternalProviderClient:
|
|||
plugins.append({"id": "web"})
|
||||
body["plugins"] = plugins
|
||||
logger.info(
|
||||
"OpenRouter web_search: attached plugins=[{id: 'web'}] "
|
||||
"(model=%s)",
|
||||
"OpenRouter web_search: attached plugins=[{id: 'web'}] (model=%s)",
|
||||
body.get("model"),
|
||||
)
|
||||
|
||||
|
|
@ -1808,7 +1851,7 @@ class ExternalProviderClient:
|
|||
and compaction_threshold > 0
|
||||
and _anthropic_supports_compaction(model)
|
||||
)
|
||||
if compaction_active:
|
||||
if compaction_active and compaction_threshold is not None:
|
||||
trigger_value = max(
|
||||
int(compaction_threshold),
|
||||
_ANTHROPIC_COMPACTION_MIN,
|
||||
|
|
@ -2864,10 +2907,17 @@ class ExternalProviderClient:
|
|||
"""
|
||||
import json as _json
|
||||
|
||||
is_openai_cloud = _is_openai_family_cloud(self.base_url)
|
||||
image_generation_requested = bool(
|
||||
enabled_tools and "image_generation" in enabled_tools and is_openai_cloud
|
||||
)
|
||||
|
||||
# Split system messages out into a single `instructions` string and
|
||||
# translate user/assistant messages into the Responses input shape.
|
||||
instructions_parts: list[str] = []
|
||||
input_items: list[dict[str, Any]] = []
|
||||
openai_replay_items: list[dict[str, Any]] = []
|
||||
previous_response_id: Optional[str] = None
|
||||
for msg in messages:
|
||||
role = msg.get("role")
|
||||
content = msg.get("content", "")
|
||||
|
|
@ -2888,6 +2938,7 @@ class ExternalProviderClient:
|
|||
|
||||
if isinstance(content, list):
|
||||
translated_parts: list[dict[str, Any]] = []
|
||||
used_previous_response_id = False
|
||||
for part in content:
|
||||
part_type = part.get("type")
|
||||
if part_type == "text":
|
||||
|
|
@ -2902,6 +2953,36 @@ class ExternalProviderClient:
|
|||
translated_parts.append(
|
||||
{"type": "input_image", "image_url": url}
|
||||
)
|
||||
elif (
|
||||
part_type == "reasoning"
|
||||
and role == "assistant"
|
||||
and image_generation_requested
|
||||
):
|
||||
replay_item = _sanitize_openai_reasoning_replay_item(part)
|
||||
if replay_item:
|
||||
openai_replay_items.append(replay_item)
|
||||
elif (
|
||||
part_type == "image_generation_call"
|
||||
and role == "assistant"
|
||||
and image_generation_requested
|
||||
):
|
||||
response_id = (
|
||||
part.get("response_id")
|
||||
or part.get("openai_response_id")
|
||||
or part.get("previous_response_id")
|
||||
)
|
||||
call_id = part.get("id") or part.get("image_generation_call_id")
|
||||
if isinstance(call_id, str) and call_id:
|
||||
if isinstance(response_id, str) and response_id:
|
||||
previous_response_id = response_id
|
||||
input_items = []
|
||||
translated_parts = []
|
||||
used_previous_response_id = True
|
||||
else:
|
||||
previous_response_id = None
|
||||
openai_replay_items.append(
|
||||
{"type": "image_generation_call", "id": call_id}
|
||||
)
|
||||
elif part_type == "input_document":
|
||||
# OpenAI Responses accepts PDFs / docs as
|
||||
# `{type:"input_file", file_data:"data:application/pdf;base64,..."}`
|
||||
|
|
@ -2939,9 +3020,59 @@ class ExternalProviderClient:
|
|||
if filename:
|
||||
block["filename"] = filename
|
||||
translated_parts.append(block)
|
||||
if translated_parts:
|
||||
if translated_parts and not used_previous_response_id:
|
||||
input_items.append({"role": role, "content": translated_parts})
|
||||
|
||||
if previous_response_id:
|
||||
# OpenAI's documented multi-turn image generation path can use
|
||||
# `previous_response_id` to carry the prior generated image and
|
||||
# paired reasoning state. Prefer that over manual item replay when
|
||||
# we captured the response id; keep replay below as a fallback for
|
||||
# older stored turns that only have an image_generation_call id.
|
||||
openai_replay_items = []
|
||||
elif (
|
||||
_openai_image_replay_requires_reasoning(model)
|
||||
and reasoning_effort != "none"
|
||||
and enable_thinking is not False
|
||||
):
|
||||
filtered_replay_items: list[dict[str, Any]] = []
|
||||
has_reasoning_replay = False
|
||||
dropped_image_replay_without_reasoning = False
|
||||
for item in openai_replay_items:
|
||||
if item.get("type") == "reasoning":
|
||||
has_reasoning_replay = True
|
||||
filtered_replay_items.append(item)
|
||||
elif item.get("type") == "image_generation_call":
|
||||
if has_reasoning_replay:
|
||||
filtered_replay_items.append(item)
|
||||
else:
|
||||
dropped_image_replay_without_reasoning = True
|
||||
else:
|
||||
filtered_replay_items.append(item)
|
||||
openai_replay_items = filtered_replay_items
|
||||
if dropped_image_replay_without_reasoning:
|
||||
yield _error_sse_line(
|
||||
400,
|
||||
"OpenAI image edit reference is missing paired reasoning state. "
|
||||
"Regenerate the image, then retry the edit.",
|
||||
self.provider_type,
|
||||
)
|
||||
return
|
||||
image_generation_has_reference = bool(
|
||||
previous_response_id
|
||||
or any(
|
||||
isinstance(item, dict) and item.get("type") == "image_generation_call"
|
||||
for item in openai_replay_items
|
||||
)
|
||||
)
|
||||
if openai_replay_items:
|
||||
insert_at = len(input_items)
|
||||
for index in range(len(input_items) - 1, -1, -1):
|
||||
if input_items[index].get("role") == "user":
|
||||
insert_at = index
|
||||
break
|
||||
input_items[insert_at:insert_at] = openai_replay_items
|
||||
|
||||
# NOTE: gpt-5.x / o3 / gpt-4.5 are reasoning-class models. They reject
|
||||
# temperature and top_p with `Unsupported parameter` 400s on
|
||||
# /v1/responses (and on /v1/chat/completions for the same families).
|
||||
|
|
@ -2957,6 +3088,8 @@ class ExternalProviderClient:
|
|||
"input": input_items,
|
||||
"stream": True,
|
||||
}
|
||||
if previous_response_id:
|
||||
body["previous_response_id"] = previous_response_id
|
||||
# `summary: "auto"` is what makes /v1/responses emit reasoning
|
||||
# summary events — without it OpenAI returns no thinking text on
|
||||
# most reasoning models, the SSE handler has no <think>…</think>
|
||||
|
|
@ -3013,7 +3146,6 @@ class ExternalProviderClient:
|
|||
# (ollama / llama.cpp / vLLM / "custom" preset) hit /v1/responses
|
||||
# without these extensions and would 400 on the unknown body
|
||||
# fields, so they intentionally fall outside this gate.
|
||||
is_openai_cloud = _is_openai_family_cloud(self.base_url)
|
||||
if is_openai_cloud and enable_prompt_caching is not False:
|
||||
body["prompt_cache_retention"] = "24h"
|
||||
|
||||
|
|
@ -3059,9 +3191,18 @@ class ExternalProviderClient:
|
|||
# plus gpt-4.1 / gpt-4o / o3 per the docs; restrict to cloud
|
||||
# OpenAI because the local llama.cpp / ollama backends don't
|
||||
# implement it and would 400.
|
||||
image_generation_enabled_openai = bool(
|
||||
enabled_tools and "image_generation" in enabled_tools and is_openai_cloud
|
||||
)
|
||||
image_generation_enabled_openai = image_generation_requested
|
||||
|
||||
def _openai_image_generation_tool() -> dict[str, Any]:
|
||||
tool: dict[str, Any] = {"type": "image_generation"}
|
||||
if image_generation_has_reference:
|
||||
# OpenAI's Responses image tool defaults to `auto`. For
|
||||
# Studio's explicit follow-up edit flow, force edit mode so
|
||||
# the provider uses the previous response / call id as image
|
||||
# context instead of treating the text as a fresh generation.
|
||||
tool["action"] = "edit"
|
||||
return tool
|
||||
|
||||
if enabled_tools:
|
||||
tools_array: list[dict[str, Any]] = []
|
||||
if "web_search" in enabled_tools:
|
||||
|
|
@ -3089,7 +3230,7 @@ class ExternalProviderClient:
|
|||
shell_env = {"type": "container_auto"}
|
||||
tools_array.append({"type": "shell", "environment": shell_env})
|
||||
if image_generation_enabled_openai:
|
||||
tools_array.append({"type": "image_generation"})
|
||||
tools_array.append(_openai_image_generation_tool())
|
||||
if tools_array:
|
||||
body["tools"] = tools_array
|
||||
|
||||
|
|
@ -3121,7 +3262,7 @@ class ExternalProviderClient:
|
|||
{"type": "shell", "environment": env_attempt}
|
||||
)
|
||||
if image_generation_enabled_openai:
|
||||
tools_array_attempt.append({"type": "image_generation"})
|
||||
tools_array_attempt.append(_openai_image_generation_tool())
|
||||
if tools_array_attempt:
|
||||
attempt_body["tools"] = tools_array_attempt
|
||||
else:
|
||||
|
|
@ -3212,7 +3353,7 @@ class ExternalProviderClient:
|
|||
# to a specific search invocation. Hence the shared list.
|
||||
# web_search_calls: { item_id -> {query} }
|
||||
web_search_calls: dict[str, dict[str, Any]] = {}
|
||||
all_url_citations: list[dict[str, str]] = []
|
||||
all_url_citations: list[dict[str, Any]] = []
|
||||
# Shell-tool (code execution) state. OpenAI emits
|
||||
# `shell_call` items (model requesting a command list)
|
||||
# paired with `shell_call_output` items (execution
|
||||
|
|
@ -3235,6 +3376,10 @@ class ExternalProviderClient:
|
|||
# see.
|
||||
latched_container_id: Optional[str] = None
|
||||
container_id_emitted = False
|
||||
current_openai_response_id: Optional[str] = None
|
||||
last_openai_reasoning_replay_item: Optional[dict[str, Any]] = None
|
||||
openai_reasoning_replay_items: dict[str, dict[str, Any]] = {}
|
||||
image_generation_calls_started: set[str] = set()
|
||||
# Buffer for a citation marker straddling two delta events;
|
||||
# prepended onto the next delta. See _split_pending_citation_tail.
|
||||
pending_marker_tail: str = ""
|
||||
|
|
@ -3245,6 +3390,18 @@ class ExternalProviderClient:
|
|||
# with leftover private-use codepoints stripped.
|
||||
pending_citation_segments: list[str] = []
|
||||
|
||||
def _record_openai_response_id(payload: dict[str, Any]) -> None:
|
||||
nonlocal current_openai_response_id
|
||||
response_obj = payload.get("response")
|
||||
candidates: list[Any] = []
|
||||
if isinstance(response_obj, dict):
|
||||
candidates.append(response_obj.get("id"))
|
||||
candidates.append(payload.get("response_id"))
|
||||
for candidate in candidates:
|
||||
if isinstance(candidate, str) and candidate:
|
||||
current_openai_response_id = candidate
|
||||
return
|
||||
|
||||
def _drain_pending_segments(force: bool) -> str:
|
||||
"""Re-attempt resolution on buffered segments in order.
|
||||
Stops at the first still-unresolved segment unless
|
||||
|
|
@ -3392,6 +3549,80 @@ class ExternalProviderClient:
|
|||
}
|
||||
)
|
||||
|
||||
def _record_openai_reasoning_replay_item(
|
||||
payload: Any,
|
||||
) -> Optional[dict[str, Any]]:
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
item_id = payload.get("id") or payload.get("item_id")
|
||||
if not isinstance(item_id, str) or not item_id:
|
||||
return None
|
||||
existing = openai_reasoning_replay_items.setdefault(
|
||||
item_id,
|
||||
{
|
||||
"type": "reasoning",
|
||||
"id": item_id,
|
||||
"summary": [],
|
||||
"status": "completed",
|
||||
},
|
||||
)
|
||||
if payload.get("type") == "reasoning":
|
||||
sanitized = _sanitize_openai_reasoning_replay_item(payload)
|
||||
if sanitized:
|
||||
existing.update(sanitized)
|
||||
return existing
|
||||
summary_text = ""
|
||||
part = payload.get("part")
|
||||
if (
|
||||
isinstance(part, dict)
|
||||
and part.get("type") == "summary_text"
|
||||
):
|
||||
text = part.get("text")
|
||||
if isinstance(text, str):
|
||||
summary_text = text
|
||||
elif (
|
||||
payload.get("type")
|
||||
== "response.reasoning_summary_text.done"
|
||||
):
|
||||
text = payload.get("text")
|
||||
if isinstance(text, str):
|
||||
summary_text = text
|
||||
if summary_text:
|
||||
summary_index = payload.get("summary_index")
|
||||
summary = existing.setdefault("summary", [])
|
||||
if isinstance(summary, list):
|
||||
summary_part = {
|
||||
"type": "summary_text",
|
||||
"text": summary_text,
|
||||
}
|
||||
if (
|
||||
isinstance(summary_index, int)
|
||||
and summary_index >= 0
|
||||
):
|
||||
while len(summary) <= summary_index:
|
||||
summary.append(
|
||||
{"type": "summary_text", "text": ""}
|
||||
)
|
||||
summary[summary_index] = summary_part
|
||||
else:
|
||||
summary.append(summary_part)
|
||||
return existing
|
||||
|
||||
def _image_generation_arguments(
|
||||
prompt: str,
|
||||
raw_item_id: Any,
|
||||
) -> dict[str, Any]:
|
||||
arguments: dict[str, Any] = {"kind": "image", "prompt": prompt}
|
||||
if isinstance(raw_item_id, str) and raw_item_id:
|
||||
arguments["openai_image_generation_call_id"] = raw_item_id
|
||||
if current_openai_response_id:
|
||||
arguments["openai_response_id"] = current_openai_response_id
|
||||
if last_openai_reasoning_replay_item:
|
||||
arguments["openai_reasoning_item"] = (
|
||||
last_openai_reasoning_replay_item
|
||||
)
|
||||
return arguments
|
||||
|
||||
def _extract_reasoning_text(payload: Any) -> str:
|
||||
if payload is None:
|
||||
return ""
|
||||
|
|
@ -3478,6 +3709,7 @@ class ExternalProviderClient:
|
|||
continue
|
||||
|
||||
event_type = event.get("type")
|
||||
_record_openai_response_id(event)
|
||||
|
||||
if event_type == "response.output_text.delta":
|
||||
delta_text = event.get("delta", "")
|
||||
|
|
@ -3534,11 +3766,6 @@ class ExternalProviderClient:
|
|||
yield _chunk_with_text(flushed)
|
||||
|
||||
elif event_type == "response.output_item.added":
|
||||
# Track the call early but do NOT emit tool_start
|
||||
# yet — action.query is not reliably populated on
|
||||
# added across OpenAI API versions, and the
|
||||
# frontend's tool_start is a one-shot push (no
|
||||
# update mechanism). Wait for output_item.done.
|
||||
item = event.get("item", {})
|
||||
if (
|
||||
isinstance(item, dict)
|
||||
|
|
@ -3579,12 +3806,34 @@ class ExternalProviderClient:
|
|||
and latched_container_id is None
|
||||
):
|
||||
latched_container_id = probe
|
||||
if (
|
||||
isinstance(item, dict)
|
||||
and item.get("type") == "image_generation_call"
|
||||
):
|
||||
raw_item_id = item.get("id")
|
||||
if isinstance(raw_item_id, str) and raw_item_id:
|
||||
arguments = _image_generation_arguments(
|
||||
"",
|
||||
raw_item_id,
|
||||
)
|
||||
image_generation_calls_started.add(raw_item_id)
|
||||
yield _emit_tool_event(
|
||||
{
|
||||
"type": "tool_start",
|
||||
"tool_name": "image_generation",
|
||||
"tool_call_id": raw_item_id,
|
||||
"arguments": arguments,
|
||||
}
|
||||
)
|
||||
|
||||
elif event_type == "response.output_item.done":
|
||||
item = event.get("item", {})
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
if item.get("type") == "reasoning":
|
||||
last_openai_reasoning_replay_item = (
|
||||
_record_openai_reasoning_replay_item(item)
|
||||
)
|
||||
summary_text = _extract_reasoning_text(
|
||||
item.get("summary")
|
||||
)
|
||||
|
|
@ -3716,25 +3965,26 @@ class ExternalProviderClient:
|
|||
# millisecond resolution so synthesised
|
||||
# ids stay unique even when two image
|
||||
# generations resolve in the same ms.
|
||||
item_id = item.get("id", "") or (
|
||||
f"img_{time.time_ns()}"
|
||||
)
|
||||
raw_item_id = item.get("id")
|
||||
item_id = raw_item_id or f"img_{time.time_ns()}"
|
||||
prompt_in = (
|
||||
item.get("revised_prompt")
|
||||
or item.get("prompt")
|
||||
or ""
|
||||
)
|
||||
yield _emit_tool_event(
|
||||
{
|
||||
"type": "tool_start",
|
||||
"tool_name": "image_generation",
|
||||
"tool_call_id": item_id,
|
||||
"arguments": {
|
||||
"kind": "image",
|
||||
"prompt": prompt_in,
|
||||
},
|
||||
}
|
||||
done_arguments = _image_generation_arguments(
|
||||
prompt_in,
|
||||
raw_item_id,
|
||||
)
|
||||
if item_id not in image_generation_calls_started:
|
||||
yield _emit_tool_event(
|
||||
{
|
||||
"type": "tool_start",
|
||||
"tool_name": "image_generation",
|
||||
"tool_call_id": item_id,
|
||||
"arguments": done_arguments,
|
||||
}
|
||||
)
|
||||
b64 = (
|
||||
item.get("result") or item.get("b64_json") or ""
|
||||
)
|
||||
|
|
@ -3744,11 +3994,13 @@ class ExternalProviderClient:
|
|||
"type": "tool_end",
|
||||
"tool_call_id": item_id,
|
||||
"result": "",
|
||||
"arguments": done_arguments,
|
||||
"image_b64": b64,
|
||||
"image_mime": (f"image/{output_format}"),
|
||||
"size": item.get("size"),
|
||||
"quality": item.get("quality"),
|
||||
"background": item.get("background"),
|
||||
"prompt": prompt_in,
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -3756,6 +4008,13 @@ class ExternalProviderClient:
|
|||
isinstance(event_type, str)
|
||||
and "reasoning" in event_type
|
||||
):
|
||||
recorded_reasoning = (
|
||||
_record_openai_reasoning_replay_item(event)
|
||||
)
|
||||
if recorded_reasoning:
|
||||
last_openai_reasoning_replay_item = (
|
||||
recorded_reasoning
|
||||
)
|
||||
reasoning_delta = _extract_reasoning_text(event)
|
||||
if reasoning_delta:
|
||||
if not reasoning_open:
|
||||
|
|
@ -3848,8 +4107,7 @@ class ExternalProviderClient:
|
|||
blocks: list[str] = []
|
||||
for cit in all_url_citations:
|
||||
line = (
|
||||
f"Title: {cit['title']}\n"
|
||||
f"URL: {cit['url']}"
|
||||
f"Title: {cit['title']}\nURL: {cit['url']}"
|
||||
)
|
||||
if cit.get("snippet"):
|
||||
line += f"\nSnippet: {cit['snippet']}"
|
||||
|
|
@ -3927,8 +4185,7 @@ class ExternalProviderClient:
|
|||
blocks = []
|
||||
for cit in all_url_citations:
|
||||
line = (
|
||||
f"Title: {cit['title']}\n"
|
||||
f"URL: {cit['url']}"
|
||||
f"Title: {cit['title']}\nURL: {cit['url']}"
|
||||
)
|
||||
if cit.get("snippet"):
|
||||
line += f"\nSnippet: {cit['snippet']}"
|
||||
|
|
|
|||
|
|
@ -471,6 +471,40 @@ class InputDocumentContentPart(BaseModel):
|
|||
)
|
||||
|
||||
|
||||
class OpenAIReasoningContentPart(BaseModel):
|
||||
"""OpenAI Responses reasoning item paired with a tool output.
|
||||
|
||||
Reasoning models can require the previous ``reasoning`` output item
|
||||
to be replayed immediately before an ``image_generation_call`` id
|
||||
when manually managing Responses context. This part is OpenAI-only;
|
||||
routes strip it for every other provider before proxying.
|
||||
"""
|
||||
|
||||
type: Literal["reasoning"]
|
||||
id: str = Field(..., description = "OpenAI reasoning output item id.")
|
||||
summary: list[dict[str, Any]] = Field(default_factory = list)
|
||||
status: Optional[Literal["in_progress", "completed", "incomplete"]] = None
|
||||
|
||||
|
||||
class ImageGenerationCallContentPart(BaseModel):
|
||||
"""OpenAI Responses image_generation call reference.
|
||||
|
||||
OpenAI accepts prior ``image_generation_call`` items in the next
|
||||
Responses ``input`` array so follow-up prompts can edit or refine a
|
||||
generated image without resending the base64 payload. The frontend
|
||||
forwards this as a synthetic assistant content part when building
|
||||
the next OpenAI Responses request; ``external_provider`` translates
|
||||
it back to the provider-specific top-level input item.
|
||||
"""
|
||||
|
||||
type: Literal["image_generation_call"]
|
||||
id: str = Field(..., description = "OpenAI image_generation_call output item id.")
|
||||
response_id: Optional[str] = Field(
|
||||
None,
|
||||
description = "OpenAI Responses response id to use as previous_response_id for follow-up edits.",
|
||||
)
|
||||
|
||||
|
||||
class CompactionContentPart(BaseModel):
|
||||
"""Anthropic server-side compaction state, attached to an assistant
|
||||
message for round-tripping on the next turn.
|
||||
|
|
@ -504,6 +538,8 @@ ContentPart = Annotated[
|
|||
Annotated[TextContentPart, Tag("text")],
|
||||
Annotated[ImageContentPart, Tag("image_url")],
|
||||
Annotated[InputDocumentContentPart, Tag("input_document")],
|
||||
Annotated[OpenAIReasoningContentPart, Tag("reasoning")],
|
||||
Annotated[ImageGenerationCallContentPart, Tag("image_generation_call")],
|
||||
Annotated[CompactionContentPart, Tag("compaction")],
|
||||
],
|
||||
Discriminator(_content_part_discriminator),
|
||||
|
|
|
|||
|
|
@ -1709,6 +1709,12 @@ def _build_external_messages(
|
|||
see ``_INPUT_DOCUMENT_PROVIDERS``). For every other provider the
|
||||
part is stripped so the unknown content type doesn't reach generic
|
||||
/chat/completions passthrough and 400 the request.
|
||||
- `reasoning`: OpenAI-only Responses reasoning item paired with a
|
||||
prior tool output. Forwarded ONLY when provider_type=="openai"
|
||||
so follow-up image edits can replay the required reasoning item.
|
||||
- `image_generation_call`: OpenAI-only Responses image reference.
|
||||
Forwarded ONLY when provider_type=="openai" so follow-up image
|
||||
edits can reference prior generated images.
|
||||
- `compaction`: Anthropic-only synthetic part (round-trips server-side
|
||||
compaction state). Forwarded ONLY when provider_type=="anthropic";
|
||||
stripped for every other provider so the unknown part doesn't
|
||||
|
|
@ -1717,6 +1723,7 @@ def _build_external_messages(
|
|||
"""
|
||||
document_provider = provider_type in _INPUT_DOCUMENT_PROVIDERS
|
||||
anthropic = provider_type == "anthropic"
|
||||
openai = provider_type == "openai"
|
||||
result = []
|
||||
for msg in messages:
|
||||
if isinstance(msg.content, str):
|
||||
|
|
@ -1737,6 +1744,30 @@ def _build_external_messages(
|
|||
"image_url": {"url": part.image_url.url},
|
||||
}
|
||||
)
|
||||
elif (
|
||||
part.type == "reasoning" and openai and msg.role == "assistant"
|
||||
):
|
||||
reasoning: dict[str, Any] = {
|
||||
"type": "reasoning",
|
||||
"id": part.id,
|
||||
"summary": part.summary,
|
||||
}
|
||||
if part.status:
|
||||
reasoning["status"] = part.status
|
||||
parts.append(reasoning)
|
||||
elif (
|
||||
part.type == "image_generation_call"
|
||||
and openai
|
||||
and msg.role == "assistant"
|
||||
):
|
||||
# ExternalProviderClient maps this onto a top-level
|
||||
# Responses input item after the current user prompt,
|
||||
# or onto `previous_response_id` when response_id is
|
||||
# available from the prior Responses turn.
|
||||
image_ref = {"type": "image_generation_call", "id": part.id}
|
||||
if getattr(part, "response_id", None):
|
||||
image_ref["response_id"] = part.response_id
|
||||
parts.append(image_ref)
|
||||
elif part.type == "input_document" and document_provider:
|
||||
# ExternalProviderClient maps this onto
|
||||
# Anthropic's `document` or OpenAI Responses'
|
||||
|
|
@ -1758,6 +1789,8 @@ def _build_external_messages(
|
|||
# provider would 400 on the unknown part, so
|
||||
# gate by provider_type.
|
||||
parts.append({"type": "compaction", "content": part.content})
|
||||
if msg.role == "assistant" and not parts:
|
||||
continue
|
||||
result.append({"role": msg.role, "content": parts})
|
||||
else:
|
||||
# Non-vision provider: strip images / documents, keep
|
||||
|
|
@ -1769,8 +1802,28 @@ def _build_external_messages(
|
|||
for p in msg.content:
|
||||
if p.type == "text":
|
||||
preserved.append({"type": "text", "text": p.text})
|
||||
elif p.type == "reasoning" and openai and msg.role == "assistant":
|
||||
reasoning: dict[str, Any] = {
|
||||
"type": "reasoning",
|
||||
"id": p.id,
|
||||
"summary": p.summary,
|
||||
}
|
||||
if p.status:
|
||||
reasoning["status"] = p.status
|
||||
preserved.append(reasoning)
|
||||
elif (
|
||||
p.type == "image_generation_call"
|
||||
and openai
|
||||
and msg.role == "assistant"
|
||||
):
|
||||
image_ref = {"type": "image_generation_call", "id": p.id}
|
||||
if getattr(p, "response_id", None):
|
||||
image_ref["response_id"] = p.response_id
|
||||
preserved.append(image_ref)
|
||||
elif p.type == "compaction" and anthropic:
|
||||
preserved.append({"type": "compaction", "content": p.content})
|
||||
if msg.role == "assistant" and not preserved:
|
||||
continue
|
||||
if len(preserved) == 1 and preserved[0]["type"] == "text":
|
||||
# Single text part collapses back to a string for
|
||||
# providers that don't accept content arrays.
|
||||
|
|
|
|||
|
|
@ -210,6 +210,7 @@ def test_image_generation_done_emits_tool_event_chunks(monkeypatch):
|
|||
assert starts[0]["arguments"] == {
|
||||
"kind": "image",
|
||||
"prompt": "A photorealistic cat sitting",
|
||||
"openai_image_generation_call_id": "img_abc",
|
||||
}
|
||||
assert ends[0]["image_b64"] == "AAAA"
|
||||
assert ends[0]["image_mime"] == "image/png"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,77 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
"use client";
|
||||
|
||||
import {
|
||||
type ReactNode,
|
||||
createContext,
|
||||
useCallback,
|
||||
useContext,
|
||||
useMemo,
|
||||
useState,
|
||||
} from "react";
|
||||
|
||||
export type GeneratedImageOverlayState = {
|
||||
image: string;
|
||||
title: string;
|
||||
metadata: string;
|
||||
filename?: string;
|
||||
openaiImageGenerationCallId?: string;
|
||||
openaiResponseId?: string;
|
||||
openaiReasoningItem?: unknown;
|
||||
threadId?: string | null;
|
||||
};
|
||||
|
||||
type GeneratedImageOverlayContextValue = {
|
||||
overlay: GeneratedImageOverlayState | null;
|
||||
openOverlay: (overlay: GeneratedImageOverlayState) => void;
|
||||
closeOverlay: () => void;
|
||||
};
|
||||
|
||||
const GeneratedImageOverlayContext =
|
||||
createContext<GeneratedImageOverlayContextValue | null>(null);
|
||||
|
||||
export function GeneratedImageOverlayProvider({
|
||||
children,
|
||||
threadId = null,
|
||||
}: {
|
||||
children: ReactNode;
|
||||
threadId?: string | null;
|
||||
}) {
|
||||
const [overlay, setOverlay] = useState<GeneratedImageOverlayState | null>(
|
||||
null,
|
||||
);
|
||||
|
||||
const openOverlay = useCallback(
|
||||
(nextOverlay: GeneratedImageOverlayState) => {
|
||||
setOverlay({ ...nextOverlay, threadId: nextOverlay.threadId ?? threadId });
|
||||
},
|
||||
[threadId],
|
||||
);
|
||||
|
||||
const closeOverlay = useCallback(() => {
|
||||
setOverlay(null);
|
||||
}, []);
|
||||
|
||||
const value = useMemo(
|
||||
() => ({ overlay, openOverlay, closeOverlay }),
|
||||
[closeOverlay, openOverlay, overlay],
|
||||
);
|
||||
|
||||
return (
|
||||
<GeneratedImageOverlayContext.Provider value={value}>
|
||||
{children}
|
||||
</GeneratedImageOverlayContext.Provider>
|
||||
);
|
||||
}
|
||||
|
||||
export function useGeneratedImageOverlay(): GeneratedImageOverlayContextValue {
|
||||
const context = useContext(GeneratedImageOverlayContext);
|
||||
if (!context) {
|
||||
throw new Error(
|
||||
"useGeneratedImageOverlay must be used within GeneratedImageOverlayProvider.",
|
||||
);
|
||||
}
|
||||
return context;
|
||||
}
|
||||
510
studio/frontend/src/components/assistant-ui/image.tsx
Normal file
510
studio/frontend/src/components/assistant-ui/image.tsx
Normal file
|
|
@ -0,0 +1,510 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
//
|
||||
// Portions adapted from assistant-ui packages/ui/src/components/assistant-ui/image.tsx
|
||||
// MIT License, Copyright (c) 2025 AgentbaseAI Inc.
|
||||
// Source: https://github.com/assistant-ui/assistant-ui/blob/main/packages/ui/src/components/assistant-ui/image.tsx
|
||||
|
||||
"use client";
|
||||
|
||||
import { cn } from "@/lib/utils";
|
||||
import type {
|
||||
ImageMessagePart,
|
||||
ImageMessagePartComponent,
|
||||
} from "@assistant-ui/react";
|
||||
import { type VariantProps, cva } from "class-variance-authority";
|
||||
import {
|
||||
CopyIcon,
|
||||
DownloadIcon,
|
||||
ImageIcon,
|
||||
ImageOffIcon,
|
||||
Loader2Icon,
|
||||
RefreshCwIcon,
|
||||
ShieldAlertIcon,
|
||||
} from "lucide-react";
|
||||
import {
|
||||
type ComponentProps,
|
||||
type PropsWithChildren,
|
||||
memo,
|
||||
useEffect,
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import { createPortal } from "react-dom";
|
||||
|
||||
const extensionForMimeType = (mimeType?: string): string => {
|
||||
switch (mimeType) {
|
||||
case "image/png":
|
||||
return "png";
|
||||
case "image/jpeg":
|
||||
case "image/jpg":
|
||||
return "jpg";
|
||||
case "image/webp":
|
||||
return "webp";
|
||||
case "image/gif":
|
||||
return "gif";
|
||||
case "image/svg+xml":
|
||||
return "svg";
|
||||
default:
|
||||
return "png";
|
||||
}
|
||||
};
|
||||
|
||||
const DATA_URI_MIME_RE = /data:([^;]+)/;
|
||||
const DATA_URI_BASE64_RE = /;base64/i;
|
||||
const IMAGE_DATA_URI_MIME_RE = /^data:([^;,]+)/;
|
||||
|
||||
export const dataUriToBlob = (dataUri: string): Blob => {
|
||||
const [meta, data] = dataUri.split(",");
|
||||
const mime = meta?.match(DATA_URI_MIME_RE)?.[1] ?? "application/octet-stream";
|
||||
if (!DATA_URI_BASE64_RE.test(meta ?? "")) {
|
||||
return new Blob([decodeURIComponent(data ?? "")], { type: mime });
|
||||
}
|
||||
const bytes = atob(data ?? "");
|
||||
const arr = new Uint8Array(bytes.length);
|
||||
for (let i = 0; i < bytes.length; i += 1) {
|
||||
arr[i] = bytes.charCodeAt(i);
|
||||
}
|
||||
return new Blob([arr], { type: mime });
|
||||
};
|
||||
|
||||
const mimeFromImage = (image: string): string | undefined =>
|
||||
image.match(IMAGE_DATA_URI_MIME_RE)?.[1];
|
||||
|
||||
export const downloadImagePart = (
|
||||
part: Pick<ImageMessagePart, "image" | "filename">,
|
||||
): void => {
|
||||
if (typeof document === "undefined") {
|
||||
return;
|
||||
}
|
||||
const ext = extensionForMimeType(mimeFromImage(part.image));
|
||||
const filename = part.filename ?? `image.${ext}`;
|
||||
const isDataUri = part.image.startsWith("data:");
|
||||
const objectUrl = isDataUri
|
||||
? URL.createObjectURL(dataUriToBlob(part.image))
|
||||
: null;
|
||||
const href = objectUrl ?? part.image;
|
||||
const a = document.createElement("a");
|
||||
a.href = href;
|
||||
a.download = filename;
|
||||
a.rel = "noopener";
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
if (objectUrl) {
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
}
|
||||
};
|
||||
|
||||
export const copyImagePart = async (
|
||||
part: Pick<ImageMessagePart, "image">,
|
||||
): Promise<void> => {
|
||||
if (
|
||||
typeof navigator === "undefined" ||
|
||||
!navigator.clipboard ||
|
||||
typeof ClipboardItem === "undefined"
|
||||
) {
|
||||
throw new Error("Clipboard API is not available in this environment.");
|
||||
}
|
||||
const blob = part.image.startsWith("data:")
|
||||
? dataUriToBlob(part.image)
|
||||
: await fetch(part.image).then((r) => r.blob());
|
||||
const mime = mimeFromImage(part.image) || blob.type || "image/png";
|
||||
await navigator.clipboard.write([new ClipboardItem({ [mime]: blob })]);
|
||||
};
|
||||
|
||||
const reportImageCopyError = (error: unknown): void => {
|
||||
if (typeof window === "undefined") {
|
||||
return;
|
||||
}
|
||||
window.dispatchEvent(
|
||||
new CustomEvent("assistant-ui:image-copy-error", { detail: error }),
|
||||
);
|
||||
};
|
||||
|
||||
const imageVariants = cva(
|
||||
"aui-image-root relative overflow-hidden rounded-lg",
|
||||
{
|
||||
variants: {
|
||||
variant: {
|
||||
outline: "border border-border",
|
||||
ghost: "",
|
||||
muted: "bg-muted/50",
|
||||
},
|
||||
size: {
|
||||
sm: "max-w-64",
|
||||
default: "max-w-96",
|
||||
lg: "max-w-[512px]",
|
||||
full: "w-full",
|
||||
},
|
||||
},
|
||||
defaultVariants: {
|
||||
variant: "outline",
|
||||
size: "default",
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
export type ImageRootProps = ComponentProps<"div"> &
|
||||
VariantProps<typeof imageVariants>;
|
||||
|
||||
function ImageRoot({
|
||||
className,
|
||||
variant,
|
||||
size,
|
||||
children,
|
||||
...props
|
||||
}: ImageRootProps) {
|
||||
return (
|
||||
<div
|
||||
data-slot="image-root"
|
||||
data-variant={variant}
|
||||
data-size={size}
|
||||
className={cn(imageVariants({ variant, size, className }))}
|
||||
{...props}
|
||||
>
|
||||
{children}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
type ImagePreviewProps = Omit<ComponentProps<"img">, "children"> & {
|
||||
containerClassName?: string;
|
||||
};
|
||||
|
||||
function ImagePreview({
|
||||
className,
|
||||
containerClassName,
|
||||
onLoad,
|
||||
onError,
|
||||
alt = "Image content",
|
||||
src,
|
||||
...props
|
||||
}: ImagePreviewProps) {
|
||||
const imgRef = useRef<HTMLImageElement>(null);
|
||||
const [loadedSrc, setLoadedSrc] = useState<string | undefined>(undefined);
|
||||
const [errorSrc, setErrorSrc] = useState<string | undefined>(undefined);
|
||||
|
||||
const loaded = loadedSrc === src;
|
||||
const error = errorSrc === src;
|
||||
|
||||
useEffect(() => {
|
||||
if (
|
||||
typeof src === "string" &&
|
||||
imgRef.current?.complete &&
|
||||
imgRef.current.naturalWidth > 0
|
||||
) {
|
||||
setLoadedSrc(src);
|
||||
}
|
||||
}, [src]);
|
||||
|
||||
return (
|
||||
<div
|
||||
data-slot="image-preview"
|
||||
className={cn("relative min-h-32", containerClassName)}
|
||||
>
|
||||
{!(loaded || error) && (
|
||||
<div
|
||||
data-slot="image-preview-loading"
|
||||
className="absolute inset-0 flex items-center justify-center bg-muted/50"
|
||||
>
|
||||
<ImageIcon className="size-8 animate-pulse text-muted-foreground" />
|
||||
</div>
|
||||
)}
|
||||
{error ? (
|
||||
<div
|
||||
data-slot="image-preview-error"
|
||||
className="flex min-h-32 items-center justify-center bg-muted/50 p-4"
|
||||
>
|
||||
<ImageOffIcon className="size-8 text-muted-foreground" />
|
||||
</div>
|
||||
) : (
|
||||
<img
|
||||
{...props}
|
||||
ref={imgRef}
|
||||
src={src}
|
||||
alt={alt}
|
||||
className={cn(
|
||||
"block h-auto w-full object-contain",
|
||||
!loaded && "invisible",
|
||||
className,
|
||||
)}
|
||||
onLoad={(e) => {
|
||||
if (typeof src === "string") {
|
||||
setLoadedSrc(src);
|
||||
}
|
||||
onLoad?.(e);
|
||||
}}
|
||||
onError={(e) => {
|
||||
if (typeof src === "string") {
|
||||
setErrorSrc(src);
|
||||
}
|
||||
onError?.(e);
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ImageFilename({
|
||||
className,
|
||||
children,
|
||||
...props
|
||||
}: ComponentProps<"span">) {
|
||||
if (!children) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<span
|
||||
data-slot="image-filename"
|
||||
className={cn(
|
||||
"block truncate px-2 py-1.5 text-muted-foreground text-xs",
|
||||
className,
|
||||
)}
|
||||
{...props}
|
||||
>
|
||||
{children}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
type ImageZoomProps = PropsWithChildren<{
|
||||
src: string;
|
||||
alt?: string;
|
||||
}>;
|
||||
|
||||
function ImageZoom({ src, alt = "Image preview", children }: ImageZoomProps) {
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
|
||||
const handleOpen = () => setIsOpen(true);
|
||||
const handleClose = () => setIsOpen(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isOpen) {
|
||||
return;
|
||||
}
|
||||
const handleKeyDown = (e: KeyboardEvent) => {
|
||||
if (e.key === "Escape") {
|
||||
setIsOpen(false);
|
||||
}
|
||||
};
|
||||
document.addEventListener("keydown", handleKeyDown);
|
||||
return () => document.removeEventListener("keydown", handleKeyDown);
|
||||
}, [isOpen]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isOpen) {
|
||||
return;
|
||||
}
|
||||
const originalOverflow = document.body.style.overflow;
|
||||
document.body.style.overflow = "hidden";
|
||||
return () => {
|
||||
document.body.style.overflow = originalOverflow;
|
||||
};
|
||||
}, [isOpen]);
|
||||
|
||||
return (
|
||||
<>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleOpen}
|
||||
className="aui-image-zoom-trigger w-full cursor-zoom-in border-0 bg-transparent p-0 text-left"
|
||||
aria-label="Click to zoom image"
|
||||
>
|
||||
{children}
|
||||
</button>
|
||||
{isOpen &&
|
||||
typeof document !== "undefined" &&
|
||||
createPortal(
|
||||
<button
|
||||
type="button"
|
||||
data-slot="image-zoom-overlay"
|
||||
className="aui-image-zoom-overlay fade-in fixed inset-0 z-50 flex animate-in items-center justify-center border-0 bg-black/80 p-0 duration-200"
|
||||
onClick={handleClose}
|
||||
aria-label="Close zoomed image"
|
||||
>
|
||||
<img
|
||||
data-slot="image-zoom-content"
|
||||
src={src}
|
||||
alt={alt}
|
||||
className="aui-image-zoom-content fade-in zoom-in-95 max-h-[90vh] max-w-[90vw] animate-in cursor-zoom-out object-contain duration-200"
|
||||
/>
|
||||
</button>,
|
||||
document.body,
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function ImageGenerating({ className }: { className?: string }) {
|
||||
return (
|
||||
<div
|
||||
data-slot="image-generating"
|
||||
className={cn(
|
||||
"flex min-h-32 items-center justify-center bg-muted/50 p-4",
|
||||
className,
|
||||
)}
|
||||
>
|
||||
<Loader2Icon className="size-8 animate-spin text-muted-foreground" />
|
||||
<span className="sr-only">Generating image…</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ImageContentFilterError({
|
||||
className,
|
||||
reason,
|
||||
}: {
|
||||
className?: string;
|
||||
reason?: string;
|
||||
}) {
|
||||
return (
|
||||
<div
|
||||
data-slot="image-content-filter-error"
|
||||
className={cn(
|
||||
"flex min-h-32 flex-col items-center justify-center gap-2 bg-muted/50 p-4 text-center",
|
||||
className,
|
||||
)}
|
||||
>
|
||||
<ShieldAlertIcon className="size-8 text-muted-foreground" />
|
||||
<p className="font-medium text-sm">Image could not be generated</p>
|
||||
{reason && <p className="text-muted-foreground text-xs">{reason}</p>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export type ImageActionsProps = {
|
||||
part: ImageMessagePart;
|
||||
/**
|
||||
* Wire to your own generation call to show a regenerate button. The button
|
||||
* renders only when this is set and the part carries a `prompt`.
|
||||
*/
|
||||
onRegenerate?: () => void | Promise<void>;
|
||||
className?: string;
|
||||
};
|
||||
|
||||
function RegenerateButton({
|
||||
onRegenerate,
|
||||
}: {
|
||||
onRegenerate: () => void | Promise<void>;
|
||||
}) {
|
||||
const [isRegenerating, setIsRegenerating] = useState(false);
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
onClick={async () => {
|
||||
setIsRegenerating(true);
|
||||
try {
|
||||
await onRegenerate();
|
||||
} finally {
|
||||
setIsRegenerating(false);
|
||||
}
|
||||
}}
|
||||
disabled={isRegenerating}
|
||||
data-slot="image-regenerate"
|
||||
aria-label="Regenerate image"
|
||||
className="inline-flex size-7 items-center justify-center rounded hover:bg-muted disabled:opacity-50"
|
||||
>
|
||||
<RefreshCwIcon
|
||||
className={cn("size-4", isRegenerating && "animate-spin")}
|
||||
/>
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
function ImageActions({ part, onRegenerate, className }: ImageActionsProps) {
|
||||
return (
|
||||
<div
|
||||
data-slot="image-actions"
|
||||
className={cn("flex items-center gap-1 p-1", className)}
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => downloadImagePart(part)}
|
||||
data-slot="image-download"
|
||||
aria-label="Download image"
|
||||
className="inline-flex size-7 items-center justify-center rounded hover:bg-muted"
|
||||
>
|
||||
<DownloadIcon className="size-4" />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
copyImagePart(part).catch((error) => {
|
||||
reportImageCopyError(error);
|
||||
});
|
||||
}}
|
||||
data-slot="image-copy"
|
||||
aria-label="Copy image"
|
||||
className="inline-flex size-7 items-center justify-center rounded hover:bg-muted"
|
||||
>
|
||||
<CopyIcon className="size-4" />
|
||||
</button>
|
||||
{onRegenerate && <RegenerateButton onRegenerate={onRegenerate} />}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const ImageImpl: ImageMessagePartComponent = (props) => {
|
||||
const { image, filename, status } = props;
|
||||
const alt = filename || "Image content";
|
||||
|
||||
if (status?.type === "running") {
|
||||
return (
|
||||
<ImageRoot>
|
||||
<ImageGenerating />
|
||||
<ImageFilename>{filename}</ImageFilename>
|
||||
</ImageRoot>
|
||||
);
|
||||
}
|
||||
|
||||
if (status?.type === "incomplete" && status.reason === "content-filter") {
|
||||
return (
|
||||
<ImageRoot>
|
||||
<ImageContentFilterError reason="The provider blocked this image." />
|
||||
</ImageRoot>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<ImageRoot>
|
||||
<ImageZoom src={image} alt={alt}>
|
||||
<ImagePreview src={image} alt={alt} />
|
||||
</ImageZoom>
|
||||
<ImageFilename>{filename}</ImageFilename>
|
||||
</ImageRoot>
|
||||
);
|
||||
};
|
||||
|
||||
const Image = memo(ImageImpl) as unknown as ImageMessagePartComponent & {
|
||||
Root: typeof ImageRoot;
|
||||
Preview: typeof ImagePreview;
|
||||
Filename: typeof ImageFilename;
|
||||
Zoom: typeof ImageZoom;
|
||||
Actions: typeof ImageActions;
|
||||
Generating: typeof ImageGenerating;
|
||||
ContentFilterError: typeof ImageContentFilterError;
|
||||
};
|
||||
|
||||
Image.displayName = "Image";
|
||||
Image.Root = ImageRoot;
|
||||
Image.Preview = ImagePreview;
|
||||
Image.Filename = ImageFilename;
|
||||
Image.Zoom = ImageZoom;
|
||||
Image.Actions = ImageActions;
|
||||
Image.Generating = ImageGenerating;
|
||||
Image.ContentFilterError = ImageContentFilterError;
|
||||
|
||||
export {
|
||||
Image,
|
||||
ImageRoot,
|
||||
ImagePreview,
|
||||
ImageFilename,
|
||||
ImageZoom,
|
||||
ImageActions,
|
||||
ImageGenerating,
|
||||
ImageContentFilterError,
|
||||
imageVariants,
|
||||
};
|
||||
|
|
@ -7,6 +7,11 @@ import {
|
|||
UserMessageAttachments,
|
||||
} from "@/components/assistant-ui/attachment";
|
||||
import { CodeToggleIcon } from "@/components/assistant-ui/code-toggle-icon";
|
||||
import {
|
||||
GeneratedImageOverlayProvider,
|
||||
useGeneratedImageOverlay,
|
||||
} from "@/components/assistant-ui/generated-image-overlay-context";
|
||||
import { downloadImagePart } from "@/components/assistant-ui/image";
|
||||
import { MarkdownText } from "@/components/assistant-ui/markdown-text";
|
||||
import { MessageTiming } from "@/components/assistant-ui/message-timing";
|
||||
import { Reasoning, ReasoningGroup } from "@/components/assistant-ui/reasoning";
|
||||
|
|
@ -39,13 +44,14 @@ import {
|
|||
import { sentAudioNames } from "@/features/chat/api/chat-adapter";
|
||||
import { parseExternalModelId } from "@/features/chat/external-providers";
|
||||
import { getExternalReasoningCapabilities } from "@/features/chat/provider-capabilities";
|
||||
import { useExternalProvidersStore } from "@/features/chat/stores/external-providers-store";
|
||||
import { useChatRuntimeStore } from "@/features/chat/stores/chat-runtime-store";
|
||||
import { useExternalProvidersStore } from "@/features/chat/stores/external-providers-store";
|
||||
import { deleteThreadMessage } from "@/features/chat/utils/delete-thread-message";
|
||||
import { applyQwenThinkingParams } from "@/features/chat/utils/qwen-params";
|
||||
import { isTauri } from "@/lib/api-base";
|
||||
import { deleteThreadMessage } from "@/features/chat/utils/delete-thread-message";
|
||||
import { AUDIO_ACCEPT, MAX_AUDIO_SIZE, fileToBase64 } from "@/lib/audio-utils";
|
||||
import { copyToClipboard } from "@/lib/copy-to-clipboard";
|
||||
import { toast } from "@/lib/toast";
|
||||
import { cn } from "@/lib/utils";
|
||||
import {
|
||||
ActionBarMorePrimitive,
|
||||
|
|
@ -79,30 +85,30 @@ import {
|
|||
TerminalIcon,
|
||||
XIcon,
|
||||
} from "lucide-react";
|
||||
import { Copy01Icon, Delete02Icon, Edit03Icon, Tick02Icon } from "@hugeicons/core-free-icons";
|
||||
import {
|
||||
Copy01Icon,
|
||||
Delete02Icon,
|
||||
Edit03Icon,
|
||||
Tick02Icon,
|
||||
} from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import {
|
||||
type ChangeEvent,
|
||||
type ComponentProps,
|
||||
type CompositionEvent,
|
||||
type FC,
|
||||
type FormEvent,
|
||||
type KeyboardEvent,
|
||||
useCallback,
|
||||
useEffect,
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import { toast } from "@/lib/toast";
|
||||
|
||||
export const Thread: FC<{
|
||||
hideComposer?: boolean;
|
||||
hideWelcome?: boolean;
|
||||
targetThreadId?: string;
|
||||
}> = ({
|
||||
hideComposer,
|
||||
hideWelcome,
|
||||
targetThreadId,
|
||||
}) => {
|
||||
}> = ({ hideComposer, hideWelcome, targetThreadId }) => {
|
||||
// Intent-aware autoscroll: replaces assistant-ui's built-in autoscroll
|
||||
// to prevent the streaming-mutation race that makes the viewport snap
|
||||
// back to the bottom while the user is scrolling up (see the hook for
|
||||
|
|
@ -113,85 +119,204 @@ export const Thread: FC<{
|
|||
const isComposerAttachPending = useAuiState(({ threads }) =>
|
||||
targetThreadId ? threads.mainThreadId !== targetThreadId : false,
|
||||
);
|
||||
const activeThreadId = useChatRuntimeStore((s) => s.activeThreadId);
|
||||
const threadId = targetThreadId ?? activeThreadId ?? null;
|
||||
|
||||
return (
|
||||
<ThreadPrimitive.Root
|
||||
className="aui-root aui-thread-root @container relative flex min-h-0 min-w-0 flex-1 basis-0 flex-col overflow-hidden"
|
||||
style={{
|
||||
["--thread-max-width" as string]: "48rem",
|
||||
["--thread-content-max-width" as string]:
|
||||
"calc(var(--thread-max-width) - 1.5rem)",
|
||||
}}
|
||||
>
|
||||
<IntentAwareScrollProvider value={autoScrollContext}>
|
||||
<ThreadPrimitive.Viewport
|
||||
ref={viewportRef}
|
||||
autoScroll={false}
|
||||
scrollToBottomOnRunStart={false}
|
||||
scrollToBottomOnInitialize={false}
|
||||
scrollToBottomOnThreadSwitch={false}
|
||||
className={cn(
|
||||
"aui-thread-viewport aui-stream-viewport relative flex min-h-0 min-w-0 flex-1 basis-0 flex-col overflow-x-auto overflow-y-auto scroll-smooth px-5",
|
||||
hideComposer ? "pt-4" : "pt-[48px]",
|
||||
)}
|
||||
>
|
||||
{!hideWelcome && (
|
||||
<AuiIf condition={({ thread }) => thread.isEmpty && !thread.isLoading}>
|
||||
<ThreadWelcome hideComposer={hideComposer} />
|
||||
</AuiIf>
|
||||
)}
|
||||
<GeneratedImageOverlayProvider key={threadId ?? "default"} threadId={threadId}>
|
||||
<ThreadPrimitive.Root
|
||||
className="aui-root aui-thread-root @container relative flex min-h-0 min-w-0 flex-1 basis-0 flex-col overflow-hidden"
|
||||
style={{
|
||||
["--thread-max-width" as string]: "48rem",
|
||||
["--thread-content-max-width" as string]:
|
||||
"calc(var(--thread-max-width) - 1.5rem)",
|
||||
}}
|
||||
>
|
||||
<IntentAwareScrollProvider value={autoScrollContext}>
|
||||
<ThreadPrimitive.Viewport
|
||||
ref={viewportRef}
|
||||
autoScroll={false}
|
||||
scrollToBottomOnRunStart={false}
|
||||
scrollToBottomOnInitialize={false}
|
||||
scrollToBottomOnThreadSwitch={false}
|
||||
className={cn(
|
||||
"aui-thread-viewport aui-stream-viewport relative flex min-h-0 min-w-0 flex-1 basis-0 flex-col overflow-x-auto overflow-y-auto scroll-smooth px-5",
|
||||
hideComposer ? "pt-4" : "pt-[48px]",
|
||||
)}
|
||||
>
|
||||
{!hideWelcome && (
|
||||
<AuiIf
|
||||
condition={({ thread }) => thread.isEmpty && !thread.isLoading}
|
||||
>
|
||||
<ThreadWelcome hideComposer={hideComposer} threadId={threadId} />
|
||||
</AuiIf>
|
||||
)}
|
||||
|
||||
<ThreadPrimitive.Messages
|
||||
components={{
|
||||
UserMessage,
|
||||
EditComposer,
|
||||
AssistantMessage,
|
||||
}}
|
||||
/>
|
||||
<ThreadPrimitive.Messages
|
||||
components={{
|
||||
UserMessage,
|
||||
EditComposer,
|
||||
AssistantMessage,
|
||||
}}
|
||||
/>
|
||||
|
||||
{/* Bottom slack so the last message has breathing room above the
|
||||
{/* Bottom slack so the last message has breathing room above the
|
||||
sticky scroll-to-bottom button (and the floating composer in
|
||||
single mode). Without this, content would butt against the
|
||||
sticky footer and feel cramped. */}
|
||||
<AuiIf condition={({ thread }) => hideWelcome || !thread.isEmpty}>
|
||||
<div
|
||||
className={cn("shrink-0", hideComposer ? "h-16" : "h-40")}
|
||||
aria-hidden={true}
|
||||
/>
|
||||
</AuiIf>
|
||||
|
||||
<AuiIf condition={({ thread }) => hideWelcome || !thread.isEmpty}>
|
||||
<ThreadPrimitive.ViewportFooter
|
||||
className={cn(
|
||||
"aui-thread-viewport-footer pointer-events-none sticky z-20 flex w-full justify-center bg-transparent",
|
||||
hideComposer ? "bottom-3" : "bottom-[140px]",
|
||||
)}
|
||||
>
|
||||
<ThreadScrollToBottom />
|
||||
</ThreadPrimitive.ViewportFooter>
|
||||
</AuiIf>
|
||||
</ThreadPrimitive.Viewport>
|
||||
|
||||
{!hideComposer && (
|
||||
<AuiIf condition={({ thread }) => hideWelcome || !thread.isEmpty}>
|
||||
<div className="aui-thread-composer-dock pointer-events-none absolute bottom-0 left-0 right-0 md:right-[10px] z-20">
|
||||
<AuiIf condition={({ thread }) => hideWelcome || !thread.isEmpty}>
|
||||
<div
|
||||
className={cn("shrink-0", hideComposer ? "h-16" : "h-40")}
|
||||
aria-hidden={true}
|
||||
className="absolute inset-x-0 bottom-0 top-[10px] bg-background"
|
||||
/>
|
||||
<div className="relative px-5 pb-2">
|
||||
<div className="pointer-events-auto mx-auto w-full max-w-(--thread-max-width)">
|
||||
<ComposerAnimated disabled={isComposerAttachPending} />
|
||||
</div>
|
||||
<p className="composer-footer-note">
|
||||
LLMs can make mistakes. Double-check responses.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</AuiIf>
|
||||
</AuiIf>
|
||||
|
||||
<AuiIf condition={({ thread }) => hideWelcome || !thread.isEmpty}>
|
||||
<ThreadPrimitive.ViewportFooter
|
||||
className={cn(
|
||||
"aui-thread-viewport-footer pointer-events-none sticky z-20 flex w-full justify-center bg-transparent",
|
||||
hideComposer ? "bottom-3" : "bottom-[140px]",
|
||||
)}
|
||||
>
|
||||
<ThreadScrollToBottom />
|
||||
</ThreadPrimitive.ViewportFooter>
|
||||
</AuiIf>
|
||||
</ThreadPrimitive.Viewport>
|
||||
|
||||
<GeneratedImageViewportOverlay hideComposer={hideComposer} />
|
||||
|
||||
{!hideComposer && (
|
||||
<AuiIf condition={({ thread }) => hideWelcome || !thread.isEmpty}>
|
||||
<ThreadComposerDock
|
||||
disabled={isComposerAttachPending}
|
||||
threadId={threadId}
|
||||
/>
|
||||
</AuiIf>
|
||||
)}
|
||||
</IntentAwareScrollProvider>
|
||||
</ThreadPrimitive.Root>
|
||||
</GeneratedImageOverlayProvider>
|
||||
);
|
||||
};
|
||||
|
||||
const GeneratedImageViewportOverlay: FC<{ hideComposer?: boolean }> = ({
|
||||
hideComposer,
|
||||
}) => {
|
||||
const { overlay, closeOverlay } = useGeneratedImageOverlay();
|
||||
|
||||
useEffect(() => {
|
||||
if (!overlay) {
|
||||
return;
|
||||
}
|
||||
document.querySelector<HTMLTextAreaElement>(".aui-composer-input")?.focus();
|
||||
}, [overlay]);
|
||||
|
||||
if (!overlay) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="pointer-events-none absolute inset-0 z-30">
|
||||
<button
|
||||
type="button"
|
||||
className="pointer-events-auto absolute inset-0 bg-background/65 backdrop-blur-[1px] dark:bg-background/55"
|
||||
onClick={closeOverlay}
|
||||
aria-label="Close generated image preview"
|
||||
/>
|
||||
<section
|
||||
className={cn(
|
||||
"pointer-events-none absolute inset-x-5 top-[48px] flex flex-col items-center",
|
||||
hideComposer ? "bottom-4" : "bottom-[150px]",
|
||||
)}
|
||||
</IntentAwareScrollProvider>
|
||||
</ThreadPrimitive.Root>
|
||||
aria-label="Generated image preview"
|
||||
>
|
||||
<div className="pointer-events-auto relative flex min-h-0 w-full max-w-[1100px] flex-1 flex-col items-center justify-center gap-3 rounded-3xl bg-muted/10 p-3 ring-1 ring-border/20">
|
||||
<div className="absolute inset-x-3 top-3 z-10 flex justify-end">
|
||||
<div className="flex shrink-0 items-center gap-1 rounded-full bg-background/70 p-1 ring-1 ring-border/20 backdrop-blur-sm">
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
className="size-7 rounded-full"
|
||||
onClick={() =>
|
||||
downloadImagePart({
|
||||
image: overlay.image,
|
||||
filename: overlay.filename,
|
||||
})
|
||||
}
|
||||
aria-label="Download generated image"
|
||||
>
|
||||
<DownloadIcon className="size-3.5" />
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
className="size-7 rounded-full"
|
||||
onClick={closeOverlay}
|
||||
aria-label="Close generated image preview"
|
||||
>
|
||||
<XIcon className="size-3.5" />
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex min-h-0 flex-1 items-center justify-center pt-1">
|
||||
<img
|
||||
src={overlay.image}
|
||||
alt={overlay.title}
|
||||
className="max-h-full max-w-full object-contain"
|
||||
/>
|
||||
</div>
|
||||
<div
|
||||
className="w-full max-w-[min(100%,46rem)] shrink-0 text-center"
|
||||
title={overlay.title}
|
||||
>
|
||||
<p className="truncate font-medium text-foreground/70 text-xs">
|
||||
Generated image
|
||||
</p>
|
||||
{overlay.metadata ? (
|
||||
<p className="truncate text-[11px] text-muted-foreground/75">
|
||||
{overlay.metadata}
|
||||
</p>
|
||||
) : null}
|
||||
{hideComposer ? null : (
|
||||
<p className="mt-1 text-muted-foreground text-xs">
|
||||
Type edits below, then send.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
const ThreadComposerDock: FC<{
|
||||
disabled?: boolean;
|
||||
threadId?: string | null;
|
||||
}> = ({ disabled, threadId }) => {
|
||||
const { overlay } = useGeneratedImageOverlay();
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
"aui-thread-composer-dock pointer-events-none absolute bottom-0 left-0 right-0 md:right-[10px]",
|
||||
overlay ? "z-40" : "z-20",
|
||||
)}
|
||||
>
|
||||
<div
|
||||
aria-hidden={true}
|
||||
className="absolute inset-x-0 bottom-0 top-[10px] bg-background"
|
||||
/>
|
||||
<div className="relative px-5 pb-2">
|
||||
<div className="pointer-events-auto mx-auto w-full max-w-(--thread-max-width)">
|
||||
<ComposerAnimated disabled={disabled} threadId={threadId} />
|
||||
</div>
|
||||
<p className="composer-footer-note">
|
||||
LLMs can make mistakes. Double-check responses.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
|
|
@ -219,13 +344,17 @@ const ThreadScrollToBottom: FC = () => {
|
|||
);
|
||||
};
|
||||
|
||||
const ThreadWelcome: FC<{ hideComposer?: boolean }> = ({ hideComposer }) => {
|
||||
const ThreadWelcome: FC<{
|
||||
hideComposer?: boolean;
|
||||
threadId?: string | null;
|
||||
}> = ({ hideComposer, threadId }) => {
|
||||
const [currentEmoji, setCurrentEmoji] = useState("large sloth drink.png");
|
||||
|
||||
useEffect(() => {
|
||||
const hour = new Date().getHours();
|
||||
if (hour >= 6 && hour < 12) setCurrentEmoji("large sloth drink.png");
|
||||
else if (hour >= 12 && hour < 17) setCurrentEmoji("sloth magnify final.png");
|
||||
else if (hour >= 12 && hour < 17)
|
||||
setCurrentEmoji("sloth magnify final.png");
|
||||
else if (hour >= 17 && hour < 21) setCurrentEmoji("sloth shy large.png");
|
||||
else setCurrentEmoji("unsloth-gem.png");
|
||||
}, []);
|
||||
|
|
@ -240,11 +369,7 @@ const ThreadWelcome: FC<{ hideComposer?: boolean }> = ({ hideComposer }) => {
|
|||
<div className="aui-thread-welcome-center flex w-full grow flex-col items-center justify-center pb-[48px]">
|
||||
<div className="aui-thread-welcome-message flex w-full flex-col justify-center gap-6 px-4">
|
||||
<div className="flex flex-col items-center gap-2 text-center">
|
||||
<img
|
||||
src={currentEmojiSrc}
|
||||
alt="Sloth mascot"
|
||||
className="size-20"
|
||||
/>
|
||||
<img src={currentEmojiSrc} alt="Sloth mascot" className="size-20" />
|
||||
<h1 className="aui-thread-welcome-message-inner fade-in slide-in-from-bottom-1 animate-in font-heading font-semibold text-2xl tracking-[-0.02em] duration-200">
|
||||
Chat with your model
|
||||
</h1>
|
||||
|
|
@ -252,18 +377,21 @@ const ThreadWelcome: FC<{ hideComposer?: boolean }> = ({ hideComposer }) => {
|
|||
Run GGUFs, safetensors, vision and audio models
|
||||
</p>
|
||||
</div>
|
||||
{!hideComposer && <ComposerAnimated />}
|
||||
{!hideComposer && <ComposerAnimated threadId={threadId} />}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
const ComposerAnimated: FC<{ disabled?: boolean }> = ({ disabled }) => {
|
||||
const ComposerAnimated: FC<{
|
||||
disabled?: boolean;
|
||||
threadId?: string | null;
|
||||
}> = ({ disabled, threadId }) => {
|
||||
return (
|
||||
<div className="relative mx-auto min-w-0 w-full max-w-(--thread-max-width)">
|
||||
<div className="relative z-10 w-full">
|
||||
<Composer disabled={disabled} />
|
||||
<Composer disabled={disabled} threadId={threadId} />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
|
@ -293,8 +421,21 @@ const PendingAudioChip: FC = () => {
|
|||
);
|
||||
};
|
||||
|
||||
const Composer: FC<{ disabled?: boolean }> = ({ disabled }) => {
|
||||
const { inputProps, isComposing, isComposingRef } = useImeComposerInputHandlers();
|
||||
const Composer: FC<{
|
||||
disabled?: boolean;
|
||||
threadId?: string | null;
|
||||
}> = ({ disabled, threadId }) => {
|
||||
const aui = useAui();
|
||||
const { overlay, closeOverlay } = useGeneratedImageOverlay();
|
||||
const setImageToolsEnabled = useChatRuntimeStore(
|
||||
(s) => s.setImageToolsEnabled,
|
||||
);
|
||||
const activeThreadId = useChatRuntimeStore((s) => s.activeThreadId);
|
||||
const setPendingImageEditReference = useChatRuntimeStore(
|
||||
(s) => s.setPendingImageEditReference,
|
||||
);
|
||||
const { inputProps, isComposing, isComposingRef } =
|
||||
useImeComposerInputHandlers();
|
||||
const composerText = useAuiState(({ composer }) => composer.text);
|
||||
const hasAttachments = useAuiState(
|
||||
({ composer }) => composer.attachments.length > 0,
|
||||
|
|
@ -304,22 +445,78 @@ const Composer: FC<{ disabled?: boolean }> = ({ disabled }) => {
|
|||
(attachment) => attachment.status.type === "running",
|
||||
),
|
||||
);
|
||||
const hasPendingAudio = useChatRuntimeStore((s) => Boolean(s.pendingAudioName));
|
||||
const hasPendingAudio = useChatRuntimeStore((s) =>
|
||||
Boolean(s.pendingAudioName),
|
||||
);
|
||||
const referenceThreadId = threadId ?? activeThreadId ?? null;
|
||||
const hasSendableContent =
|
||||
composerText.trim().length > 0 || hasAttachments || hasPendingAudio;
|
||||
const shouldBlockSend = useCallback(
|
||||
() =>
|
||||
!hasSendableContent || isComposingRef.current || hasPendingAttachments,
|
||||
[hasPendingAttachments, hasSendableContent, isComposingRef],
|
||||
);
|
||||
|
||||
const handleSubmit = useCallback(
|
||||
(event: FormEvent<HTMLFormElement>) => {
|
||||
if (
|
||||
disabled ||
|
||||
!hasSendableContent ||
|
||||
isComposingRef.current ||
|
||||
hasPendingAttachments
|
||||
) {
|
||||
(event: Parameters<NonNullable<ComponentProps<"form">["onSubmit"]>>[0]) => {
|
||||
if (disabled || shouldBlockSend()) {
|
||||
event.preventDefault();
|
||||
return;
|
||||
}
|
||||
|
||||
if (overlay) {
|
||||
const trimmed = composerText.trim();
|
||||
if (!trimmed) {
|
||||
event.preventDefault();
|
||||
return;
|
||||
}
|
||||
if (!overlay.openaiImageGenerationCallId) {
|
||||
event.preventDefault();
|
||||
toast.error("This generated image cannot be edited", {
|
||||
description:
|
||||
"The original image reference is missing. Generate the image again, then retry the edit.",
|
||||
});
|
||||
closeOverlay();
|
||||
return;
|
||||
}
|
||||
if ((overlay.threadId ?? null) !== referenceThreadId) {
|
||||
event.preventDefault();
|
||||
toast.error("This generated image belongs to another chat", {
|
||||
description: "Open the original chat and retry the edit.",
|
||||
});
|
||||
closeOverlay();
|
||||
return;
|
||||
}
|
||||
setImageToolsEnabled(true);
|
||||
setPendingImageEditReference({
|
||||
threadId: overlay.threadId ?? referenceThreadId,
|
||||
openaiImageGenerationCallId: overlay.openaiImageGenerationCallId,
|
||||
...(overlay.openaiResponseId
|
||||
? { openaiResponseId: overlay.openaiResponseId }
|
||||
: {}),
|
||||
openaiReasoningItem: overlay.openaiReasoningItem,
|
||||
});
|
||||
flushResourcesSync(() => {
|
||||
aui
|
||||
.composer()
|
||||
.setText(
|
||||
`Use the selected generated image as the reference and apply this edit: ${trimmed}. Preserve everything else exactly.`,
|
||||
);
|
||||
});
|
||||
closeOverlay();
|
||||
}
|
||||
},
|
||||
[disabled, hasPendingAttachments, hasSendableContent, isComposingRef],
|
||||
[
|
||||
aui,
|
||||
closeOverlay,
|
||||
composerText,
|
||||
disabled,
|
||||
overlay,
|
||||
referenceThreadId,
|
||||
setImageToolsEnabled,
|
||||
setPendingImageEditReference,
|
||||
shouldBlockSend,
|
||||
],
|
||||
);
|
||||
|
||||
const composerContent = (
|
||||
|
|
@ -342,11 +539,12 @@ const Composer: FC<{ disabled?: boolean }> = ({ disabled }) => {
|
|||
/>
|
||||
<ComposerAction
|
||||
disabled={
|
||||
disabled || !hasSendableContent || isComposing || hasPendingAttachments
|
||||
}
|
||||
blockSend={() =>
|
||||
!hasSendableContent || isComposingRef.current || hasPendingAttachments
|
||||
disabled ||
|
||||
!hasSendableContent ||
|
||||
isComposing ||
|
||||
hasPendingAttachments
|
||||
}
|
||||
shouldBlockSend={shouldBlockSend}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
|
|
@ -553,7 +751,6 @@ const ComposerAudioUpload: FC = () => {
|
|||
);
|
||||
};
|
||||
|
||||
|
||||
const ReasoningToggle: FC = () => {
|
||||
const modelLoaded = useChatRuntimeStore(
|
||||
(s) => !!s.params.checkpoint && !s.modelLoading,
|
||||
|
|
@ -565,8 +762,12 @@ const ReasoningToggle: FC = () => {
|
|||
const setReasoningEnabled = useChatRuntimeStore((s) => s.setReasoningEnabled);
|
||||
const reasoningStyle = useChatRuntimeStore((s) => s.reasoningStyle);
|
||||
const reasoningEffort = useChatRuntimeStore((s) => s.reasoningEffort);
|
||||
const supportsReasoningOff = useChatRuntimeStore((s) => s.supportsReasoningOff);
|
||||
const reasoningEffortLevels = useChatRuntimeStore((s) => s.reasoningEffortLevels);
|
||||
const supportsReasoningOff = useChatRuntimeStore(
|
||||
(s) => s.supportsReasoningOff,
|
||||
);
|
||||
const reasoningEffortLevels = useChatRuntimeStore(
|
||||
(s) => s.reasoningEffortLevels,
|
||||
);
|
||||
const setReasoningEffort = useChatRuntimeStore((s) => s.setReasoningEffort);
|
||||
const lastOpenRouterChosenModel = useChatRuntimeStore(
|
||||
(s) => s.lastOpenRouterChosenModel,
|
||||
|
|
@ -619,7 +820,8 @@ const ReasoningToggle: FC = () => {
|
|||
effectiveReasoningEnabled && reasoningEffort !== "none";
|
||||
const disabled = !(modelLoaded && effectiveSupportsReasoning);
|
||||
const formatEffortLabel = (level: typeof reasoningEffort): string => {
|
||||
if (level !== "xhigh") return level.charAt(0).toUpperCase() + level.slice(1);
|
||||
if (level !== "xhigh")
|
||||
return level.charAt(0).toUpperCase() + level.slice(1);
|
||||
const normalized = externalSelection?.modelId?.trim().toLowerCase() ?? "";
|
||||
if (
|
||||
normalized.startsWith("claude-opus-4-6") ||
|
||||
|
|
@ -677,23 +879,25 @@ const ReasoningToggle: FC = () => {
|
|||
{effectiveReasoningEffortLevels
|
||||
.filter((level) => level !== "none")
|
||||
.map((level) => (
|
||||
<DropdownMenuItem
|
||||
key={level}
|
||||
onSelect={() => {
|
||||
setReasoningEffort(level);
|
||||
setReasoningEnabled(true);
|
||||
applyQwenThinkingParams(true);
|
||||
// Kimi's $web_search builtin forbids thinking, so
|
||||
// enabling thinking flips the Search pill off.
|
||||
if (isKimiExternal && toolsEnabled) {
|
||||
setToolsEnabled(false);
|
||||
}
|
||||
}}
|
||||
>
|
||||
{formatEffortLabel(level)}
|
||||
{effectiveReasoningVisualEnabled && reasoningEffort === level ? " \u2713" : ""}
|
||||
</DropdownMenuItem>
|
||||
))}
|
||||
<DropdownMenuItem
|
||||
key={level}
|
||||
onSelect={() => {
|
||||
setReasoningEffort(level);
|
||||
setReasoningEnabled(true);
|
||||
applyQwenThinkingParams(true);
|
||||
// Kimi's $web_search builtin forbids thinking, so
|
||||
// enabling thinking flips the Search pill off.
|
||||
if (isKimiExternal && toolsEnabled) {
|
||||
setToolsEnabled(false);
|
||||
}
|
||||
}}
|
||||
>
|
||||
{formatEffortLabel(level)}
|
||||
{effectiveReasoningVisualEnabled && reasoningEffort === level
|
||||
? " \u2713"
|
||||
: ""}
|
||||
</DropdownMenuItem>
|
||||
))}
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
);
|
||||
|
|
@ -808,8 +1012,7 @@ const WebSearchToggle: FC = () => {
|
|||
? externalProviders.find((p) => p.id === externalSelection.providerId)
|
||||
: undefined;
|
||||
const isKimiExternal = selectedExternalProvider?.providerType === "kimi";
|
||||
const disabled =
|
||||
!modelLoaded || !(supportsTools || supportsBuiltinWebSearch);
|
||||
const disabled = !modelLoaded || !(supportsTools || supportsBuiltinWebSearch);
|
||||
|
||||
return (
|
||||
<button
|
||||
|
|
@ -899,7 +1102,9 @@ const ImagesToggle: FC = () => {
|
|||
className="composer-pill-btn"
|
||||
data-active={imageToolsEnabled && !disabled ? "true" : "false"}
|
||||
aria-label={
|
||||
imageToolsEnabled ? "Disable image generation" : "Enable image generation"
|
||||
imageToolsEnabled
|
||||
? "Disable image generation"
|
||||
: "Enable image generation"
|
||||
}
|
||||
>
|
||||
<ImageIcon className="size-3.5" />
|
||||
|
|
@ -966,10 +1171,10 @@ const ToolStatusDisplay: FC = () => {
|
|||
);
|
||||
};
|
||||
|
||||
const ComposerAction: FC<{ disabled?: boolean; blockSend?: () => boolean }> = ({
|
||||
disabled,
|
||||
blockSend,
|
||||
}) => {
|
||||
const ComposerAction: FC<{
|
||||
disabled?: boolean;
|
||||
shouldBlockSend?: () => boolean;
|
||||
}> = ({ disabled, shouldBlockSend }) => {
|
||||
return (
|
||||
<div className="aui-composer-action-wrapper composer-action-wrapper">
|
||||
<div className="flex items-center gap-0.5">
|
||||
|
|
@ -1016,7 +1221,7 @@ const ComposerAction: FC<{ disabled?: boolean; blockSend?: () => boolean }> = ({
|
|||
size="icon"
|
||||
disabled={disabled}
|
||||
onClick={(event) => {
|
||||
if (blockSend?.()) {
|
||||
if (shouldBlockSend?.()) {
|
||||
event.preventDefault();
|
||||
}
|
||||
}}
|
||||
|
|
@ -1284,7 +1489,11 @@ const UserActionBar: FC = () => {
|
|||
<CopyButton />
|
||||
<ActionBarPrimitive.Edit asChild={true}>
|
||||
<TooltipIconButton tooltip="Edit" className="aui-user-action-edit">
|
||||
<HugeiconsIcon icon={Edit03Icon} strokeWidth={1.75} className="size-icon" />
|
||||
<HugeiconsIcon
|
||||
icon={Edit03Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-icon"
|
||||
/>
|
||||
</TooltipIconButton>
|
||||
</ActionBarPrimitive.Edit>
|
||||
<DeleteMessageButton />
|
||||
|
|
|
|||
|
|
@ -3,9 +3,14 @@
|
|||
|
||||
"use client";
|
||||
|
||||
import { type ToolCallMessagePartComponent, useAuiState } from "@assistant-ui/react";
|
||||
import { ImageIcon, LoaderIcon } from "lucide-react";
|
||||
import { memo, useEffect, useState } from "react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { cn } from "@/lib/utils";
|
||||
import type { ToolCallMessagePartComponent } from "@assistant-ui/react";
|
||||
import { DownloadIcon, ImageIcon, PencilIcon } from "lucide-react";
|
||||
import type { CSSProperties, MouseEvent } from "react";
|
||||
import { memo, useState } from "react";
|
||||
import { useGeneratedImageOverlay } from "./generated-image-overlay-context";
|
||||
import { Image, downloadImagePart } from "./image";
|
||||
import {
|
||||
ToolFallbackContent,
|
||||
ToolFallbackRoot,
|
||||
|
|
@ -38,6 +43,9 @@ import {
|
|||
interface ImageGenerationArgs {
|
||||
prompt?: string;
|
||||
kind?: string;
|
||||
openai_image_generation_call_id?: unknown;
|
||||
openai_response_id?: unknown;
|
||||
openai_reasoning_item?: unknown;
|
||||
}
|
||||
|
||||
interface ImageGenerationResult {
|
||||
|
|
@ -46,6 +54,83 @@ interface ImageGenerationResult {
|
|||
size?: string;
|
||||
quality?: string;
|
||||
background?: string;
|
||||
prompt?: string;
|
||||
}
|
||||
|
||||
type GeneratedImagePart = {
|
||||
type: "image";
|
||||
image: string;
|
||||
filename?: string;
|
||||
};
|
||||
|
||||
const extensionForMime = (mime: string): string => {
|
||||
switch (mime.toLowerCase()) {
|
||||
case "image/jpeg":
|
||||
case "image/jpg":
|
||||
return "jpg";
|
||||
case "image/webp":
|
||||
return "webp";
|
||||
case "image/gif":
|
||||
return "gif";
|
||||
case "image/svg+xml":
|
||||
return "svg";
|
||||
default:
|
||||
return "png";
|
||||
}
|
||||
};
|
||||
|
||||
const imageFilenameFromPrompt = (prompt: string, mime: string): string => {
|
||||
const slug = prompt
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-|-$/g, "")
|
||||
.slice(0, 48);
|
||||
return `${slug || "generated-image"}.${extensionForMime(mime)}`;
|
||||
};
|
||||
|
||||
const formatGeneratedImageLabel = (prompt: string): string => {
|
||||
if (!prompt) {
|
||||
return "Generated image";
|
||||
}
|
||||
return prompt.length > 80
|
||||
? `Generated image: ${prompt.slice(0, 80)}…`
|
||||
: `Generated image: ${prompt}`;
|
||||
};
|
||||
|
||||
const loadingDots = Array.from({ length: 64 }, (_, index) => {
|
||||
const row = Math.floor(index / 8);
|
||||
const col = index % 8;
|
||||
return (
|
||||
<span
|
||||
key={index}
|
||||
className="generated-image-loading-dot"
|
||||
style={
|
||||
{
|
||||
"--dot-row": row,
|
||||
"--dot-col": col,
|
||||
} as CSSProperties
|
||||
}
|
||||
/>
|
||||
);
|
||||
});
|
||||
|
||||
function GeneratedImagePlaceholder({ label }: { label: string }) {
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
"generated-image-loading-card flex aspect-square w-[480px] max-w-full items-center justify-center rounded-2xl border border-border/70 bg-muted/15",
|
||||
)}
|
||||
aria-busy="true"
|
||||
aria-label={label}
|
||||
aria-live="polite"
|
||||
>
|
||||
<span className="sr-only">{label}</span>
|
||||
<div className="generated-image-loading-wave" aria-hidden={true}>
|
||||
{loadingDots}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const ImageGenerationToolUIImpl: ToolCallMessagePartComponent = ({
|
||||
|
|
@ -53,6 +138,7 @@ const ImageGenerationToolUIImpl: ToolCallMessagePartComponent = ({
|
|||
result,
|
||||
status,
|
||||
}) => {
|
||||
const { openOverlay } = useGeneratedImageOverlay();
|
||||
const parsedArgs = (args as ImageGenerationArgs) ?? {};
|
||||
const prompt = parsedArgs.prompt ?? "";
|
||||
const isRunning = status?.type === "running";
|
||||
|
|
@ -66,33 +152,74 @@ const ImageGenerationToolUIImpl: ToolCallMessagePartComponent = ({
|
|||
const imageSrc = imageResult?.image_b64
|
||||
? `data:${mime};base64,${imageResult.image_b64}`
|
||||
: null;
|
||||
const imageTitle =
|
||||
imageResult?.prompt?.trim() || prompt.trim() || "Generated image";
|
||||
const imageMetadata = [imageResult?.size, imageResult?.quality, mime]
|
||||
.filter(Boolean)
|
||||
.join(" · ");
|
||||
const openaiImageGenerationCallId =
|
||||
typeof parsedArgs.openai_image_generation_call_id === "string"
|
||||
? parsedArgs.openai_image_generation_call_id
|
||||
: undefined;
|
||||
const openaiResponseId =
|
||||
typeof parsedArgs.openai_response_id === "string"
|
||||
? parsedArgs.openai_response_id
|
||||
: undefined;
|
||||
const imagePart: GeneratedImagePart | null = imageSrc
|
||||
? {
|
||||
type: "image",
|
||||
image: imageSrc,
|
||||
filename: imageFilenameFromPrompt(prompt, mime),
|
||||
}
|
||||
: null;
|
||||
|
||||
// Collapse the card once the model has resumed streaming prose
|
||||
// after the image. Mirrors CodeExecutionToolUI so the inline image
|
||||
// doesn't collapse mid-stream and the user can click to re-expand.
|
||||
const hasText = useAuiState(({ message }) =>
|
||||
message.content.some(
|
||||
(p) =>
|
||||
p.type === "text" &&
|
||||
"text" in p &&
|
||||
(p as { text: string }).text.length > 0,
|
||||
),
|
||||
);
|
||||
const [open, setOpen] = useState(true);
|
||||
useEffect(() => {
|
||||
if (isRunning) {
|
||||
setOpen(true);
|
||||
} else if (hasText && !imageSrc) {
|
||||
setOpen(false);
|
||||
}
|
||||
}, [isRunning, hasText, imageSrc]);
|
||||
const isPendingImage = !imagePart && status?.type === "running";
|
||||
|
||||
const runningLabel = "Generating image…";
|
||||
const completedLabel = prompt
|
||||
? prompt.length > 80
|
||||
? `Generated image: ${prompt.slice(0, 80)}…`
|
||||
: `Generated image: ${prompt}`
|
||||
: "Generated image";
|
||||
const completedLabel = formatGeneratedImageLabel(prompt);
|
||||
|
||||
const showPreview = () => {
|
||||
if (!imagePart) {
|
||||
return;
|
||||
}
|
||||
openOverlay({
|
||||
image: imagePart.image,
|
||||
title: imageTitle,
|
||||
metadata: imageMetadata,
|
||||
filename: imagePart.filename,
|
||||
openaiImageGenerationCallId,
|
||||
openaiResponseId,
|
||||
openaiReasoningItem: parsedArgs.openai_reasoning_item,
|
||||
});
|
||||
};
|
||||
|
||||
const stopOverlayActionPropagation = (
|
||||
event: MouseEvent<HTMLButtonElement>,
|
||||
) => {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
};
|
||||
|
||||
const handleDownload = (event: MouseEvent<HTMLButtonElement>) => {
|
||||
stopOverlayActionPropagation(event);
|
||||
if (imagePart) {
|
||||
downloadImagePart(imagePart);
|
||||
}
|
||||
};
|
||||
|
||||
const handleEditClick = (event: MouseEvent<HTMLButtonElement>) => {
|
||||
stopOverlayActionPropagation(event);
|
||||
showPreview();
|
||||
};
|
||||
|
||||
if (isPendingImage) {
|
||||
return (
|
||||
<div className="aui-tool-fallback-root w-full py-1">
|
||||
<GeneratedImagePlaceholder label={runningLabel} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<ToolFallbackRoot open={open} onOpenChange={setOpen}>
|
||||
|
|
@ -102,20 +229,47 @@ const ImageGenerationToolUIImpl: ToolCallMessagePartComponent = ({
|
|||
icon={ImageIcon}
|
||||
/>
|
||||
<ToolFallbackContent>
|
||||
{isRunning && !imageSrc ? (
|
||||
<div className="flex items-center gap-2 text-sm text-muted-foreground">
|
||||
<LoaderIcon className="size-3.5 animate-spin" />
|
||||
<span>{runningLabel}</span>
|
||||
</div>
|
||||
) : imageSrc ? (
|
||||
<figure className="m-0 flex flex-col gap-1.5">
|
||||
<img
|
||||
src={imageSrc}
|
||||
alt={prompt || "Generated image"}
|
||||
className="max-w-full rounded-md border border-border/60"
|
||||
/>
|
||||
{imagePart ? (
|
||||
<figure className="m-0 flex flex-col gap-2">
|
||||
<div className="group/generated-image relative aspect-square w-[480px] max-w-full overflow-hidden rounded-2xl border border-border/70 bg-muted/30 shadow-sm">
|
||||
<button
|
||||
type="button"
|
||||
className="block size-full cursor-zoom-in overflow-hidden rounded-2xl focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-background"
|
||||
onClick={showPreview}
|
||||
aria-label="Open generated image preview"
|
||||
>
|
||||
<Image.Preview
|
||||
src={imagePart.image}
|
||||
alt={imageTitle}
|
||||
containerClassName="size-full min-h-0 bg-background"
|
||||
className="size-full object-cover"
|
||||
/>
|
||||
</button>
|
||||
<div className="pointer-events-none absolute inset-x-0 bottom-0 flex items-end justify-between gap-2 bg-gradient-to-t from-black/55 via-black/20 to-transparent p-3 opacity-100 transition-opacity sm:opacity-0 sm:group-hover/generated-image:opacity-100 sm:group-focus-within/generated-image:opacity-100">
|
||||
<Button
|
||||
type="button"
|
||||
variant="dark"
|
||||
size="sm"
|
||||
className="pointer-events-auto h-8 rounded-full bg-black/70 text-white hover:bg-black/85"
|
||||
onClick={handleEditClick}
|
||||
>
|
||||
<PencilIcon className="size-3.5" />
|
||||
Edit
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="dark"
|
||||
size="icon-sm"
|
||||
className="pointer-events-auto rounded-full bg-black/70 text-white hover:bg-black/85"
|
||||
onClick={handleDownload}
|
||||
aria-label="Download generated image"
|
||||
>
|
||||
<DownloadIcon className="size-4" />
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
{prompt ? (
|
||||
<figcaption className="text-xs leading-snug text-muted-foreground">
|
||||
<figcaption className="max-w-[480px] text-xs leading-snug text-muted-foreground">
|
||||
{prompt}
|
||||
</figcaption>
|
||||
) : null}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { getAuthToken } from "@/features/auth/session";
|
||||
import { getAuthToken } from "@/features/auth";
|
||||
import { apiUrl } from "@/lib/api-base";
|
||||
import { toast } from "@/lib/toast";
|
||||
import type { MessageTiming, ToolCallMessagePart } from "@assistant-ui/core";
|
||||
|
|
@ -30,12 +30,17 @@ import {
|
|||
providerSupportsBuiltinWebSearch,
|
||||
providerSupportsFastMode,
|
||||
} from "../provider-capabilities";
|
||||
import { useChatRuntimeStore } from "../stores/chat-runtime-store";
|
||||
import {
|
||||
type PendingImageEditReference,
|
||||
useChatRuntimeStore,
|
||||
} from "../stores/chat-runtime-store";
|
||||
import { useExternalProvidersStore } from "../stores/external-providers-store";
|
||||
import { isMultimodalResponse } from "../types/api";
|
||||
import type {
|
||||
OpenAIChatCompletionsRequest,
|
||||
OpenAIChatMessage,
|
||||
OpenAIMessageContent,
|
||||
OpenAIReasoningContentPart,
|
||||
} from "../types/api";
|
||||
import type { ChatModelSummary } from "../types/runtime";
|
||||
import { getImageInputUnavailableReason } from "../utils/image-input-support";
|
||||
|
|
@ -399,37 +404,30 @@ function collectImageParts(
|
|||
message: RunMessage,
|
||||
): Array<{ type: "image_url"; image_url: { url: string } }> {
|
||||
const parts: Array<{ type: "image_url"; image_url: { url: string } }> = [];
|
||||
const pushImagePart = (part: { type: string }) => {
|
||||
if (part.type !== "image" || !("image" in part)) {
|
||||
return;
|
||||
}
|
||||
const src = (part as { image: string }).image;
|
||||
if (!src) {
|
||||
return;
|
||||
}
|
||||
parts.push({
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: src.startsWith("data:") ? src : `data:image/png;base64,${src}`,
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
for (const part of message.content ?? []) {
|
||||
if (part.type === "image" && "image" in part) {
|
||||
const src = (part as { image: string }).image;
|
||||
if (src) {
|
||||
parts.push({
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: src.startsWith("data:") ? src : `data:image/png;base64,${src}`,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
pushImagePart(part);
|
||||
}
|
||||
|
||||
if ("attachments" in message && (message.attachments?.length ?? 0) > 0) {
|
||||
for (const attachment of message.attachments ?? []) {
|
||||
for (const part of attachment.content ?? []) {
|
||||
if (part.type === "image" && "image" in part) {
|
||||
const src = (part as { image: string }).image;
|
||||
if (src) {
|
||||
parts.push({
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: src.startsWith("data:")
|
||||
? src
|
||||
: `data:image/png;base64,${src}`,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
pushImagePart(part);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -437,6 +435,66 @@ function collectImageParts(
|
|||
return parts;
|
||||
}
|
||||
|
||||
function normalizeOpenAIReasoningItem(
|
||||
value: unknown,
|
||||
): OpenAIReasoningContentPart | null {
|
||||
if (!value || typeof value !== "object") {
|
||||
return null;
|
||||
}
|
||||
const item = value as Record<string, unknown>;
|
||||
if (item.type !== "reasoning" || typeof item.id !== "string" || !item.id) {
|
||||
return null;
|
||||
}
|
||||
const summary = Array.isArray(item.summary)
|
||||
? item.summary.flatMap((part) => {
|
||||
if (!part || typeof part !== "object") {
|
||||
return [];
|
||||
}
|
||||
const summaryPart = part as Record<string, unknown>;
|
||||
return summaryPart.type === "summary_text" &&
|
||||
typeof summaryPart.text === "string"
|
||||
? [{ type: "summary_text" as const, text: summaryPart.text }]
|
||||
: [];
|
||||
})
|
||||
: [];
|
||||
const normalized: OpenAIReasoningContentPart = {
|
||||
type: "reasoning",
|
||||
id: item.id,
|
||||
summary,
|
||||
};
|
||||
if (
|
||||
item.status === "in_progress" ||
|
||||
item.status === "completed" ||
|
||||
item.status === "incomplete"
|
||||
) {
|
||||
normalized.status = item.status;
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function toOpenAIImageEditReferenceMessage(
|
||||
reference: PendingImageEditReference,
|
||||
): OpenAIChatMessage | null {
|
||||
if (!reference.openaiImageGenerationCallId) {
|
||||
return null;
|
||||
}
|
||||
const content: Exclude<OpenAIMessageContent, string> = [];
|
||||
const reasoningItem = normalizeOpenAIReasoningItem(
|
||||
reference.openaiReasoningItem,
|
||||
);
|
||||
if (reasoningItem) {
|
||||
content.push(reasoningItem);
|
||||
}
|
||||
content.push({
|
||||
type: "image_generation_call",
|
||||
id: reference.openaiImageGenerationCallId,
|
||||
...(reference.openaiResponseId
|
||||
? { response_id: reference.openaiResponseId }
|
||||
: {}),
|
||||
});
|
||||
return { role: "assistant", content };
|
||||
}
|
||||
|
||||
// Refusal flag stamped on assistant metadata when the backend emits the
|
||||
// `anthropic_refusal` _toolEvent. We drop the refused pair from the next
|
||||
// request body (Anthropic guidance: leaving refusals in context keeps
|
||||
|
|
@ -480,10 +538,16 @@ function toOpenAIMessage(message: RunMessage): {
|
|||
if (imageParts.length > 0) {
|
||||
return {
|
||||
role: message.role,
|
||||
content: [{ type: "text", text: textContent }, ...imageParts],
|
||||
content: [
|
||||
...(textContent ? [{ type: "text" as const, text: textContent }] : []),
|
||||
...imageParts,
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
if (!textContent) {
|
||||
return null;
|
||||
}
|
||||
return { role: message.role, content: textContent };
|
||||
}
|
||||
|
||||
|
|
@ -918,17 +982,52 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
// the user switches chats while waiting for model load / auto-load.
|
||||
const resolvedThreadId =
|
||||
(unstable_threadId ?? runtime.activeThreadId) || undefined;
|
||||
const resolvedThreadKey = resolvedThreadId ?? null;
|
||||
const pendingImageEditReferenceForRun = runtime.pendingImageEditReference;
|
||||
const selectedImageEditReference =
|
||||
(pendingImageEditReferenceForRun?.threadId ?? null) ===
|
||||
resolvedThreadKey
|
||||
? pendingImageEditReferenceForRun
|
||||
: null;
|
||||
const clearSelectedImageEditReference = () => {
|
||||
if (!selectedImageEditReference) {
|
||||
return;
|
||||
}
|
||||
const store = useChatRuntimeStore.getState();
|
||||
const pending = store.pendingImageEditReference;
|
||||
if (
|
||||
pending?.openaiImageGenerationCallId ===
|
||||
selectedImageEditReference.openaiImageGenerationCallId &&
|
||||
pending.openaiResponseId ===
|
||||
selectedImageEditReference.openaiResponseId &&
|
||||
(pending.threadId ?? null) ===
|
||||
(selectedImageEditReference.threadId ?? null)
|
||||
) {
|
||||
store.clearPendingImageEditReference();
|
||||
}
|
||||
};
|
||||
|
||||
// Wait for in-progress model load to finish before inferring
|
||||
if (runtime.modelLoading) {
|
||||
toast.info("Waiting for model to finish loading…");
|
||||
await waitForModelReady(abortSignal);
|
||||
try {
|
||||
await waitForModelReady(abortSignal);
|
||||
} catch (error) {
|
||||
clearSelectedImageEditReference();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
if (!useChatRuntimeStore.getState().params.checkpoint) {
|
||||
// Auto-load the smallest downloaded model
|
||||
const { loaded, blockedByTrustRemoteCode } =
|
||||
await autoLoadSmallestModel();
|
||||
let loaded: boolean;
|
||||
let blockedByTrustRemoteCode: boolean;
|
||||
try {
|
||||
({ loaded, blockedByTrustRemoteCode } = await autoLoadSmallestModel());
|
||||
} catch (error) {
|
||||
clearSelectedImageEditReference();
|
||||
throw error;
|
||||
}
|
||||
if (!loaded) {
|
||||
toast.error(
|
||||
blockedByTrustRemoteCode
|
||||
|
|
@ -940,6 +1039,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
: "Pick a model in the top bar, then retry.",
|
||||
},
|
||||
);
|
||||
clearSelectedImageEditReference();
|
||||
throw new Error("Load a model first.");
|
||||
}
|
||||
}
|
||||
|
|
@ -964,6 +1064,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
description:
|
||||
"Turn on Enable connections in Settings → Connections to use hosted models.",
|
||||
});
|
||||
clearSelectedImageEditReference();
|
||||
throw new Error("Connections disabled.");
|
||||
}
|
||||
const externalProvider = isExternalRequest
|
||||
|
|
@ -979,6 +1080,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
toast.error("Connection not found.", {
|
||||
description: "Open Settings → Connections and add it again.",
|
||||
});
|
||||
clearSelectedImageEditReference();
|
||||
throw new Error("Connection not found.");
|
||||
}
|
||||
// Local providers (llama.cpp / vLLM / Ollama) allow an empty key — only block hosted providers.
|
||||
|
|
@ -989,36 +1091,34 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
toast.error("Missing API key for selected connection.", {
|
||||
description: "Open Settings → Connections and set the API key again.",
|
||||
});
|
||||
clearSelectedImageEditReference();
|
||||
throw new Error("Missing connection API key.");
|
||||
}
|
||||
|
||||
const webSearchEnabledForThisTurn =
|
||||
Boolean(
|
||||
externalProvider &&
|
||||
toolsEnabled &&
|
||||
providerSupportsBuiltinWebSearch(externalProvider.providerType),
|
||||
);
|
||||
const codeExecEnabledForThisTurn =
|
||||
Boolean(
|
||||
externalProvider &&
|
||||
externalSelection &&
|
||||
codeToolsEnabled &&
|
||||
providerSupportsBuiltinCodeExecution(
|
||||
externalProvider.providerType,
|
||||
externalSelection.modelId,
|
||||
externalProvider.baseUrl,
|
||||
),
|
||||
);
|
||||
const webSearchEnabledForThisTurn = Boolean(
|
||||
externalProvider &&
|
||||
toolsEnabled &&
|
||||
providerSupportsBuiltinWebSearch(externalProvider.providerType),
|
||||
);
|
||||
const codeExecEnabledForThisTurn = Boolean(
|
||||
externalProvider &&
|
||||
externalSelection &&
|
||||
codeToolsEnabled &&
|
||||
providerSupportsBuiltinCodeExecution(
|
||||
externalProvider.providerType,
|
||||
externalSelection.modelId,
|
||||
externalProvider.baseUrl,
|
||||
),
|
||||
);
|
||||
// Fetch pill is independent of Search (Anthropic bills web_fetch
|
||||
// separately from web_search). Sourced from `webFetchToolsEnabled`;
|
||||
// on providers without web_fetch the toggle is forced off in
|
||||
// chat-page's runtime setState.
|
||||
const webFetchEnabledForThisTurn =
|
||||
Boolean(
|
||||
externalProvider &&
|
||||
webFetchToolsEnabled &&
|
||||
providerSupportsBuiltinWebFetch(externalProvider.providerType),
|
||||
);
|
||||
const webFetchEnabledForThisTurn = Boolean(
|
||||
externalProvider &&
|
||||
webFetchToolsEnabled &&
|
||||
providerSupportsBuiltinWebFetch(externalProvider.providerType),
|
||||
);
|
||||
const providerShipsWebFetch = Boolean(
|
||||
externalProvider &&
|
||||
providerSupportsBuiltinWebFetch(externalProvider.providerType),
|
||||
|
|
@ -1038,6 +1138,15 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
),
|
||||
);
|
||||
|
||||
if (selectedImageEditReference && !imageGenerationEnabledForThisTurn) {
|
||||
clearSelectedImageEditReference();
|
||||
toast.error("Image editing is unavailable", {
|
||||
description:
|
||||
"Select an OpenAI image-generation model, then retry the edit.",
|
||||
});
|
||||
throw new Error("Image generation edit unavailable.");
|
||||
}
|
||||
|
||||
// Two-pass build: a refused assistant turn also drops the user
|
||||
// prompt that triggered it (leaving it in context re-triggers
|
||||
// the classifier). Refusal flag rides assistant
|
||||
|
|
@ -1060,6 +1169,27 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
.filter((message): message is NonNullable<typeof message> =>
|
||||
Boolean(message),
|
||||
);
|
||||
if (selectedImageEditReference) {
|
||||
const referenceMessage = toOpenAIImageEditReferenceMessage(
|
||||
selectedImageEditReference,
|
||||
);
|
||||
if (!referenceMessage) {
|
||||
clearSelectedImageEditReference();
|
||||
toast.error("This generated image cannot be edited", {
|
||||
description:
|
||||
"The original image reference is missing. Generate the image again, then retry the edit.",
|
||||
});
|
||||
throw new Error("Generated image edit reference missing.");
|
||||
}
|
||||
let insertAt = outboundMessages.length;
|
||||
for (let i = outboundMessages.length - 1; i >= 0; i -= 1) {
|
||||
if (outboundMessages[i]?.role === "user") {
|
||||
insertAt = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
outboundMessages.splice(insertAt, 0, referenceMessage);
|
||||
}
|
||||
|
||||
const safeSystemPrompt =
|
||||
typeof params.systemPrompt === "string" ? params.systemPrompt : "";
|
||||
|
|
@ -1084,24 +1214,45 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
// was on and suppressed live web_fetch calls.
|
||||
const anyWebEnabledForThisTurn =
|
||||
webSearchEnabledForThisTurn || webFetchEnabledForThisTurn;
|
||||
if (!anyWebEnabledForThisTurn && !codeExecEnabledForThisTurn) {
|
||||
if (
|
||||
!anyWebEnabledForThisTurn &&
|
||||
!codeExecEnabledForThisTurn &&
|
||||
!imageGenerationEnabledForThisTurn
|
||||
) {
|
||||
disabledToolGuard =
|
||||
`You do not have ${webLabel}, code execution, or image generation tools in this conversation. ` +
|
||||
"Answer from your own knowledge. " +
|
||||
"If a request genuinely requires tool use, live data fetch, running code, or image generation, " +
|
||||
"inform the user that you do not have access to these capabilities. " +
|
||||
"Do not return tool-call syntax inside your response.";
|
||||
} else if (!anyWebEnabledForThisTurn && !codeExecEnabledForThisTurn) {
|
||||
disabledToolGuard =
|
||||
`You do not have ${webLabel} or code execution tools in this conversation. ` +
|
||||
"Answer from your own knowledge. " +
|
||||
"If a request genuinely requires tool use, live data fetch or running code, " +
|
||||
"You may still use image generation tools when they are available and useful. " +
|
||||
"If a request genuinely requires live data fetch or running code, " +
|
||||
"inform the user that you do not have access to these capabilities. " +
|
||||
"Do not return tool-call syntax inside your response.";
|
||||
} else if (!anyWebEnabledForThisTurn) {
|
||||
const availableTools = [
|
||||
codeExecEnabledForThisTurn ? "code execution" : null,
|
||||
imageGenerationEnabledForThisTurn ? "image generation" : null,
|
||||
].filter(Boolean);
|
||||
disabledToolGuard =
|
||||
`You do not have ${webLabel} tools in this conversation. ` +
|
||||
"You may still use code execution tools when they are available and useful. " +
|
||||
(availableTools.length > 0
|
||||
? `You may still use ${availableTools.join(" and ")} tools when they are available and useful. `
|
||||
: "") +
|
||||
"If a request genuinely requires live data fetch or web search tool use, " +
|
||||
"inform the user that you do not have access to these capabilities. " +
|
||||
"Do not return tool-call syntax inside your response.";
|
||||
} else if (!codeExecEnabledForThisTurn) {
|
||||
const availableTools = [
|
||||
webLabel,
|
||||
imageGenerationEnabledForThisTurn ? "image generation" : null,
|
||||
].filter(Boolean);
|
||||
disabledToolGuard =
|
||||
"You do not have code execution tools in this conversation. " +
|
||||
`You may still use ${webLabel} tools when they are available and useful. ` +
|
||||
`You may still use ${availableTools.join(" and ")} tools when they are available and useful. ` +
|
||||
"If a request genuinely requires running code or code execution tool use, " +
|
||||
"inform the user that you do not have access to these capabilities. " +
|
||||
"Do not return tool-call syntax inside your response.";
|
||||
|
|
@ -1163,6 +1314,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
const gatedThreadKey = resolvedThreadId || "__default";
|
||||
runtime.setThreadRunning(gatedThreadKey, true);
|
||||
runtime.setThreadRunning(gatedThreadKey, false);
|
||||
clearSelectedImageEditReference();
|
||||
throw new Error(imageGateReason);
|
||||
}
|
||||
}
|
||||
|
|
@ -1474,8 +1626,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
) {
|
||||
void updateStoredChatThreadEventually(t.id, {
|
||||
openaiCodeExecContainerId: null,
|
||||
})
|
||||
.catch(() => {});
|
||||
}).catch(() => {});
|
||||
continue;
|
||||
}
|
||||
openaiCodeExecContainerId = t.openaiCodeExecContainerId;
|
||||
|
|
@ -1519,8 +1670,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
openaiCodeExecContainerId = created.id;
|
||||
void updateStoredChatThreadEventually(resolvedThreadId, {
|
||||
openaiCodeExecContainerId: created.id,
|
||||
})
|
||||
.catch(() => {});
|
||||
}).catch(() => {});
|
||||
} catch {
|
||||
// Fall back to backend's container_auto path on
|
||||
// failure — keeps the chat moving; the next turn
|
||||
|
|
@ -1628,7 +1778,9 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
// attaches `cache_control.ttl` when the value is one of
|
||||
// "5m" / "1h" (see external_provider.py near line 1375),
|
||||
// so unknown values are a no-op end-to-end.
|
||||
...(supportsProviderPromptCacheTtl(externalProvider.providerType) &&
|
||||
...(supportsProviderPromptCacheTtl(
|
||||
externalProvider.providerType,
|
||||
) &&
|
||||
(externalProvider.enablePromptCaching ?? true) &&
|
||||
isPromptCacheTtl(externalProvider.promptCacheTtl)
|
||||
? { prompt_cache_ttl: externalProvider.promptCacheTtl }
|
||||
|
|
@ -1706,10 +1858,15 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
let retriedWithRefreshedKey = false;
|
||||
while (true) {
|
||||
try {
|
||||
const stream = streamChatCompletions(
|
||||
await buildRequestPayload(retriedWithRefreshedKey),
|
||||
abortSignal,
|
||||
);
|
||||
let requestPayload: OpenAIChatCompletionsRequest;
|
||||
try {
|
||||
requestPayload = await buildRequestPayload(retriedWithRefreshedKey);
|
||||
} catch (error) {
|
||||
clearSelectedImageEditReference();
|
||||
throw error;
|
||||
}
|
||||
clearSelectedImageEditReference();
|
||||
const stream = streamChatCompletions(requestPayload, abortSignal);
|
||||
|
||||
for await (const chunk of stream) {
|
||||
// Handle tool status events
|
||||
|
|
@ -1777,8 +1934,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
: "openaiCodeExecContainerId";
|
||||
void updateStoredChatThreadEventually(resolvedThreadId, {
|
||||
[field]: null,
|
||||
})
|
||||
.catch(() => {});
|
||||
}).catch(() => {});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
|
@ -1822,6 +1978,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
size?: string;
|
||||
quality?: string;
|
||||
background?: string;
|
||||
prompt?: string;
|
||||
};
|
||||
const imageB64 = toolEvent.image_b64 as string | undefined;
|
||||
if (
|
||||
|
|
@ -1843,6 +2000,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
size: toolEvent.size as string | undefined,
|
||||
quality: toolEvent.quality as string | undefined,
|
||||
background: toolEvent.background as string | undefined,
|
||||
prompt: toolEvent.prompt as string | undefined,
|
||||
};
|
||||
} else if (imgIdx !== -1) {
|
||||
const text = rawResult.slice(0, imgIdx);
|
||||
|
|
@ -1860,8 +2018,20 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
} else {
|
||||
parsedResult = rawResult;
|
||||
}
|
||||
const nextArgs =
|
||||
toolEvent.arguments &&
|
||||
typeof toolEvent.arguments === "object"
|
||||
? (toolEvent.arguments as ToolCallMessagePart["args"])
|
||||
: undefined;
|
||||
const mergedArgs = nextArgs
|
||||
? { ...(toolCallParts[idx].args ?? {}), ...nextArgs }
|
||||
: toolCallParts[idx].args;
|
||||
toolCallParts[idx] = {
|
||||
...toolCallParts[idx],
|
||||
args: mergedArgs,
|
||||
argsText: mergedArgs
|
||||
? JSON.stringify(mergedArgs)
|
||||
: toolCallParts[idx].argsText,
|
||||
result: parsedResult,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -207,15 +207,21 @@ const OPENAI_CODE_EXECUTION_MODEL_PREFIXES = [
|
|||
|
||||
/**
|
||||
* Strict check that a provider configuration points at OpenAI's
|
||||
* managed cloud (api.openai.com), as opposed to a custom OpenAI-compat
|
||||
* backend (ollama / llama.cpp / vLLM / generic "custom" preset). The
|
||||
* shell tool ONLY exists on OpenAI cloud; sending it to anything else
|
||||
* 400s the request. Mirror of the backend's
|
||||
* `is_openai_cloud = "api.openai.com" in self.base_url` guard.
|
||||
* managed cloud (api.openai.com) or Azure OpenAI Foundry
|
||||
* (*.openai.azure.com), as opposed to a custom OpenAI-compat backend
|
||||
* (ollama / llama.cpp / vLLM / generic "custom" preset). The shell and
|
||||
* image-generation tools only exist on cloud backends; sending them to
|
||||
* anything else 400s the request. Mirror of the backend's
|
||||
* `_is_openai_family_cloud` host check.
|
||||
*/
|
||||
function isOpenAICloudBaseUrl(baseUrl: string | null | undefined): boolean {
|
||||
if (!baseUrl) return true; // No override → uses the default openai.com base.
|
||||
return baseUrl.trim().toLowerCase().includes("api.openai.com");
|
||||
try {
|
||||
const host = new URL(baseUrl).hostname.toLowerCase();
|
||||
return host === "api.openai.com" || host.endsWith(".openai.azure.com");
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function providerSupportsBuiltinCodeExecution(
|
||||
|
|
|
|||
|
|
@ -64,6 +64,12 @@ function saveLastExternalCheckpoint(value: string | null): void {
|
|||
}
|
||||
|
||||
export type ReasoningStyle = "enable_thinking" | "reasoning_effort";
|
||||
export type PendingImageEditReference = {
|
||||
threadId: string | null;
|
||||
openaiImageGenerationCallId: string;
|
||||
openaiResponseId?: string;
|
||||
openaiReasoningItem?: unknown;
|
||||
};
|
||||
export type ReasoningEffort =
|
||||
| "none"
|
||||
| "minimal"
|
||||
|
|
@ -300,6 +306,7 @@ type ChatRuntimeStore = {
|
|||
settingsPanelOpen: boolean;
|
||||
pendingAudioBase64: string | null;
|
||||
pendingAudioName: string | null;
|
||||
pendingImageEditReference: PendingImageEditReference | null;
|
||||
contextUsage: {
|
||||
promptTokens: number;
|
||||
completionTokens: number;
|
||||
|
|
@ -353,6 +360,10 @@ type ChatRuntimeStore = {
|
|||
setChatTemplateOverride: (template: string | null) => void;
|
||||
setPendingAudio: (base64: string, name: string) => void;
|
||||
clearPendingAudio: () => void;
|
||||
setPendingImageEditReference: (
|
||||
reference: PendingImageEditReference | null,
|
||||
) => void;
|
||||
clearPendingImageEditReference: () => void;
|
||||
setContextUsage: (usage: ChatRuntimeStore["contextUsage"]) => void;
|
||||
};
|
||||
|
||||
|
|
@ -607,6 +618,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
|
|||
settingsPanelOpen: false,
|
||||
pendingAudioBase64: null,
|
||||
pendingAudioName: null,
|
||||
pendingImageEditReference: null,
|
||||
contextUsage: null,
|
||||
modelLoading: false,
|
||||
activeNativePathToken: null,
|
||||
|
|
@ -793,6 +805,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
|
|||
defaultChatTemplate: null,
|
||||
chatTemplateOverride: null,
|
||||
loadedChatTemplateOverride: null,
|
||||
pendingImageEditReference: null,
|
||||
}));
|
||||
},
|
||||
setReasoningEnabled: (reasoningEnabled, options) =>
|
||||
|
|
@ -884,5 +897,9 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
|
|||
set({ pendingAudioBase64: base64, pendingAudioName: name }),
|
||||
clearPendingAudio: () =>
|
||||
set({ pendingAudioBase64: null, pendingAudioName: null }),
|
||||
setPendingImageEditReference: (pendingImageEditReference) =>
|
||||
set({ pendingImageEditReference }),
|
||||
clearPendingImageEditReference: () =>
|
||||
set({ pendingImageEditReference: null }),
|
||||
setContextUsage: (contextUsage) => set({ contextUsage }),
|
||||
}));
|
||||
|
|
|
|||
|
|
@ -192,12 +192,31 @@ export interface AudioGenerationResponse {
|
|||
}>;
|
||||
}
|
||||
|
||||
export type OpenAIMessageContent =
|
||||
| string
|
||||
| Array<
|
||||
| { type: "text"; text: string }
|
||||
| { type: "image_url"; image_url: { url: string } }
|
||||
>;
|
||||
export type OpenAIReasoningSummaryPart = {
|
||||
type: "summary_text";
|
||||
text: string;
|
||||
};
|
||||
|
||||
export type OpenAIReasoningContentPart = {
|
||||
type: "reasoning";
|
||||
id: string;
|
||||
summary: OpenAIReasoningSummaryPart[];
|
||||
status?: "in_progress" | "completed" | "incomplete";
|
||||
};
|
||||
|
||||
export type OpenAIImageGenerationCallContentPart = {
|
||||
type: "image_generation_call";
|
||||
id: string;
|
||||
response_id?: string;
|
||||
};
|
||||
|
||||
export type OpenAIMessageContentPart =
|
||||
| { type: "text"; text: string }
|
||||
| { type: "image_url"; image_url: { url: string } }
|
||||
| OpenAIReasoningContentPart
|
||||
| OpenAIImageGenerationCallContentPart;
|
||||
|
||||
export type OpenAIMessageContent = string | OpenAIMessageContentPart[];
|
||||
|
||||
export interface OpenAIChatMessage {
|
||||
role: "system" | "user" | "assistant";
|
||||
|
|
|
|||
|
|
@ -1188,6 +1188,53 @@
|
|||
border-color: var(--border) !important;
|
||||
}
|
||||
|
||||
.generated-image-loading-card {
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
contain: paint;
|
||||
}
|
||||
|
||||
.generated-image-loading-wave {
|
||||
position: relative;
|
||||
display: grid;
|
||||
grid-template-columns: repeat(8, minmax(0, 1fr));
|
||||
gap: 14px;
|
||||
width: min(66%, 18rem);
|
||||
padding: 1.5rem;
|
||||
border-radius: 1.5rem;
|
||||
}
|
||||
|
||||
.generated-image-loading-dot {
|
||||
width: 7px;
|
||||
height: 7px;
|
||||
border-radius: 9999px;
|
||||
background: color-mix(in oklch, var(--muted-foreground) 82%, var(--primary));
|
||||
opacity: 0.12;
|
||||
transform: translate3d(0, 4px, 0) scale(0.72);
|
||||
animation: generated-image-dot-wave 1850ms var(--ease-out-quart) infinite;
|
||||
animation-delay: calc((var(--dot-row) * 72ms) + (var(--dot-col) * 72ms));
|
||||
will-change: transform, opacity;
|
||||
}
|
||||
|
||||
@keyframes generated-image-dot-wave {
|
||||
0%,
|
||||
22%,
|
||||
100% {
|
||||
opacity: 0.1;
|
||||
transform: translate3d(0, 4px, 0) scale(0.72);
|
||||
}
|
||||
|
||||
46% {
|
||||
opacity: 0.46;
|
||||
transform: translate3d(0, -3px, 0) scale(0.96);
|
||||
}
|
||||
|
||||
66% {
|
||||
opacity: 0.2;
|
||||
transform: translate3d(0, 0, 0) scale(0.82);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* prefers-reduced-motion: honour the OS-level "reduce motion" preference.
|
||||
* Tailwind animate-in/out, Radix open/close transforms, infinite shine/pulse
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue