@@ -1016,7 +1221,7 @@ const ComposerAction: FC<{ disabled?: boolean; blockSend?: () => boolean }> = ({
size="icon"
disabled={disabled}
onClick={(event) => {
- if (blockSend?.()) {
+ if (shouldBlockSend?.()) {
event.preventDefault();
}
}}
@@ -1284,7 +1489,11 @@ const UserActionBar: FC = () => {
-
+
diff --git a/studio/frontend/src/components/assistant-ui/tool-ui-image-generation.tsx b/studio/frontend/src/components/assistant-ui/tool-ui-image-generation.tsx
index 246d8b6978..e32d23acf6 100644
--- a/studio/frontend/src/components/assistant-ui/tool-ui-image-generation.tsx
+++ b/studio/frontend/src/components/assistant-ui/tool-ui-image-generation.tsx
@@ -3,9 +3,14 @@
"use client";
-import { type ToolCallMessagePartComponent, useAuiState } from "@assistant-ui/react";
-import { ImageIcon, LoaderIcon } from "lucide-react";
-import { memo, useEffect, useState } from "react";
+import { Button } from "@/components/ui/button";
+import { cn } from "@/lib/utils";
+import type { ToolCallMessagePartComponent } from "@assistant-ui/react";
+import { DownloadIcon, ImageIcon, PencilIcon } from "lucide-react";
+import type { CSSProperties, MouseEvent } from "react";
+import { memo, useState } from "react";
+import { useGeneratedImageOverlay } from "./generated-image-overlay-context";
+import { Image, downloadImagePart } from "./image";
import {
ToolFallbackContent,
ToolFallbackRoot,
@@ -38,6 +43,9 @@ import {
interface ImageGenerationArgs {
prompt?: string;
kind?: string;
+ openai_image_generation_call_id?: unknown;
+ openai_response_id?: unknown;
+ openai_reasoning_item?: unknown;
}
interface ImageGenerationResult {
@@ -46,6 +54,83 @@ interface ImageGenerationResult {
size?: string;
quality?: string;
background?: string;
+ prompt?: string;
+}
+
+type GeneratedImagePart = {
+ type: "image";
+ image: string;
+ filename?: string;
+};
+
+const extensionForMime = (mime: string): string => {
+ switch (mime.toLowerCase()) {
+ case "image/jpeg":
+ case "image/jpg":
+ return "jpg";
+ case "image/webp":
+ return "webp";
+ case "image/gif":
+ return "gif";
+ case "image/svg+xml":
+ return "svg";
+ default:
+ return "png";
+ }
+};
+
+const imageFilenameFromPrompt = (prompt: string, mime: string): string => {
+ const slug = prompt
+ .trim()
+ .toLowerCase()
+ .replace(/[^a-z0-9]+/g, "-")
+ .replace(/^-|-$/g, "")
+ .slice(0, 48);
+ return `${slug || "generated-image"}.${extensionForMime(mime)}`;
+};
+
+const formatGeneratedImageLabel = (prompt: string): string => {
+ if (!prompt) {
+ return "Generated image";
+ }
+ return prompt.length > 80
+ ? `Generated image: ${prompt.slice(0, 80)}…`
+ : `Generated image: ${prompt}`;
+};
+
+const loadingDots = Array.from({ length: 64 }, (_, index) => {
+ const row = Math.floor(index / 8);
+ const col = index % 8;
+ return (
+
+ );
+});
+
+function GeneratedImagePlaceholder({ label }: { label: string }) {
+ return (
+
+
{label}
+
+ {loadingDots}
+
+
+ );
}
const ImageGenerationToolUIImpl: ToolCallMessagePartComponent = ({
@@ -53,6 +138,7 @@ const ImageGenerationToolUIImpl: ToolCallMessagePartComponent = ({
result,
status,
}) => {
+ const { openOverlay } = useGeneratedImageOverlay();
const parsedArgs = (args as ImageGenerationArgs) ?? {};
const prompt = parsedArgs.prompt ?? "";
const isRunning = status?.type === "running";
@@ -66,33 +152,74 @@ const ImageGenerationToolUIImpl: ToolCallMessagePartComponent = ({
const imageSrc = imageResult?.image_b64
? `data:${mime};base64,${imageResult.image_b64}`
: null;
+ const imageTitle =
+ imageResult?.prompt?.trim() || prompt.trim() || "Generated image";
+ const imageMetadata = [imageResult?.size, imageResult?.quality, mime]
+ .filter(Boolean)
+ .join(" · ");
+ const openaiImageGenerationCallId =
+ typeof parsedArgs.openai_image_generation_call_id === "string"
+ ? parsedArgs.openai_image_generation_call_id
+ : undefined;
+ const openaiResponseId =
+ typeof parsedArgs.openai_response_id === "string"
+ ? parsedArgs.openai_response_id
+ : undefined;
+ const imagePart: GeneratedImagePart | null = imageSrc
+ ? {
+ type: "image",
+ image: imageSrc,
+ filename: imageFilenameFromPrompt(prompt, mime),
+ }
+ : null;
- // Collapse the card once the model has resumed streaming prose
- // after the image. Mirrors CodeExecutionToolUI so the inline image
- // doesn't collapse mid-stream and the user can click to re-expand.
- const hasText = useAuiState(({ message }) =>
- message.content.some(
- (p) =>
- p.type === "text" &&
- "text" in p &&
- (p as { text: string }).text.length > 0,
- ),
- );
const [open, setOpen] = useState(true);
- useEffect(() => {
- if (isRunning) {
- setOpen(true);
- } else if (hasText && !imageSrc) {
- setOpen(false);
- }
- }, [isRunning, hasText, imageSrc]);
+ const isPendingImage = !imagePart && status?.type === "running";
const runningLabel = "Generating image…";
- const completedLabel = prompt
- ? prompt.length > 80
- ? `Generated image: ${prompt.slice(0, 80)}…`
- : `Generated image: ${prompt}`
- : "Generated image";
+ const completedLabel = formatGeneratedImageLabel(prompt);
+
+ const showPreview = () => {
+ if (!imagePart) {
+ return;
+ }
+ openOverlay({
+ image: imagePart.image,
+ title: imageTitle,
+ metadata: imageMetadata,
+ filename: imagePart.filename,
+ openaiImageGenerationCallId,
+ openaiResponseId,
+ openaiReasoningItem: parsedArgs.openai_reasoning_item,
+ });
+ };
+
+ const stopOverlayActionPropagation = (
+ event: MouseEvent
,
+ ) => {
+ event.preventDefault();
+ event.stopPropagation();
+ };
+
+ const handleDownload = (event: MouseEvent) => {
+ stopOverlayActionPropagation(event);
+ if (imagePart) {
+ downloadImagePart(imagePart);
+ }
+ };
+
+ const handleEditClick = (event: MouseEvent) => {
+ stopOverlayActionPropagation(event);
+ showPreview();
+ };
+
+ if (isPendingImage) {
+ return (
+
+
+
+ );
+ }
return (
@@ -102,20 +229,47 @@ const ImageGenerationToolUIImpl: ToolCallMessagePartComponent = ({
icon={ImageIcon}
/>
- {isRunning && !imageSrc ? (
-
-
- {runningLabel}
-
- ) : imageSrc ? (
-
-
+ {imagePart ? (
+
+
{prompt ? (
-
+
{prompt}
) : null}
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index be2799e727..49a5eebd6b 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-import { getAuthToken } from "@/features/auth/session";
+import { getAuthToken } from "@/features/auth";
import { apiUrl } from "@/lib/api-base";
import { toast } from "@/lib/toast";
import type { MessageTiming, ToolCallMessagePart } from "@assistant-ui/core";
@@ -30,12 +30,17 @@ import {
providerSupportsBuiltinWebSearch,
providerSupportsFastMode,
} from "../provider-capabilities";
-import { useChatRuntimeStore } from "../stores/chat-runtime-store";
+import {
+ type PendingImageEditReference,
+ useChatRuntimeStore,
+} from "../stores/chat-runtime-store";
import { useExternalProvidersStore } from "../stores/external-providers-store";
import { isMultimodalResponse } from "../types/api";
import type {
OpenAIChatCompletionsRequest,
+ OpenAIChatMessage,
OpenAIMessageContent,
+ OpenAIReasoningContentPart,
} from "../types/api";
import type { ChatModelSummary } from "../types/runtime";
import { getImageInputUnavailableReason } from "../utils/image-input-support";
@@ -399,37 +404,30 @@ function collectImageParts(
message: RunMessage,
): Array<{ type: "image_url"; image_url: { url: string } }> {
const parts: Array<{ type: "image_url"; image_url: { url: string } }> = [];
+ const pushImagePart = (part: { type: string }) => {
+ if (part.type !== "image" || !("image" in part)) {
+ return;
+ }
+ const src = (part as { image: string }).image;
+ if (!src) {
+ return;
+ }
+ parts.push({
+ type: "image_url",
+ image_url: {
+ url: src.startsWith("data:") ? src : `data:image/png;base64,${src}`,
+ },
+ });
+ };
for (const part of message.content ?? []) {
- if (part.type === "image" && "image" in part) {
- const src = (part as { image: string }).image;
- if (src) {
- parts.push({
- type: "image_url",
- image_url: {
- url: src.startsWith("data:") ? src : `data:image/png;base64,${src}`,
- },
- });
- }
- }
+ pushImagePart(part);
}
if ("attachments" in message && (message.attachments?.length ?? 0) > 0) {
for (const attachment of message.attachments ?? []) {
for (const part of attachment.content ?? []) {
- if (part.type === "image" && "image" in part) {
- const src = (part as { image: string }).image;
- if (src) {
- parts.push({
- type: "image_url",
- image_url: {
- url: src.startsWith("data:")
- ? src
- : `data:image/png;base64,${src}`,
- },
- });
- }
- }
+ pushImagePart(part);
}
}
}
@@ -437,6 +435,66 @@ function collectImageParts(
return parts;
}
+function normalizeOpenAIReasoningItem(
+ value: unknown,
+): OpenAIReasoningContentPart | null {
+ if (!value || typeof value !== "object") {
+ return null;
+ }
+ const item = value as Record;
+ if (item.type !== "reasoning" || typeof item.id !== "string" || !item.id) {
+ return null;
+ }
+ const summary = Array.isArray(item.summary)
+ ? item.summary.flatMap((part) => {
+ if (!part || typeof part !== "object") {
+ return [];
+ }
+ const summaryPart = part as Record;
+ return summaryPart.type === "summary_text" &&
+ typeof summaryPart.text === "string"
+ ? [{ type: "summary_text" as const, text: summaryPart.text }]
+ : [];
+ })
+ : [];
+ const normalized: OpenAIReasoningContentPart = {
+ type: "reasoning",
+ id: item.id,
+ summary,
+ };
+ if (
+ item.status === "in_progress" ||
+ item.status === "completed" ||
+ item.status === "incomplete"
+ ) {
+ normalized.status = item.status;
+ }
+ return normalized;
+}
+
+function toOpenAIImageEditReferenceMessage(
+ reference: PendingImageEditReference,
+): OpenAIChatMessage | null {
+ if (!reference.openaiImageGenerationCallId) {
+ return null;
+ }
+ const content: Exclude = [];
+ const reasoningItem = normalizeOpenAIReasoningItem(
+ reference.openaiReasoningItem,
+ );
+ if (reasoningItem) {
+ content.push(reasoningItem);
+ }
+ content.push({
+ type: "image_generation_call",
+ id: reference.openaiImageGenerationCallId,
+ ...(reference.openaiResponseId
+ ? { response_id: reference.openaiResponseId }
+ : {}),
+ });
+ return { role: "assistant", content };
+}
+
// Refusal flag stamped on assistant metadata when the backend emits the
// `anthropic_refusal` _toolEvent. We drop the refused pair from the next
// request body (Anthropic guidance: leaving refusals in context keeps
@@ -480,10 +538,16 @@ function toOpenAIMessage(message: RunMessage): {
if (imageParts.length > 0) {
return {
role: message.role,
- content: [{ type: "text", text: textContent }, ...imageParts],
+ content: [
+ ...(textContent ? [{ type: "text" as const, text: textContent }] : []),
+ ...imageParts,
+ ],
};
}
+ if (!textContent) {
+ return null;
+ }
return { role: message.role, content: textContent };
}
@@ -918,17 +982,52 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
// the user switches chats while waiting for model load / auto-load.
const resolvedThreadId =
(unstable_threadId ?? runtime.activeThreadId) || undefined;
+ const resolvedThreadKey = resolvedThreadId ?? null;
+ const pendingImageEditReferenceForRun = runtime.pendingImageEditReference;
+ const selectedImageEditReference =
+ (pendingImageEditReferenceForRun?.threadId ?? null) ===
+ resolvedThreadKey
+ ? pendingImageEditReferenceForRun
+ : null;
+ const clearSelectedImageEditReference = () => {
+ if (!selectedImageEditReference) {
+ return;
+ }
+ const store = useChatRuntimeStore.getState();
+ const pending = store.pendingImageEditReference;
+ if (
+ pending?.openaiImageGenerationCallId ===
+ selectedImageEditReference.openaiImageGenerationCallId &&
+ pending.openaiResponseId ===
+ selectedImageEditReference.openaiResponseId &&
+ (pending.threadId ?? null) ===
+ (selectedImageEditReference.threadId ?? null)
+ ) {
+ store.clearPendingImageEditReference();
+ }
+ };
// Wait for in-progress model load to finish before inferring
if (runtime.modelLoading) {
toast.info("Waiting for model to finish loading…");
- await waitForModelReady(abortSignal);
+ try {
+ await waitForModelReady(abortSignal);
+ } catch (error) {
+ clearSelectedImageEditReference();
+ throw error;
+ }
}
if (!useChatRuntimeStore.getState().params.checkpoint) {
// Auto-load the smallest downloaded model
- const { loaded, blockedByTrustRemoteCode } =
- await autoLoadSmallestModel();
+ let loaded: boolean;
+ let blockedByTrustRemoteCode: boolean;
+ try {
+ ({ loaded, blockedByTrustRemoteCode } = await autoLoadSmallestModel());
+ } catch (error) {
+ clearSelectedImageEditReference();
+ throw error;
+ }
if (!loaded) {
toast.error(
blockedByTrustRemoteCode
@@ -940,6 +1039,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
: "Pick a model in the top bar, then retry.",
},
);
+ clearSelectedImageEditReference();
throw new Error("Load a model first.");
}
}
@@ -964,6 +1064,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
description:
"Turn on Enable connections in Settings → Connections to use hosted models.",
});
+ clearSelectedImageEditReference();
throw new Error("Connections disabled.");
}
const externalProvider = isExternalRequest
@@ -979,6 +1080,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
toast.error("Connection not found.", {
description: "Open Settings → Connections and add it again.",
});
+ clearSelectedImageEditReference();
throw new Error("Connection not found.");
}
// Local providers (llama.cpp / vLLM / Ollama) allow an empty key — only block hosted providers.
@@ -989,36 +1091,34 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
toast.error("Missing API key for selected connection.", {
description: "Open Settings → Connections and set the API key again.",
});
+ clearSelectedImageEditReference();
throw new Error("Missing connection API key.");
}
- const webSearchEnabledForThisTurn =
- Boolean(
- externalProvider &&
- toolsEnabled &&
- providerSupportsBuiltinWebSearch(externalProvider.providerType),
- );
- const codeExecEnabledForThisTurn =
- Boolean(
- externalProvider &&
- externalSelection &&
- codeToolsEnabled &&
- providerSupportsBuiltinCodeExecution(
- externalProvider.providerType,
- externalSelection.modelId,
- externalProvider.baseUrl,
- ),
- );
+ const webSearchEnabledForThisTurn = Boolean(
+ externalProvider &&
+ toolsEnabled &&
+ providerSupportsBuiltinWebSearch(externalProvider.providerType),
+ );
+ const codeExecEnabledForThisTurn = Boolean(
+ externalProvider &&
+ externalSelection &&
+ codeToolsEnabled &&
+ providerSupportsBuiltinCodeExecution(
+ externalProvider.providerType,
+ externalSelection.modelId,
+ externalProvider.baseUrl,
+ ),
+ );
// Fetch pill is independent of Search (Anthropic bills web_fetch
// separately from web_search). Sourced from `webFetchToolsEnabled`;
// on providers without web_fetch the toggle is forced off in
// chat-page's runtime setState.
- const webFetchEnabledForThisTurn =
- Boolean(
- externalProvider &&
- webFetchToolsEnabled &&
- providerSupportsBuiltinWebFetch(externalProvider.providerType),
- );
+ const webFetchEnabledForThisTurn = Boolean(
+ externalProvider &&
+ webFetchToolsEnabled &&
+ providerSupportsBuiltinWebFetch(externalProvider.providerType),
+ );
const providerShipsWebFetch = Boolean(
externalProvider &&
providerSupportsBuiltinWebFetch(externalProvider.providerType),
@@ -1038,6 +1138,15 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
),
);
+ if (selectedImageEditReference && !imageGenerationEnabledForThisTurn) {
+ clearSelectedImageEditReference();
+ toast.error("Image editing is unavailable", {
+ description:
+ "Select an OpenAI image-generation model, then retry the edit.",
+ });
+ throw new Error("Image generation edit unavailable.");
+ }
+
// Two-pass build: a refused assistant turn also drops the user
// prompt that triggered it (leaving it in context re-triggers
// the classifier). Refusal flag rides assistant
@@ -1060,6 +1169,27 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
.filter((message): message is NonNullable =>
Boolean(message),
);
+ if (selectedImageEditReference) {
+ const referenceMessage = toOpenAIImageEditReferenceMessage(
+ selectedImageEditReference,
+ );
+ if (!referenceMessage) {
+ clearSelectedImageEditReference();
+ toast.error("This generated image cannot be edited", {
+ description:
+ "The original image reference is missing. Generate the image again, then retry the edit.",
+ });
+ throw new Error("Generated image edit reference missing.");
+ }
+ let insertAt = outboundMessages.length;
+ for (let i = outboundMessages.length - 1; i >= 0; i -= 1) {
+ if (outboundMessages[i]?.role === "user") {
+ insertAt = i;
+ break;
+ }
+ }
+ outboundMessages.splice(insertAt, 0, referenceMessage);
+ }
const safeSystemPrompt =
typeof params.systemPrompt === "string" ? params.systemPrompt : "";
@@ -1084,24 +1214,45 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
// was on and suppressed live web_fetch calls.
const anyWebEnabledForThisTurn =
webSearchEnabledForThisTurn || webFetchEnabledForThisTurn;
- if (!anyWebEnabledForThisTurn && !codeExecEnabledForThisTurn) {
+ if (
+ !anyWebEnabledForThisTurn &&
+ !codeExecEnabledForThisTurn &&
+ !imageGenerationEnabledForThisTurn
+ ) {
+ disabledToolGuard =
+ `You do not have ${webLabel}, code execution, or image generation tools in this conversation. ` +
+ "Answer from your own knowledge. " +
+ "If a request genuinely requires tool use, live data fetch, running code, or image generation, " +
+ "inform the user that you do not have access to these capabilities. " +
+ "Do not return tool-call syntax inside your response.";
+ } else if (!anyWebEnabledForThisTurn && !codeExecEnabledForThisTurn) {
disabledToolGuard =
`You do not have ${webLabel} or code execution tools in this conversation. ` +
- "Answer from your own knowledge. " +
- "If a request genuinely requires tool use, live data fetch or running code, " +
+ "You may still use image generation tools when they are available and useful. " +
+ "If a request genuinely requires live data fetch or running code, " +
"inform the user that you do not have access to these capabilities. " +
"Do not return tool-call syntax inside your response.";
} else if (!anyWebEnabledForThisTurn) {
+ const availableTools = [
+ codeExecEnabledForThisTurn ? "code execution" : null,
+ imageGenerationEnabledForThisTurn ? "image generation" : null,
+ ].filter(Boolean);
disabledToolGuard =
`You do not have ${webLabel} tools in this conversation. ` +
- "You may still use code execution tools when they are available and useful. " +
+ (availableTools.length > 0
+ ? `You may still use ${availableTools.join(" and ")} tools when they are available and useful. `
+ : "") +
"If a request genuinely requires live data fetch or web search tool use, " +
"inform the user that you do not have access to these capabilities. " +
"Do not return tool-call syntax inside your response.";
} else if (!codeExecEnabledForThisTurn) {
+ const availableTools = [
+ webLabel,
+ imageGenerationEnabledForThisTurn ? "image generation" : null,
+ ].filter(Boolean);
disabledToolGuard =
"You do not have code execution tools in this conversation. " +
- `You may still use ${webLabel} tools when they are available and useful. ` +
+ `You may still use ${availableTools.join(" and ")} tools when they are available and useful. ` +
"If a request genuinely requires running code or code execution tool use, " +
"inform the user that you do not have access to these capabilities. " +
"Do not return tool-call syntax inside your response.";
@@ -1163,6 +1314,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
const gatedThreadKey = resolvedThreadId || "__default";
runtime.setThreadRunning(gatedThreadKey, true);
runtime.setThreadRunning(gatedThreadKey, false);
+ clearSelectedImageEditReference();
throw new Error(imageGateReason);
}
}
@@ -1474,8 +1626,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
) {
void updateStoredChatThreadEventually(t.id, {
openaiCodeExecContainerId: null,
- })
- .catch(() => {});
+ }).catch(() => {});
continue;
}
openaiCodeExecContainerId = t.openaiCodeExecContainerId;
@@ -1519,8 +1670,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
openaiCodeExecContainerId = created.id;
void updateStoredChatThreadEventually(resolvedThreadId, {
openaiCodeExecContainerId: created.id,
- })
- .catch(() => {});
+ }).catch(() => {});
} catch {
// Fall back to backend's container_auto path on
// failure — keeps the chat moving; the next turn
@@ -1628,7 +1778,9 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
// attaches `cache_control.ttl` when the value is one of
// "5m" / "1h" (see external_provider.py near line 1375),
// so unknown values are a no-op end-to-end.
- ...(supportsProviderPromptCacheTtl(externalProvider.providerType) &&
+ ...(supportsProviderPromptCacheTtl(
+ externalProvider.providerType,
+ ) &&
(externalProvider.enablePromptCaching ?? true) &&
isPromptCacheTtl(externalProvider.promptCacheTtl)
? { prompt_cache_ttl: externalProvider.promptCacheTtl }
@@ -1706,10 +1858,15 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
let retriedWithRefreshedKey = false;
while (true) {
try {
- const stream = streamChatCompletions(
- await buildRequestPayload(retriedWithRefreshedKey),
- abortSignal,
- );
+ let requestPayload: OpenAIChatCompletionsRequest;
+ try {
+ requestPayload = await buildRequestPayload(retriedWithRefreshedKey);
+ } catch (error) {
+ clearSelectedImageEditReference();
+ throw error;
+ }
+ clearSelectedImageEditReference();
+ const stream = streamChatCompletions(requestPayload, abortSignal);
for await (const chunk of stream) {
// Handle tool status events
@@ -1777,8 +1934,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
: "openaiCodeExecContainerId";
void updateStoredChatThreadEventually(resolvedThreadId, {
[field]: null,
- })
- .catch(() => {});
+ }).catch(() => {});
}
continue;
}
@@ -1822,6 +1978,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
size?: string;
quality?: string;
background?: string;
+ prompt?: string;
};
const imageB64 = toolEvent.image_b64 as string | undefined;
if (
@@ -1843,6 +2000,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
size: toolEvent.size as string | undefined,
quality: toolEvent.quality as string | undefined,
background: toolEvent.background as string | undefined,
+ prompt: toolEvent.prompt as string | undefined,
};
} else if (imgIdx !== -1) {
const text = rawResult.slice(0, imgIdx);
@@ -1860,8 +2018,20 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
} else {
parsedResult = rawResult;
}
+ const nextArgs =
+ toolEvent.arguments &&
+ typeof toolEvent.arguments === "object"
+ ? (toolEvent.arguments as ToolCallMessagePart["args"])
+ : undefined;
+ const mergedArgs = nextArgs
+ ? { ...(toolCallParts[idx].args ?? {}), ...nextArgs }
+ : toolCallParts[idx].args;
toolCallParts[idx] = {
...toolCallParts[idx],
+ args: mergedArgs,
+ argsText: mergedArgs
+ ? JSON.stringify(mergedArgs)
+ : toolCallParts[idx].argsText,
result: parsedResult,
};
}
diff --git a/studio/frontend/src/features/chat/provider-capabilities.ts b/studio/frontend/src/features/chat/provider-capabilities.ts
index 1748e098b9..ef805305be 100644
--- a/studio/frontend/src/features/chat/provider-capabilities.ts
+++ b/studio/frontend/src/features/chat/provider-capabilities.ts
@@ -207,15 +207,21 @@ const OPENAI_CODE_EXECUTION_MODEL_PREFIXES = [
/**
* Strict check that a provider configuration points at OpenAI's
- * managed cloud (api.openai.com), as opposed to a custom OpenAI-compat
- * backend (ollama / llama.cpp / vLLM / generic "custom" preset). The
- * shell tool ONLY exists on OpenAI cloud; sending it to anything else
- * 400s the request. Mirror of the backend's
- * `is_openai_cloud = "api.openai.com" in self.base_url` guard.
+ * managed cloud (api.openai.com) or Azure OpenAI Foundry
+ * (*.openai.azure.com), as opposed to a custom OpenAI-compat backend
+ * (ollama / llama.cpp / vLLM / generic "custom" preset). The shell and
+ * image-generation tools only exist on cloud backends; sending them to
+ * anything else 400s the request. Mirror of the backend's
+ * `_is_openai_family_cloud` host check.
*/
function isOpenAICloudBaseUrl(baseUrl: string | null | undefined): boolean {
if (!baseUrl) return true; // No override → uses the default openai.com base.
- return baseUrl.trim().toLowerCase().includes("api.openai.com");
+ try {
+ const host = new URL(baseUrl).hostname.toLowerCase();
+ return host === "api.openai.com" || host.endsWith(".openai.azure.com");
+ } catch {
+ return false;
+ }
}
export function providerSupportsBuiltinCodeExecution(
diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
index 9a6f0c982f..c78f02a474 100644
--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
@@ -64,6 +64,12 @@ function saveLastExternalCheckpoint(value: string | null): void {
}
export type ReasoningStyle = "enable_thinking" | "reasoning_effort";
+export type PendingImageEditReference = {
+ threadId: string | null;
+ openaiImageGenerationCallId: string;
+ openaiResponseId?: string;
+ openaiReasoningItem?: unknown;
+};
export type ReasoningEffort =
| "none"
| "minimal"
@@ -300,6 +306,7 @@ type ChatRuntimeStore = {
settingsPanelOpen: boolean;
pendingAudioBase64: string | null;
pendingAudioName: string | null;
+ pendingImageEditReference: PendingImageEditReference | null;
contextUsage: {
promptTokens: number;
completionTokens: number;
@@ -353,6 +360,10 @@ type ChatRuntimeStore = {
setChatTemplateOverride: (template: string | null) => void;
setPendingAudio: (base64: string, name: string) => void;
clearPendingAudio: () => void;
+ setPendingImageEditReference: (
+ reference: PendingImageEditReference | null,
+ ) => void;
+ clearPendingImageEditReference: () => void;
setContextUsage: (usage: ChatRuntimeStore["contextUsage"]) => void;
};
@@ -607,6 +618,7 @@ export const useChatRuntimeStore = create((set, get) => ({
settingsPanelOpen: false,
pendingAudioBase64: null,
pendingAudioName: null,
+ pendingImageEditReference: null,
contextUsage: null,
modelLoading: false,
activeNativePathToken: null,
@@ -793,6 +805,7 @@ export const useChatRuntimeStore = create((set, get) => ({
defaultChatTemplate: null,
chatTemplateOverride: null,
loadedChatTemplateOverride: null,
+ pendingImageEditReference: null,
}));
},
setReasoningEnabled: (reasoningEnabled, options) =>
@@ -884,5 +897,9 @@ export const useChatRuntimeStore = create((set, get) => ({
set({ pendingAudioBase64: base64, pendingAudioName: name }),
clearPendingAudio: () =>
set({ pendingAudioBase64: null, pendingAudioName: null }),
+ setPendingImageEditReference: (pendingImageEditReference) =>
+ set({ pendingImageEditReference }),
+ clearPendingImageEditReference: () =>
+ set({ pendingImageEditReference: null }),
setContextUsage: (contextUsage) => set({ contextUsage }),
}));
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index f18407413d..b7a61d24b6 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -192,12 +192,31 @@ export interface AudioGenerationResponse {
}>;
}
-export type OpenAIMessageContent =
- | string
- | Array<
- | { type: "text"; text: string }
- | { type: "image_url"; image_url: { url: string } }
- >;
+export type OpenAIReasoningSummaryPart = {
+ type: "summary_text";
+ text: string;
+};
+
+export type OpenAIReasoningContentPart = {
+ type: "reasoning";
+ id: string;
+ summary: OpenAIReasoningSummaryPart[];
+ status?: "in_progress" | "completed" | "incomplete";
+};
+
+export type OpenAIImageGenerationCallContentPart = {
+ type: "image_generation_call";
+ id: string;
+ response_id?: string;
+};
+
+export type OpenAIMessageContentPart =
+ | { type: "text"; text: string }
+ | { type: "image_url"; image_url: { url: string } }
+ | OpenAIReasoningContentPart
+ | OpenAIImageGenerationCallContentPart;
+
+export type OpenAIMessageContent = string | OpenAIMessageContentPart[];
export interface OpenAIChatMessage {
role: "system" | "user" | "assistant";
diff --git a/studio/frontend/src/index.css b/studio/frontend/src/index.css
index 8f132cd95b..8a1fe13678 100644
--- a/studio/frontend/src/index.css
+++ b/studio/frontend/src/index.css
@@ -1188,6 +1188,53 @@
border-color: var(--border) !important;
}
+.generated-image-loading-card {
+ position: relative;
+ overflow: hidden;
+ contain: paint;
+}
+
+.generated-image-loading-wave {
+ position: relative;
+ display: grid;
+ grid-template-columns: repeat(8, minmax(0, 1fr));
+ gap: 14px;
+ width: min(66%, 18rem);
+ padding: 1.5rem;
+ border-radius: 1.5rem;
+}
+
+.generated-image-loading-dot {
+ width: 7px;
+ height: 7px;
+ border-radius: 9999px;
+ background: color-mix(in oklch, var(--muted-foreground) 82%, var(--primary));
+ opacity: 0.12;
+ transform: translate3d(0, 4px, 0) scale(0.72);
+ animation: generated-image-dot-wave 1850ms var(--ease-out-quart) infinite;
+ animation-delay: calc((var(--dot-row) * 72ms) + (var(--dot-col) * 72ms));
+ will-change: transform, opacity;
+}
+
+@keyframes generated-image-dot-wave {
+ 0%,
+ 22%,
+ 100% {
+ opacity: 0.1;
+ transform: translate3d(0, 4px, 0) scale(0.72);
+ }
+
+ 46% {
+ opacity: 0.46;
+ transform: translate3d(0, -3px, 0) scale(0.96);
+ }
+
+ 66% {
+ opacity: 0.2;
+ transform: translate3d(0, 0, 0) scale(0.82);
+ }
+}
+
/*
* prefers-reduced-motion: honour the OS-level "reduce motion" preference.
* Tailwind animate-in/out, Radix open/close transforms, infinite shine/pulse