unsloth/studio/frontend/src/components/assistant-ui/markdown-text.tsx
Daniel Han c8d311a053
feat(studio): display images from Python tool execution in chat UI (#4778)
* feat(studio): display images from Python tool execution in chat UI

When the model calls the Python tool to create a matplotlib plot or
other image file, the image now displays inline in the chat output
instead of being invisible to the user.

Backend:
- Detect new image files (png/jpg/gif/webp/bmp) after Python subprocess
  completes by diffing os.listdir before/after execution
- Append __IMAGES__ sentinel to tool result for frontend consumption
- Strip sentinel before injecting result into LLM context (role: tool)
  so the model never sees file paths
- Add GET /sandbox/{session_id}/{filename} endpoint with JWT auth
  (header or query param), path traversal protection, extension
  allowlist, realpath containment check, and nosniff header

Frontend:
- Parse __IMAGES__ sentinel in tool_end SSE events, create structured
  result with text/images/sessionId
- Render <img> tags in Python tool UI pointing at the sandbox endpoint

Also fixes a bug where SyntaxError in user code was misreported as
"unsafe code detected" instead of showing the actual Python traceback.
The _check_code_safety function now lets SyntaxError pass through to
the subprocess for a proper error message.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix(studio): improve SVG detection and strip XML preamble

Handle <?xml ...?> declarations before <svg> tags in code fences,
strip XML declaration from SVGs before data URI rendering, and
update the sloth suggestion prompt to request showing code.

* fix(studio): persist parentId so retries survive reload

The append() handler was destructuring only { message } from
ExportedMessageRepositoryItem and discarding parentId. When loading
a saved thread, load() used ExportedMessageRepository.fromArray()
which chains all messages sequentially, flattening retry branches
into a linear list.

Now append() writes parentId to the MessageRecord, and load()
reconstructs the tree when parentIds are present. Old threads
without parentId fall back to the existing fromArray() behavior.

* fix(studio): address review findings for image display and retry persistence

Image detection:
- Use mtime comparison instead of filename-only diff so overwritten
  files (e.g. plt.savefig("chart.png") called twice) are detected

Sentinel parsing:
- Use rsplit/lastIndexOf instead of split/indexOf so user code that
  prints __IMAGES__: does not collide with the backend sentinel

Mixed legacy/new threads:
- For old messages without a stored parentId, infer sequential parent
  from the previous message instead of null, preventing multiple roots

Sandbox endpoint:
- Change Cache-Control from "public, max-age=3600" to "private,
  no-store" since these are authenticated responses

---------

Co-authored-by: Daniel Han <danielhanchen@users.noreply.github.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
2026-04-02 05:08:16 -07:00

437 lines
13 KiB
TypeScript

// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"use client";
import { copyToClipboard } from "@/lib/copy-to-clipboard";
import { preprocessLaTeX } from "@/lib/latex";
import { INTERNAL, useMessagePartText } from "@assistant-ui/react";
import { Copy02Icon, Tick02Icon } from "@hugeicons/core-free-icons";
import { HugeiconsIcon } from "@hugeicons/react";
import { code } from "@streamdown/code";
import { createMathPlugin } from "@streamdown/math";
import { mermaid } from "@streamdown/mermaid";
import { DownloadIcon, Maximize2Icon, Minimize2Icon } from "lucide-react";
import { useEffect, useMemo, useRef, useState } from "react";
import { Block, type BlockProps, Streamdown } from "streamdown";
import "katex/dist/katex.min.css";
import { AudioPlayer } from "./audio-player";
const math = createMathPlugin({ singleDollarTextMath: true });
const { withSmoothContextProvider } = INTERNAL;
const STREAMDOWN_COMPONENTS = {
a: ({
href,
children,
...props
}: React.ComponentProps<"a">) => (
<a
href={href}
target="_blank"
rel="noopener noreferrer"
className="text-primary underline underline-offset-2 decoration-primary/40 hover:decoration-primary transition-colors"
{...props}
>
{children}
</a>
),
};
const COPY_RESET_MS = 2000;
const MERMAID_SOURCE_RE = /```mermaid\s*([\s\S]*?)```/i;
const CODE_FENCE_RE = /^```([^\r\n`]*)\r?\n([\s\S]*?)\r?\n?```$/;
const ACTION_PANEL_CLASS =
"pointer-events-auto flex shrink-0 items-center gap-2 rounded-md border border-sidebar bg-sidebar/80 px-1.5 py-1 supports-[backdrop-filter]:bg-sidebar/70 supports-[backdrop-filter]:backdrop-blur";
const ACTION_BUTTON_CLASS =
"cursor-pointer p-1 text-muted-foreground transition-all hover:text-foreground disabled:cursor-not-allowed disabled:opacity-50";
type CodeFence = {
language: string | null;
source: string;
};
function getMermaidSource(blockContent: string): string | null {
const source = blockContent.match(MERMAID_SOURCE_RE)?.[1]?.trim();
return source && source.length > 0 ? source : null;
}
function getCodeFence(blockContent: string): CodeFence | null {
const match = blockContent.trimEnd().match(CODE_FENCE_RE);
if (!match) {
return null;
}
return {
language: match[1]?.trim() || null,
source: match[2],
};
}
function getCodeFilename(language: string | null) {
const extByLanguage: Record<string, string> = {
bash: "sh",
javascript: "js",
js: "js",
json: "json",
jsx: "jsx",
markdown: "md",
md: "md",
python: "py",
py: "py",
shell: "sh",
sh: "sh",
sql: "sql",
ts: "ts",
tsx: "tsx",
typescript: "ts",
svg: "svg",
yaml: "yml",
yml: "yml",
};
const normalized = language?.toLowerCase();
const fallbackExt = normalized?.replace(/[^a-z0-9]+/g, "-");
const ext = normalized
? extByLanguage[normalized] || fallbackExt || "txt"
: "txt";
return `snippet.${ext}`;
}
function isSvgFence(codeFence: CodeFence): boolean {
const lang = codeFence.language?.toLowerCase() ?? "";
if (lang === "svg") return true;
if (lang === "xml" || lang === "html") {
const trimmed = codeFence.source.trimStart();
// Match <svg directly or <?xml ...?> followed by <svg
if (trimmed.startsWith("<svg")) return true;
if (trimmed.startsWith("<?xml") && trimmed.includes("<svg")) return true;
}
return false;
}
function isHtmlFence(codeFence: CodeFence): boolean {
const lang = codeFence.language?.toLowerCase() ?? "";
return lang === "html" && !isSvgFence(codeFence);
}
const UNSAFE_SVG_RE = /<script[\s>]|on\w+\s*=|javascript:|<foreignObject[\s>]|<iframe[\s>]|<embed[\s>]|<object[\s>]/i;
function sanitizeSvg(source: string): string | null {
if (UNSAFE_SVG_RE.test(source)) return null;
// Strip XML declaration (<?xml ...?>) -- not needed for data URI
// rendering and can cause issues with some renderers.
return source.replace(/^\s*<\?xml[^?]*\?>\s*/i, "");
}
function SvgPreview({ source }: { source: string }) {
const dataUri = `data:image/svg+xml;charset=utf-8,${encodeURIComponent(source)}`;
return (
<div className="mt-2 flex justify-center rounded-lg border border-border bg-white p-4 dark:bg-neutral-100">
<img
src={dataUri}
alt="SVG preview"
style={{ maxWidth: "100%", maxHeight: 512 }}
/>
</div>
);
}
const HTML_PREVIEW_DEFAULT_HEIGHT = 400;
const HTML_PREVIEW_MAX_HEIGHT = 800;
function HtmlPreview({ source }: { source: string }) {
const iframeRef = useRef<HTMLIFrameElement>(null);
const [height, setHeight] = useState(HTML_PREVIEW_DEFAULT_HEIGHT);
const [enlarged, setEnlarged] = useState(false);
useEffect(() => {
const handler = (e: MessageEvent) => {
if (e.source !== iframeRef.current?.contentWindow) return;
if (typeof e.data?.htmlPreviewHeight === "number") {
setHeight(Math.min(Math.max(e.data.htmlPreviewHeight, 100), HTML_PREVIEW_MAX_HEIGHT));
}
};
window.addEventListener("message", handler);
return () => window.removeEventListener("message", handler);
}, []);
useEffect(() => {
if (!enlarged) return;
const handler = (e: KeyboardEvent) => {
if (e.key === "Escape") setEnlarged(false);
};
window.addEventListener("keydown", handler);
return () => window.removeEventListener("keydown", handler);
}, [enlarged]);
const resizeScript = `<script>new ResizeObserver(()=>{
parent.postMessage({htmlPreviewHeight:document.documentElement.scrollHeight},"*");
}).observe(document.documentElement);</script>`;
const srcDoc = source + resizeScript;
if (enlarged) {
return (
<>
<div className="mt-2 overflow-hidden rounded-lg border border-border" style={{ height }}>
{/* Placeholder keeps layout stable while overlay is shown */}
</div>
<div
className="fixed inset-0 z-50 flex flex-col bg-background/80 backdrop-blur-sm"
onClick={(e) => { if (e.target === e.currentTarget) setEnlarged(false); }}
>
<div className="flex items-center justify-end gap-2 px-4 py-2">
<button
type="button"
className="flex items-center gap-1.5 rounded-md border border-border bg-background px-3 py-1.5 text-sm text-muted-foreground transition-colors hover:bg-muted hover:text-foreground"
onClick={() => setEnlarged(false)}
title="Exit fullscreen (Esc)"
>
<Minimize2Icon className="size-4" />
Exit fullscreen
</button>
</div>
<div className="mx-4 mb-4 flex-1 overflow-hidden rounded-lg border border-border bg-background">
<iframe
ref={iframeRef}
srcDoc={srcDoc}
sandbox="allow-scripts"
style={{ width: "100%", height: "100%", border: "none", display: "block" }}
title="HTML preview"
/>
</div>
</div>
</>
);
}
return (
<div className="group/html-preview relative mt-2 overflow-hidden rounded-lg border border-border">
<button
type="button"
className="absolute top-2 right-2 z-10 rounded-md border border-border bg-background/80 p-1.5 text-muted-foreground opacity-0 transition-all hover:bg-muted hover:text-foreground group-hover/html-preview:opacity-100 supports-[backdrop-filter]:backdrop-blur"
onClick={() => setEnlarged(true)}
title="Enlarge preview"
>
<Maximize2Icon className="size-4" />
</button>
<iframe
ref={iframeRef}
srcDoc={srcDoc}
sandbox="allow-scripts"
style={{ width: "100%", height, border: "none", display: "block" }}
title="HTML preview"
/>
</div>
);
}
function downloadTextFile(filename: string, text: string): void {
const blob = new Blob([text], { type: "text/plain;charset=utf-8" });
const url = URL.createObjectURL(blob);
const anchor = document.createElement("a");
anchor.href = url;
anchor.download = filename;
document.body.appendChild(anchor);
anchor.click();
document.body.removeChild(anchor);
window.setTimeout(() => URL.revokeObjectURL(url), 0);
}
function useCopiedState() {
const [copied, setCopied] = useState(false);
const resetTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
useEffect(() => {
return () => {
if (resetTimeoutRef.current) {
clearTimeout(resetTimeoutRef.current);
}
};
}, []);
const showCopied = () => {
setCopied(true);
if (resetTimeoutRef.current) {
clearTimeout(resetTimeoutRef.current);
}
resetTimeoutRef.current = setTimeout(() => {
setCopied(false);
resetTimeoutRef.current = null;
}, COPY_RESET_MS);
};
return { copied, showCopied };
}
function MermaidCopyButton({ source }: { source: string }) {
const { copied, showCopied } = useCopiedState();
return (
<button
type="button"
className="absolute top-3.5 right-20 z-20 cursor-pointer text-muted-foreground transition-all hover:text-foreground"
title="Copy Mermaid source"
onClick={() => {
if (!copyToClipboard(source)) {
return;
}
showCopied();
}}
>
<HugeiconsIcon
icon={copied ? Tick02Icon : Copy02Icon}
className="size-5"
/>
</button>
);
}
function CodeBlockActions({
disabled,
language,
source,
}: {
disabled: boolean;
language: string | null;
source: string;
}) {
const { copied, showCopied } = useCopiedState();
return (
<div className="pointer-events-none absolute top-3.5 right-3 z-20 flex items-center justify-end">
<div className={ACTION_PANEL_CLASS}>
<button
type="button"
className={ACTION_BUTTON_CLASS}
title="Copy code"
disabled={disabled}
onClick={() => {
if (!copyToClipboard(source)) {
return;
}
showCopied();
}}
>
<HugeiconsIcon
icon={copied ? Tick02Icon : Copy02Icon}
className="size-3.5"
/>
</button>
<button
type="button"
className={ACTION_BUTTON_CLASS}
title="Download file"
disabled={disabled}
onClick={() => {
downloadTextFile(getCodeFilename(language), source);
}}
>
<DownloadIcon className="size-3.5" />
</button>
</div>
</div>
);
}
function StreamdownBlock(props: BlockProps) {
const hasMermaidFence = props.content.includes("```mermaid");
const mermaidSource = getMermaidSource(props.content);
const codeFence = getCodeFence(props.content);
if (props.isIncomplete && hasMermaidFence) {
return (
<div className="my-4 flex h-48 items-center justify-center rounded-xl border border-border bg-muted/30 text-sm text-muted-foreground animate-pulse">
Loading diagram...
</div>
);
}
if (props.isIncomplete && codeFence && isSvgFence(codeFence)) {
return (
<div className="relative isolate">
<div className="my-4 rounded-xl border border-border bg-muted/30 p-4">
<div className="mb-2 text-xs font-medium text-muted-foreground">svg</div>
<pre className="overflow-x-auto text-xs text-muted-foreground whitespace-pre-wrap break-all">
<code>{codeFence.source}</code>
</pre>
</div>
</div>
);
}
if (props.isIncomplete && codeFence && isHtmlFence(codeFence)) {
return (
<div className="my-4 flex h-48 items-center justify-center rounded-xl border border-border bg-muted/30 text-sm text-muted-foreground animate-pulse">
Loading preview...
</div>
);
}
if (mermaidSource) {
return (
<div className="relative isolate">
<Block {...props} />
<MermaidCopyButton source={mermaidSource} />
</div>
);
}
if (codeFence) {
const svgSource = !props.isIncomplete && isSvgFence(codeFence) ? sanitizeSvg(codeFence.source) : null;
const htmlSource = !props.isIncomplete && isHtmlFence(codeFence) ? codeFence.source : null;
return (
<>
<div className="relative isolate">
<Block {...props} />
<CodeBlockActions
disabled={props.isIncomplete}
language={codeFence.language}
source={codeFence.source}
/>
</div>
{svgSource && <SvgPreview source={svgSource} />}
{htmlSource && <HtmlPreview source={htmlSource} />}
</>
);
}
return <Block {...props} />;
}
const AUDIO_PLAYER_RE = /<audio-player\s+src="([^"]+)"\s*\/>/;
const MarkdownTextImpl = () => {
const { text, status } = useMessagePartText();
const processedText = useMemo(() => preprocessLaTeX(text), [text]);
const audioMatch = text.match(AUDIO_PLAYER_RE);
if (audioMatch) {
return <AudioPlayer src={audioMatch[1]} />;
}
return (
<div data-status={status.type} className="min-w-0 max-w-full">
<Streamdown
mode="streaming"
isAnimating={status.type === "running"}
plugins={{ code, math, mermaid }}
components={STREAMDOWN_COMPONENTS}
controls={{
code: false,
mermaid: {
fullscreen: true,
download: true,
copy: false,
panZoom: true,
},
}}
shikiTheme={["github-light", "github-dark"]}
BlockComponent={StreamdownBlock}
>
{processedText}
</Streamdown>
</div>
);
};
export const MarkdownText = withSmoothContextProvider(MarkdownTextImpl);