feat(tui): render interactive transcript images

This commit is contained in:
Simon Klee 2026-08-03 08:20:05 +02:00
commit f89ac17f17
No known key found for this signature in database
GPG key ID: B91696044D47BEA3
4 changed files with 98 additions and 4 deletions

View file

@ -84,6 +84,15 @@ export const settings: Setting[] = [
values: ["none", "auto"], values: ["none", "auto"],
keywords: ["transcript", "messages"], keywords: ["transcript", "messages"],
}, },
{
title: "Transcript images",
category: "Session",
path: ["session", "image_preview"],
default: false,
values: [false, true],
labels: ["off", "on"],
keywords: ["attachments", "images", "tool output"],
},
{ {
title: "Enabled", title: "Enabled",
category: "Tabs", category: "Tabs",

View file

@ -122,6 +122,9 @@ export const Info = Schema.Struct({
grouping: Schema.optional(Schema.Literals(["auto", "none"])).annotate({ grouping: Schema.optional(Schema.Literals(["auto", "none"])).annotate({
description: "Group related transcript items automatically or render each item separately", description: "Group related transcript items automatically or render each item separately",
}), }),
image_preview: Schema.optional(Schema.Boolean).annotate({
description: "Show user attachment and tool-result images in the session transcript",
}),
markdown: Schema.optional(Schema.Literals(["source", "rendered"])).annotate({ markdown: Schema.optional(Schema.Literals(["source", "rendered"])).annotate({
description: "Show Markdown syntax markers or conceal them in rendered transcript content", description: "Show Markdown syntax markers or conceal them in rendered transcript content",
}), }),

View file

@ -24,7 +24,7 @@ import { useTuiPaths, useTuiTerminalEnvironment } from "../../context/runtime"
import { Spinner, SPINNER_FRAMES } from "../../component/spinner" import { Spinner, SPINNER_FRAMES } from "../../component/spinner"
import { PatchDiff } from "../../component/patch-diff" import { PatchDiff } from "../../component/patch-diff"
import { ThemeContextProvider, useTheme, useThemes } from "../../context/theme" import { ThemeContextProvider, useTheme, useThemes } from "../../context/theme"
import { BoxRenderable, ScrollBoxRenderable, addDefaultParsers, TextAttributes, RGBA } from "@opentui/core" import { BoxRenderable, ScrollBoxRenderable, addDefaultParsers, TextAttributes, RGBA, MouseEvent } from "@opentui/core"
import { Prompt, type PromptRef } from "../../component/prompt" import { Prompt, type PromptRef } from "../../component/prompt"
import type { import type {
ModelInfo, ModelInfo,
@ -53,6 +53,7 @@ import { useEditorContext } from "../../context/editor"
import { openEditor } from "../../editor" import { openEditor } from "../../editor"
import { useDialog } from "../../ui/dialog" import { useDialog } from "../../ui/dialog"
import { DialogSessionRename } from "../../component/dialog-session-rename" import { DialogSessionRename } from "../../component/dialog-session-rename"
import { DialogImagePreview } from "../../component/dialog-image-preview"
import { DialogMessage } from "./dialog-message" import { DialogMessage } from "./dialog-message"
import { DialogFork } from "./dialog-fork" import { DialogFork } from "./dialog-fork"
import { DialogTimeline } from "./dialog-timeline" import { DialogTimeline } from "./dialog-timeline"
@ -1531,8 +1532,9 @@ function SessionGroupView(props: {
</InlineToolRow> </InlineToolRow>
</Show> </Show>
<Show when={expanded() && grouped().length > 0}> <Show when={expanded() && grouped().length > 0}>
<For each={grouped()}>{(part) => <ToolPart part={part} />}</For> <For each={grouped()}>{(part) => <ToolPart part={part} images={false} />}</For>
</Show> </Show>
<ToolImages parts={grouped()} />
<For each={pending()}>{(part) => <ToolPart part={part} />}</For> <For each={pending()}>{(part) => <ToolPart part={part} />}</For>
</Show> </Show>
</Show> </Show>
@ -1842,6 +1844,11 @@ function UserMessage(props: { message: SessionMessageUser }) {
const data = useData() const data = useData()
const local = useLocal() const local = useLocal()
const files = createMemo(() => props.message.files ?? []) const files = createMemo(() => props.message.files ?? [])
const images = createMemo(() =>
files().flatMap((file) =>
file.mime.startsWith("image/") ? [{ uri: `data:${file.mime};base64,${file.data}` }] : [],
),
)
const themes = useThemes() const themes = useThemes()
const theme = useTheme("elevated") const theme = useTheme("elevated")
const mode = themes.mode const mode = themes.mode
@ -1861,6 +1868,7 @@ function UserMessage(props: { message: SessionMessageUser }) {
borderColor={queued() ? theme.border.default : color()} borderColor={queued() ? theme.border.default : color()}
customBorderChars={SplitBorder.customBorderChars} customBorderChars={SplitBorder.customBorderChars}
> >
<SessionImages images={images()} />
<box <box
onMouseOver={() => { onMouseOver={() => {
setHover(true) setHover(true)
@ -2084,7 +2092,7 @@ function TextPart(props: { last: boolean; part: SessionMessageAssistantText }) {
// Pending messages moved to individual tool pending functions // Pending messages moved to individual tool pending functions
function ToolPart(props: { part: SessionMessageAssistantTool }) { function ToolPart(props: { part: SessionMessageAssistantTool; images?: boolean }) {
const display = createMemo(() => toolDisplay(props.part.name)) const display = createMemo(() => toolDisplay(props.part.name))
const toolprops = { const toolprops = {
@ -2108,7 +2116,7 @@ function ToolPart(props: { part: SessionMessageAssistantTool }) {
}, },
} }
return ( const content = (
<Switch> <Switch>
<Match when={display() === "shell"}> <Match when={display() === "shell"}>
<Shell {...toolprops} /> <Shell {...toolprops} />
@ -2154,6 +2162,79 @@ function ToolPart(props: { part: SessionMessageAssistantTool }) {
</Match> </Match>
</Switch> </Switch>
) )
return [
content,
<Show when={props.images !== false}>
<ToolImages parts={[props.part]} />
</Show>,
]
}
function ToolImages(props: { parts: readonly SessionMessageAssistantTool[] }) {
const images = createMemo(() => props.parts.flatMap(inlineToolImages))
return <SessionImages images={images()} />
}
function SessionImages(props: { images: readonly { uri: string }[] }) {
const ctx = use()
const dialog = useDialog()
const dimensions = useTerminalDimensions()
const images = createMemo(() => (ctx.config.session?.image_preview ? props.images : []))
const height = createMemo(() => Math.max(4, Math.min(8, Math.floor(dimensions().height / 4))))
const visible = createMemo(() => images().slice(0, 3))
return (
<Show when={visible().length > 0}>
<box flexDirection="row" flexShrink={0} paddingTop={1} paddingLeft={2} paddingRight={2} paddingBottom={1} gap={1}>
<For each={visible()}>
{(image, index) => {
const [failed, setFailed] = createSignal(false)
return (
<box
width={height() * 2}
height={height()}
flexBasis={height() * 2}
flexShrink={1}
alignItems="center"
justifyContent="center"
onMouseUp={(event: MouseEvent) => {
if (event.button !== 0) return
event.stopPropagation()
dialog.replace(() => <DialogImagePreview images={images()} initial={index()} />)
}}
>
<Show when={!failed()} fallback={<text>No preview</text>}>
<image
source={image.uri}
fit="cover"
protocol="auto"
width="100%"
height="100%"
onError={() => setFailed(true)}
/>
</Show>
</box>
)
}}
</For>
<Show when={images().length > visible().length}>
<box width={8} height={height()} flexShrink={1} alignItems="center" justifyContent="center">
<text wrapMode="none" truncate>
+{images().length - visible().length} more
</text>
</box>
</Show>
</box>
</Show>
)
}
function inlineToolImages(part: SessionMessageAssistantTool) {
return toolDisplayContent(part.state).flatMap((content) =>
content.type === "file" && content.mime.startsWith("image/") && content.uri.startsWith("data:image/")
? [{ uri: content.uri }]
: [],
)
} }
type ToolProps = { type ToolProps = {

View file

@ -20,6 +20,7 @@ test("validates the session tabs setting", () => {
expect(decode({ tabs: { enabled: true, vertical: true } })).toEqual({ tabs: { enabled: true, vertical: true } }) expect(decode({ tabs: { enabled: true, vertical: true } })).toEqual({ tabs: { enabled: true, vertical: true } })
expect(() => decode({ tabs: { enabled: "on" } })).toThrow() expect(() => decode({ tabs: { enabled: "on" } })).toThrow()
expect(decode({ prompt: { image_preview: true } })).toEqual({ prompt: { image_preview: true } }) expect(decode({ prompt: { image_preview: true } })).toEqual({ prompt: { image_preview: true } })
expect(decode({ session: { image_preview: true } })).toEqual({ session: { image_preview: true } })
}) })
test("resolves nested config and keybind defaults", () => { test("resolves nested config and keybind defaults", () => {