feat(tui): add verbose turn token usage (#39281)
This commit is contained in:
parent
1be6d94267
commit
4d59b059ee
5 changed files with 70 additions and 6 deletions
|
|
@ -77,6 +77,7 @@ export function DevToolsBar() {
|
||||||
const runtime = createMemo(() => runtimeStatus(frontendSamples()))
|
const runtime = createMemo(() => runtimeStatus(frontendSamples()))
|
||||||
const timing = () => config.data.debug?.timing ?? false
|
const timing = () => config.data.debug?.timing ?? false
|
||||||
const turnTokens = () => config.data.debug?.turn_tokens ?? false
|
const turnTokens = () => config.data.debug?.turn_tokens ?? false
|
||||||
|
const verboseTurnTokens = () => turnTokens() === "verbose"
|
||||||
|
|
||||||
const offEscape = keymap.intercept(
|
const offEscape = keymap.intercept(
|
||||||
"key",
|
"key",
|
||||||
|
|
@ -380,6 +381,16 @@ export function DevToolsBar() {
|
||||||
>
|
>
|
||||||
{turnTokens() ? "[x]" : "[ ]"} Turn token usage
|
{turnTokens() ? "[x]" : "[ ]"} Turn token usage
|
||||||
</Action>
|
</Action>
|
||||||
|
<Action
|
||||||
|
onClick={() =>
|
||||||
|
void config.update((draft) => {
|
||||||
|
draft.debug = { ...draft.debug, turn_tokens: verboseTurnTokens() ? true : "verbose" }
|
||||||
|
})
|
||||||
|
}
|
||||||
|
hoverBackground
|
||||||
|
>
|
||||||
|
{verboseTurnTokens() ? "[x]" : "[ ]"} Turn token usage (verbose)
|
||||||
|
</Action>
|
||||||
</box>
|
</box>
|
||||||
<For each={groups()}>
|
<For each={groups()}>
|
||||||
{(group) => (
|
{(group) => (
|
||||||
|
|
|
||||||
|
|
@ -254,8 +254,8 @@ export const settings: Setting[] = [
|
||||||
category: "Debug",
|
category: "Debug",
|
||||||
path: ["debug", "turn_tokens"],
|
path: ["debug", "turn_tokens"],
|
||||||
default: false,
|
default: false,
|
||||||
values: [false, true],
|
values: [false, true, "verbose"],
|
||||||
labels: ["off", "on"],
|
labels: ["off", "on", "verbose"],
|
||||||
keywords: ["tokens", "usage", "debug"],
|
keywords: ["tokens", "usage", "debug"],
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -159,7 +159,9 @@ export const Info = Schema.Struct({
|
||||||
Schema.Struct({
|
Schema.Struct({
|
||||||
devtools: Schema.optional(Schema.Boolean).annotate({ description: "Show the DevTools debug bar" }),
|
devtools: Schema.optional(Schema.Boolean).annotate({ description: "Show the DevTools debug bar" }),
|
||||||
timing: Schema.optional(Schema.Boolean).annotate({ description: "Show time-to-first-draw diagnostics" }),
|
timing: Schema.optional(Schema.Boolean).annotate({ description: "Show time-to-first-draw diagnostics" }),
|
||||||
turn_tokens: Schema.optional(Schema.Boolean).annotate({ description: "Show per-turn token usage diagnostics" }),
|
turn_tokens: Schema.optional(Schema.Union([Schema.Boolean, Schema.Literal("verbose")])).annotate({
|
||||||
|
description: "Show per-turn token usage diagnostics, optionally with tool call inputs",
|
||||||
|
}),
|
||||||
}),
|
}),
|
||||||
).annotate({ description: "Debugging settings" }),
|
).annotate({ description: "Debugging settings" }),
|
||||||
animations: Schema.optional(Schema.Boolean).annotate({ description: "Enable interface animations" }),
|
animations: Schema.optional(Schema.Boolean).annotate({ description: "Enable interface animations" }),
|
||||||
|
|
|
||||||
|
|
@ -1104,6 +1104,7 @@ function TurnTokenUsage(props: {
|
||||||
}) {
|
}) {
|
||||||
const config = useConfig()
|
const config = useConfig()
|
||||||
const { themeV2 } = useTheme()
|
const { themeV2 } = useTheme()
|
||||||
|
const verbose = () => config.data.debug?.turn_tokens === "verbose"
|
||||||
const steps = createMemo(() => {
|
const steps = createMemo(() => {
|
||||||
let previousCache = props.previousCache
|
let previousCache = props.previousCache
|
||||||
return props.messageIDs.flatMap((messageID) => {
|
return props.messageIDs.flatMap((messageID) => {
|
||||||
|
|
@ -1123,6 +1124,7 @@ function TurnTokenUsage(props: {
|
||||||
return [
|
return [
|
||||||
{
|
{
|
||||||
finish: message.finish === "tool-calls" ? "tool-call" : (message.finish ?? "unknown"),
|
finish: message.finish === "tool-calls" ? "tool-call" : (message.finish ?? "unknown"),
|
||||||
|
tools: verbose() ? message.content.filter((part) => part.type === "tool") : [],
|
||||||
newTokens,
|
newTokens,
|
||||||
cached: message.tokens.cache.read,
|
cached: message.tokens.cache.read,
|
||||||
total,
|
total,
|
||||||
|
|
@ -1138,7 +1140,7 @@ function TurnTokenUsage(props: {
|
||||||
total: Math.max("Total".length, ...steps().map((item) => item.total.toLocaleString().length)),
|
total: Math.max("Total".length, ...steps().map((item) => item.total.toLocaleString().length)),
|
||||||
}))
|
}))
|
||||||
return (
|
return (
|
||||||
<Show when={config.data.debug?.turn_tokens === true && steps().length > 0}>
|
<Show when={Boolean(config.data.debug?.turn_tokens) && steps().length > 0}>
|
||||||
<box paddingLeft={3} flexDirection="column">
|
<box paddingLeft={3} flexDirection="column">
|
||||||
<box flexDirection="row">
|
<box flexDirection="row">
|
||||||
<text width={INLINE_TOOL_ICON_WIDTH} fg={themeV2.text.subdued}>
|
<text width={INLINE_TOOL_ICON_WIDTH} fg={themeV2.text.subdued}>
|
||||||
|
|
@ -1161,7 +1163,7 @@ function TurnTokenUsage(props: {
|
||||||
<For each={steps()}>
|
<For each={steps()}>
|
||||||
{(item) => (
|
{(item) => (
|
||||||
<box paddingLeft={INLINE_TOOL_ICON_WIDTH} flexDirection="column">
|
<box paddingLeft={INLINE_TOOL_ICON_WIDTH} flexDirection="column">
|
||||||
<text fg={themeV2.text.subdued}>
|
<text fg={verbose() && item.finish === "tool-call" ? undefined : themeV2.text.subdued}>
|
||||||
{item.finish.padEnd(columns().step + 2)}
|
{item.finish.padEnd(columns().step + 2)}
|
||||||
<span style={{ attributes: TextAttributes.BOLD }}>
|
<span style={{ attributes: TextAttributes.BOLD }}>
|
||||||
{item.newTokens.toLocaleString().padStart(columns().newTokens)}
|
{item.newTokens.toLocaleString().padStart(columns().newTokens)}
|
||||||
|
|
@ -1171,6 +1173,7 @@ function TurnTokenUsage(props: {
|
||||||
{" "}
|
{" "}
|
||||||
{item.total.toLocaleString().padStart(columns().total)}
|
{item.total.toLocaleString().padStart(columns().total)}
|
||||||
</text>
|
</text>
|
||||||
|
<TurnTokenToolCalls tools={item.tools} />
|
||||||
<Show when={item.reuseDrop !== undefined}>
|
<Show when={item.reuseDrop !== undefined}>
|
||||||
<text fg={themeV2.text.feedback.warning.default}>
|
<text fg={themeV2.text.feedback.warning.default}>
|
||||||
! Likely cache bust: {item.reuseDrop?.toLocaleString()} fewer cached tokens than the previous step
|
! Likely cache bust: {item.reuseDrop?.toLocaleString()} fewer cached tokens than the previous step
|
||||||
|
|
@ -1184,6 +1187,54 @@ function TurnTokenUsage(props: {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function TurnTokenToolCalls(props: { tools: SessionMessageAssistantTool[] }) {
|
||||||
|
const { themeV2 } = useTheme()
|
||||||
|
const nameWidth = () => Math.max(0, ...props.tools.map((tool) => tool.name.length)) + 2
|
||||||
|
return (
|
||||||
|
<Show when={props.tools.length > 0}>
|
||||||
|
<box paddingLeft={2} flexDirection="column">
|
||||||
|
<For each={props.tools}>
|
||||||
|
{(tool) => (
|
||||||
|
<box flexDirection="row">
|
||||||
|
<text
|
||||||
|
width={nameWidth()}
|
||||||
|
flexShrink={0}
|
||||||
|
fg={themeV2.text.subdued}
|
||||||
|
attributes={TextAttributes.BOLD}
|
||||||
|
>
|
||||||
|
{tool.name}
|
||||||
|
</text>
|
||||||
|
<text
|
||||||
|
fg={themeV2.text.subdued}
|
||||||
|
attributes={TextAttributes.DIM}
|
||||||
|
wrapMode="word"
|
||||||
|
flexGrow={1}
|
||||||
|
minWidth={0}
|
||||||
|
>
|
||||||
|
{turnTokenToolSummary(tool)}
|
||||||
|
</text>
|
||||||
|
</box>
|
||||||
|
)}
|
||||||
|
</For>
|
||||||
|
</box>
|
||||||
|
</Show>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function turnTokenToolSummary(tool: SessionMessageAssistantTool) {
|
||||||
|
const data = tool.state.input
|
||||||
|
if (typeof data === "string") return data
|
||||||
|
const primaryKey = ["command", "id", "pattern", "url", "query", "path", "description", "code"].find(
|
||||||
|
(key) => key in data,
|
||||||
|
)
|
||||||
|
const input = Object.entries(data).filter(([, value]) =>
|
||||||
|
["string", "number", "boolean"].includes(typeof value),
|
||||||
|
)
|
||||||
|
const primary = input.find(([key]) => key === primaryKey)?.[1]
|
||||||
|
const details = input.filter(([key]) => key !== primaryKey).map(([key, value]) => `${key}: ${String(value)}`)
|
||||||
|
return [primary === undefined ? "" : String(primary), ...details].filter(Boolean).join(" ")
|
||||||
|
}
|
||||||
|
|
||||||
function BackgroundToolHint(props: { messages: SessionMessageInfo[] }) {
|
function BackgroundToolHint(props: { messages: SessionMessageInfo[] }) {
|
||||||
const { themeV2 } = useTheme()
|
const { themeV2 } = useTheme()
|
||||||
const shortcut = Keymap.useShortcut("session.background")
|
const shortcut = Keymap.useShortcut("session.background")
|
||||||
|
|
|
||||||
|
|
@ -41,7 +41,7 @@ export function createSessionRows(sessionID: Accessor<string>) {
|
||||||
const config = useConfig()
|
const config = useConfig()
|
||||||
const [rows, setRows] = createStore<SessionRow[]>([])
|
const [rows, setRows] = createStore<SessionRow[]>([])
|
||||||
const revertBoundary = () => data.session.get(sessionID())?.revert?.messageID
|
const revertBoundary = () => data.session.get(sessionID())?.revert?.messageID
|
||||||
const turnTokens = () => config.data.debug?.turn_tokens === true
|
const turnTokens = () => Boolean(config.data.debug?.turn_tokens)
|
||||||
|
|
||||||
function reduce() {
|
function reduce() {
|
||||||
const messages = data.session.message.list(sessionID())
|
const messages = data.session.message.list(sessionID())
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue