fix(core): guard model-data migration on valid JSON

This commit is contained in:
starptech 2026-06-03 16:50:43 +02:00
commit 54a2dd9235
10 changed files with 77 additions and 95 deletions

View file

@ -2,7 +2,8 @@ ALTER TABLE `part` ADD `data_model` text;
--> statement-breakpoint --> statement-breakpoint
UPDATE part UPDATE part
SET data_model = json_remove(data, '$.state.metadata') SET data_model = json_remove(data, '$.state.metadata')
WHERE length(CAST(data AS BLOB)) > 65536 WHERE json_valid(data)
AND length(CAST(data AS BLOB)) > 65536
AND json_extract(data, '$.type') = 'tool' AND json_extract(data, '$.type') = 'tool'
AND json_extract(data, '$.state.status') = 'completed' AND json_extract(data, '$.state.status') = 'completed'
AND length(CAST(json_extract(data, '$.state.metadata') AS BLOB)) > 65536; AND length(CAST(json_extract(data, '$.state.metadata') AS BLOB)) > 65536;

View file

@ -11,7 +11,8 @@ export default {
yield* tx.run(` yield* tx.run(`
UPDATE part UPDATE part
SET data_model = json_remove(data, '$.state.metadata') SET data_model = json_remove(data, '$.state.metadata')
WHERE length(CAST(data AS BLOB)) > 65536 WHERE json_valid(data)
AND length(CAST(data AS BLOB)) > 65536
AND json_extract(data, '$.type') = 'tool' AND json_extract(data, '$.type') = 'tool'
AND json_extract(data, '$.state.status') = 'completed' AND json_extract(data, '$.state.status') = 'completed'
AND length(CAST(json_extract(data, '$.state.metadata') AS BLOB)) > 65536 AND length(CAST(json_extract(data, '$.state.metadata') AS BLOB)) > 65536

View file

@ -6,10 +6,14 @@ type V1PartData<Data extends SessionV1.Part = SessionV1.Part> = Data extends Ses
? Omit<Data, "id" | "sessionID" | "messageID"> ? Omit<Data, "id" | "sessionID" | "messageID">
: never : never
export type ModelData = Omit<V1PartData<SessionV1.ToolPart>, "state"> & {
state: Omit<SessionV1.ToolStateCompleted, "metadata">
}
export const THRESHOLD = 64 * 1024 export const THRESHOLD = 64 * 1024
// Strip UI-only metadata only when the stored prompt projection benefits. // Strip UI-only metadata only when the stored prompt projection benefits.
export function create(data: unknown): V1PartData | null { export function create(data: unknown): ModelData | null {
if (!data || typeof data !== "object") return null if (!data || typeof data !== "object") return null
if (!("type" in data) || data.type !== "tool") return null if (!("type" in data) || data.type !== "tool") return null
if (!("state" in data) || !data.state || typeof data.state !== "object") return null if (!("state" in data) || !data.state || typeof data.state !== "object") return null
@ -18,5 +22,5 @@ export function create(data: unknown): V1PartData | null {
const metadata = JSON.stringify(data.state.metadata) const metadata = JSON.stringify(data.state.metadata)
if (!metadata || Buffer.byteLength(metadata) <= THRESHOLD) return null if (!metadata || Buffer.byteLength(metadata) <= THRESHOLD) return null
const { metadata: _, ...state } = data.state const { metadata: _, ...state } = data.state
return { ...data, state } as V1PartData return { ...data, state } as ModelData
} }

View file

@ -9,6 +9,7 @@ import type { SessionSchema } from "./schema"
import type { MessageID, PartID, SessionV1 } from "../v1/session" import type { MessageID, PartID, SessionV1 } from "../v1/session"
import { WorkspaceV2 } from "../workspace" import { WorkspaceV2 } from "../workspace"
import { Timestamps } from "../database/schema.sql" import { Timestamps } from "../database/schema.sql"
import type { ModelData } from "./model-data"
type SessionMessageData = Omit<(typeof SessionMessage.Message)["Encoded"], "type" | "id"> type SessionMessageData = Omit<(typeof SessionMessage.Message)["Encoded"], "type" | "id">
type V1MessageData = Omit<SessionV1.Info, "id" | "sessionID"> type V1MessageData = Omit<SessionV1.Info, "id" | "sessionID">
@ -86,7 +87,7 @@ export const PartTable = sqliteTable(
...Timestamps, ...Timestamps,
data: text({ mode: "json" }).notNull().$type<V1PartData>(), data: text({ mode: "json" }).notNull().$type<V1PartData>(),
// Derived prompt projection; data remains canonical. // Derived prompt projection; data remains canonical.
data_model: text({ mode: "json" }).$type<V1PartData>(), data_model: text({ mode: "json" }).$type<ModelData>(),
}, },
(table) => [ (table) => [
index("part_message_id_id_idx").on(table.message_id, table.id), index("part_message_id_id_idx").on(table.message_id, table.id),

View file

@ -86,7 +86,8 @@ describe("DatabaseMigration", () => {
const large = JSON.stringify({ type: "tool", state: { status: "completed", metadata: { diff: "x".repeat(70_000) } } }) const large = JSON.stringify({ type: "tool", state: { status: "completed", metadata: { diff: "x".repeat(70_000) } } })
const unicode = JSON.stringify({ type: "tool", state: { status: "completed", metadata: { diff: "😀".repeat(20_000) } } }) const unicode = JSON.stringify({ type: "tool", state: { status: "completed", metadata: { diff: "😀".repeat(20_000) } } })
const small = JSON.stringify({ type: "tool", state: { status: "completed", metadata: { diff: "small" } } }) const small = JSON.stringify({ type: "tool", state: { status: "completed", metadata: { diff: "small" } } })
yield* db.run(sql`INSERT INTO part (id, data) VALUES (${"large"}, ${large}), (${"unicode"}, ${unicode}), (${"small"}, ${small})`) const malformed = "{" + "x".repeat(70_000)
yield* db.run(sql`INSERT INTO part (id, data) VALUES (${"large"}, ${large}), (${"unicode"}, ${unicode}), (${"small"}, ${small}), (${"malformed"}, ${malformed})`)
yield* DatabaseMigration.applyOnly(db, [partModelDataMigration]) yield* DatabaseMigration.applyOnly(db, [partModelDataMigration])
@ -98,6 +99,7 @@ describe("DatabaseMigration", () => {
expect(yield* db.get(sql`SELECT data_model FROM part WHERE id = ${"unicode"}`)).toEqual({ expect(yield* db.get(sql`SELECT data_model FROM part WHERE id = ${"unicode"}`)).toEqual({
data_model: JSON.stringify({ type: "tool", state: { status: "completed" } }), data_model: JSON.stringify({ type: "tool", state: { status: "completed" } }),
}) })
expect(yield* db.get(sql`SELECT data_model FROM part WHERE id = ${"malformed"}`)).toEqual({ data_model: null })
}), }),
) )
}) })

View file

@ -319,20 +319,18 @@ export function Prompt(props: PromptProps) {
let promptPartTypeId = 0 let promptPartTypeId = 0
const event = useEvent() const event = useEvent()
onCleanup( event.on(TuiEvent.PromptAppend.type, (evt, { workspace }) => {
event.on(TuiEvent.PromptAppend.type, (evt, { workspace }) => { if (workspace !== project.workspace.current()) return
if (workspace !== project.workspace.current()) return if (!input || input.isDestroyed) return
input.insertText(evt.properties.text)
setTimeout(() => {
// setTimeout is a workaround and needs to be addressed properly
if (!input || input.isDestroyed) return if (!input || input.isDestroyed) return
input.insertText(evt.properties.text) input.getLayoutNode().markDirty()
setTimeout(() => { input.gotoBufferEnd()
// setTimeout is a workaround and needs to be addressed properly renderer.requestRender()
if (!input || input.isDestroyed) return }, 0)
input.getLayoutNode().markDirty() })
input.gotoBufferEnd()
renderer.requestRender()
}, 0)
}),
)
createEffect(() => { createEffect(() => {
if (!input || input.isDestroyed) return if (!input || input.isDestroyed) return

View file

@ -7,7 +7,6 @@ import {
For, For,
Match, Match,
on, on,
onCleanup,
onMount, onMount,
Show, Show,
Switch, Switch,
@ -292,23 +291,21 @@ export function Session() {
}) })
let lastSwitch: string | undefined = undefined let lastSwitch: string | undefined = undefined
onCleanup( event.on("message.part.updated", (evt) => {
event.on("message.part.updated", (evt) => { const part = evt.properties.part
const part = evt.properties.part if (part.type !== "tool") return
if (part.type !== "tool") return if (part.sessionID !== route.sessionID) return
if (part.sessionID !== route.sessionID) return if (part.state.status !== "completed") return
if (part.state.status !== "completed") return if (part.id === lastSwitch) return
if (part.id === lastSwitch) return
if (part.tool === "plan_exit") { if (part.tool === "plan_exit") {
local.agent.set("build") local.agent.set("build")
lastSwitch = part.id lastSwitch = part.id
} else if (part.tool === "plan_enter") { } else if (part.tool === "plan_enter") {
local.agent.set("plan") local.agent.set("plan")
lastSwitch = part.id lastSwitch = part.id
} }
}), })
)
let seeded = false let seeded = false
let scroll: ScrollBoxRenderable let scroll: ScrollBoxRenderable
@ -324,27 +321,25 @@ export function Session() {
const dialog = useDialog() const dialog = useDialog()
const renderer = useRenderer() const renderer = useRenderer()
onCleanup( event.on("session.status", (evt) => {
event.on("session.status", (evt) => { if (evt.properties.sessionID !== route.sessionID) return
if (evt.properties.sessionID !== route.sessionID) return if (evt.properties.status.type !== "retry") return
if (evt.properties.status.type !== "retry") return if (!evt.properties.status.action) return
if (!evt.properties.status.action) return if (dialog.stack.length > 0) return
if (dialog.stack.length > 0) return
const keys = goUpsellKeys(evt.properties.status.action) const keys = goUpsellKeys(evt.properties.status.action)
if (!keys) return if (!keys) return
const seen = kv.get(keys.lastSeenAt) const seen = kv.get(keys.lastSeenAt)
if (typeof seen === "number" && Date.now() - seen < GO_UPSELL_WINDOW) return if (typeof seen === "number" && Date.now() - seen < GO_UPSELL_WINDOW) return
if (kv.get(keys.dontShow)) return if (kv.get(keys.dontShow)) return
void DialogRetryAction.show(dialog, evt.properties.status.action).then((dontShowAgain) => { void DialogRetryAction.show(dialog, evt.properties.status.action).then((dontShowAgain) => {
if (dontShowAgain) kv.set(keys.dontShow, true) if (dontShowAgain) kv.set(keys.dontShow, true)
kv.set(keys.lastSeenAt, Date.now()) kv.set(keys.lastSeenAt, Date.now())
}) })
}), })
)
const exit = useExit() const exit = useExit()

View file

@ -170,16 +170,14 @@ export const layer = Layer.effect(
def: D, def: D,
fn: (data: EventV2.Data<D>) => Effect.Effect<void, unknown>, fn: (data: EventV2.Data<D>) => Effect.Effect<void, unknown>,
) => ) =>
events events.listen((event) => {
.listen((event) => { if (event.type !== def.type || event.location?.directory !== _ctx.directory) return Effect.void
if (event.type !== def.type || event.location?.directory !== _ctx.directory) return Effect.void return fn(event.data as EventV2.Data<D>).pipe(
return fn(event.data as EventV2.Data<D>).pipe( Effect.catchCause((cause) =>
Effect.catchCause((cause) => Effect.sync(() => log.error("share subscriber failed", { type: def.type, cause })),
Effect.sync(() => log.error("share subscriber failed", { type: def.type, cause })), ),
), )
) })
})
.pipe(Effect.tap((unsubscribe) => Effect.addFinalizer(() => unsubscribe)))
yield* watch(Session.Event.Updated, (data) => yield* watch(Session.Event.Updated, (data) =>
Effect.gen(function* () { Effect.gen(function* () {

View file

@ -762,6 +762,17 @@ describe("session.compaction.prune", () => {
expect(Object.getOwnPropertyDescriptor(small.state, "metadata")?.get).toBeUndefined() expect(Object.getOwnPropertyDescriptor(small.state, "metadata")?.get).toBeUndefined()
expect(small.state.metadata).toEqual({ description: "small" }) expect(small.state.metadata).toEqual({ description: "small" })
} }
part.state.metadata = { output: "x".repeat(200_000), description: "large again" }
yield* ssn.updatePart(part)
const largeAgain = (yield* MessageV2.filterCompactedEffect(info.id))
.flatMap((msg) => msg.parts)
.find((item) => item.id === part.id)
expect(largeAgain?.type).toBe("tool")
if (largeAgain?.type === "tool" && largeAgain.state.status === "completed") {
expect(Object.getOwnPropertyDescriptor(largeAgain.state, "metadata")?.get).toBeFunction()
expect(largeAgain.state.metadata.output).toHaveLength(200_000)
}
} }
} }
}), }),

View file

@ -19,7 +19,6 @@ import { eq } from "drizzle-orm"
import { provideTmpdirInstance } from "../fixture/fixture" import { provideTmpdirInstance } from "../fixture/fixture"
import { resetDatabase } from "../fixture/db" import { resetDatabase } from "../fixture/db"
import { testEffect } from "../lib/effect" import { testEffect } from "../lib/effect"
import { disposeInstance } from "@/effect/instance-registry"
const env = Layer.mergeAll( const env = Layer.mergeAll(
Session.defaultLayer, Session.defaultLayer,
@ -41,10 +40,10 @@ const json = (req: Parameters<typeof HttpClientResponse.fromWeb>[0], body: unkno
const none = HttpClient.make(() => Effect.die("unexpected http call")) const none = HttpClient.make(() => Effect.die("unexpected http call"))
function live(client: HttpClient.HttpClient, events = EventV2Bridge.defaultLayer) { function live(client: HttpClient.HttpClient) {
const http = Layer.succeed(HttpClient.HttpClient, client) const http = Layer.succeed(HttpClient.HttpClient, client)
return ShareNext.layer.pipe( return ShareNext.layer.pipe(
Layer.provide(events), Layer.provide(EventV2Bridge.defaultLayer),
Layer.provide(Account.layer.pipe(Layer.provide(AccountRepo.defaultLayer), Layer.provide(http))), Layer.provide(Account.layer.pipe(Layer.provide(AccountRepo.defaultLayer), Layer.provide(http))),
Layer.provide(Config.defaultLayer), Layer.provide(Config.defaultLayer),
Layer.provide(Database.defaultLayer), Layer.provide(Database.defaultLayer),
@ -102,34 +101,6 @@ beforeEach(async () => {
}) })
describe("ShareNext", () => { describe("ShareNext", () => {
it.live("unsubscribes event listeners when the instance is disposed", () =>
provideTmpdirInstance((directory) => {
let active = 0
const events = Layer.mock(EventV2Bridge.Service, {
listen: () =>
Effect.sync(() => {
active++
return Effect.sync(() => {
active--
})
}),
})
return Effect.gen(function* () {
const share = yield* ShareNext.Service
let peak = 0
for (let index = 0; index < 20; index++) {
yield* share.init()
peak = Math.max(peak, active)
yield* Effect.promise(() => disposeInstance(directory))
}
expect(peak).toBe(5)
expect(active).toBe(0)
}).pipe(Effect.provide(live(none, events)))
}),
)
it.live("request uses legacy share API without active org account", () => it.live("request uses legacy share API without active org account", () =>
provideTmpdirInstance( provideTmpdirInstance(
() => () =>