fix(core): guard model-data migration on valid JSON
This commit is contained in:
parent
0104216d17
commit
54a2dd9235
10 changed files with 77 additions and 95 deletions
|
|
@ -2,7 +2,8 @@ ALTER TABLE `part` ADD `data_model` text;
|
||||||
--> statement-breakpoint
|
--> statement-breakpoint
|
||||||
UPDATE part
|
UPDATE part
|
||||||
SET data_model = json_remove(data, '$.state.metadata')
|
SET data_model = json_remove(data, '$.state.metadata')
|
||||||
WHERE length(CAST(data AS BLOB)) > 65536
|
WHERE json_valid(data)
|
||||||
|
AND length(CAST(data AS BLOB)) > 65536
|
||||||
AND json_extract(data, '$.type') = 'tool'
|
AND json_extract(data, '$.type') = 'tool'
|
||||||
AND json_extract(data, '$.state.status') = 'completed'
|
AND json_extract(data, '$.state.status') = 'completed'
|
||||||
AND length(CAST(json_extract(data, '$.state.metadata') AS BLOB)) > 65536;
|
AND length(CAST(json_extract(data, '$.state.metadata') AS BLOB)) > 65536;
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,8 @@ export default {
|
||||||
yield* tx.run(`
|
yield* tx.run(`
|
||||||
UPDATE part
|
UPDATE part
|
||||||
SET data_model = json_remove(data, '$.state.metadata')
|
SET data_model = json_remove(data, '$.state.metadata')
|
||||||
WHERE length(CAST(data AS BLOB)) > 65536
|
WHERE json_valid(data)
|
||||||
|
AND length(CAST(data AS BLOB)) > 65536
|
||||||
AND json_extract(data, '$.type') = 'tool'
|
AND json_extract(data, '$.type') = 'tool'
|
||||||
AND json_extract(data, '$.state.status') = 'completed'
|
AND json_extract(data, '$.state.status') = 'completed'
|
||||||
AND length(CAST(json_extract(data, '$.state.metadata') AS BLOB)) > 65536
|
AND length(CAST(json_extract(data, '$.state.metadata') AS BLOB)) > 65536
|
||||||
|
|
|
||||||
|
|
@ -6,10 +6,14 @@ type V1PartData<Data extends SessionV1.Part = SessionV1.Part> = Data extends Ses
|
||||||
? Omit<Data, "id" | "sessionID" | "messageID">
|
? Omit<Data, "id" | "sessionID" | "messageID">
|
||||||
: never
|
: never
|
||||||
|
|
||||||
|
export type ModelData = Omit<V1PartData<SessionV1.ToolPart>, "state"> & {
|
||||||
|
state: Omit<SessionV1.ToolStateCompleted, "metadata">
|
||||||
|
}
|
||||||
|
|
||||||
export const THRESHOLD = 64 * 1024
|
export const THRESHOLD = 64 * 1024
|
||||||
|
|
||||||
// Strip UI-only metadata only when the stored prompt projection benefits.
|
// Strip UI-only metadata only when the stored prompt projection benefits.
|
||||||
export function create(data: unknown): V1PartData | null {
|
export function create(data: unknown): ModelData | null {
|
||||||
if (!data || typeof data !== "object") return null
|
if (!data || typeof data !== "object") return null
|
||||||
if (!("type" in data) || data.type !== "tool") return null
|
if (!("type" in data) || data.type !== "tool") return null
|
||||||
if (!("state" in data) || !data.state || typeof data.state !== "object") return null
|
if (!("state" in data) || !data.state || typeof data.state !== "object") return null
|
||||||
|
|
@ -18,5 +22,5 @@ export function create(data: unknown): V1PartData | null {
|
||||||
const metadata = JSON.stringify(data.state.metadata)
|
const metadata = JSON.stringify(data.state.metadata)
|
||||||
if (!metadata || Buffer.byteLength(metadata) <= THRESHOLD) return null
|
if (!metadata || Buffer.byteLength(metadata) <= THRESHOLD) return null
|
||||||
const { metadata: _, ...state } = data.state
|
const { metadata: _, ...state } = data.state
|
||||||
return { ...data, state } as V1PartData
|
return { ...data, state } as ModelData
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@ import type { SessionSchema } from "./schema"
|
||||||
import type { MessageID, PartID, SessionV1 } from "../v1/session"
|
import type { MessageID, PartID, SessionV1 } from "../v1/session"
|
||||||
import { WorkspaceV2 } from "../workspace"
|
import { WorkspaceV2 } from "../workspace"
|
||||||
import { Timestamps } from "../database/schema.sql"
|
import { Timestamps } from "../database/schema.sql"
|
||||||
|
import type { ModelData } from "./model-data"
|
||||||
|
|
||||||
type SessionMessageData = Omit<(typeof SessionMessage.Message)["Encoded"], "type" | "id">
|
type SessionMessageData = Omit<(typeof SessionMessage.Message)["Encoded"], "type" | "id">
|
||||||
type V1MessageData = Omit<SessionV1.Info, "id" | "sessionID">
|
type V1MessageData = Omit<SessionV1.Info, "id" | "sessionID">
|
||||||
|
|
@ -86,7 +87,7 @@ export const PartTable = sqliteTable(
|
||||||
...Timestamps,
|
...Timestamps,
|
||||||
data: text({ mode: "json" }).notNull().$type<V1PartData>(),
|
data: text({ mode: "json" }).notNull().$type<V1PartData>(),
|
||||||
// Derived prompt projection; data remains canonical.
|
// Derived prompt projection; data remains canonical.
|
||||||
data_model: text({ mode: "json" }).$type<V1PartData>(),
|
data_model: text({ mode: "json" }).$type<ModelData>(),
|
||||||
},
|
},
|
||||||
(table) => [
|
(table) => [
|
||||||
index("part_message_id_id_idx").on(table.message_id, table.id),
|
index("part_message_id_id_idx").on(table.message_id, table.id),
|
||||||
|
|
|
||||||
|
|
@ -86,7 +86,8 @@ describe("DatabaseMigration", () => {
|
||||||
const large = JSON.stringify({ type: "tool", state: { status: "completed", metadata: { diff: "x".repeat(70_000) } } })
|
const large = JSON.stringify({ type: "tool", state: { status: "completed", metadata: { diff: "x".repeat(70_000) } } })
|
||||||
const unicode = JSON.stringify({ type: "tool", state: { status: "completed", metadata: { diff: "😀".repeat(20_000) } } })
|
const unicode = JSON.stringify({ type: "tool", state: { status: "completed", metadata: { diff: "😀".repeat(20_000) } } })
|
||||||
const small = JSON.stringify({ type: "tool", state: { status: "completed", metadata: { diff: "small" } } })
|
const small = JSON.stringify({ type: "tool", state: { status: "completed", metadata: { diff: "small" } } })
|
||||||
yield* db.run(sql`INSERT INTO part (id, data) VALUES (${"large"}, ${large}), (${"unicode"}, ${unicode}), (${"small"}, ${small})`)
|
const malformed = "{" + "x".repeat(70_000)
|
||||||
|
yield* db.run(sql`INSERT INTO part (id, data) VALUES (${"large"}, ${large}), (${"unicode"}, ${unicode}), (${"small"}, ${small}), (${"malformed"}, ${malformed})`)
|
||||||
|
|
||||||
yield* DatabaseMigration.applyOnly(db, [partModelDataMigration])
|
yield* DatabaseMigration.applyOnly(db, [partModelDataMigration])
|
||||||
|
|
||||||
|
|
@ -98,6 +99,7 @@ describe("DatabaseMigration", () => {
|
||||||
expect(yield* db.get(sql`SELECT data_model FROM part WHERE id = ${"unicode"}`)).toEqual({
|
expect(yield* db.get(sql`SELECT data_model FROM part WHERE id = ${"unicode"}`)).toEqual({
|
||||||
data_model: JSON.stringify({ type: "tool", state: { status: "completed" } }),
|
data_model: JSON.stringify({ type: "tool", state: { status: "completed" } }),
|
||||||
})
|
})
|
||||||
|
expect(yield* db.get(sql`SELECT data_model FROM part WHERE id = ${"malformed"}`)).toEqual({ data_model: null })
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -319,20 +319,18 @@ export function Prompt(props: PromptProps) {
|
||||||
let promptPartTypeId = 0
|
let promptPartTypeId = 0
|
||||||
const event = useEvent()
|
const event = useEvent()
|
||||||
|
|
||||||
onCleanup(
|
event.on(TuiEvent.PromptAppend.type, (evt, { workspace }) => {
|
||||||
event.on(TuiEvent.PromptAppend.type, (evt, { workspace }) => {
|
if (workspace !== project.workspace.current()) return
|
||||||
if (workspace !== project.workspace.current()) return
|
if (!input || input.isDestroyed) return
|
||||||
|
input.insertText(evt.properties.text)
|
||||||
|
setTimeout(() => {
|
||||||
|
// setTimeout is a workaround and needs to be addressed properly
|
||||||
if (!input || input.isDestroyed) return
|
if (!input || input.isDestroyed) return
|
||||||
input.insertText(evt.properties.text)
|
input.getLayoutNode().markDirty()
|
||||||
setTimeout(() => {
|
input.gotoBufferEnd()
|
||||||
// setTimeout is a workaround and needs to be addressed properly
|
renderer.requestRender()
|
||||||
if (!input || input.isDestroyed) return
|
}, 0)
|
||||||
input.getLayoutNode().markDirty()
|
})
|
||||||
input.gotoBufferEnd()
|
|
||||||
renderer.requestRender()
|
|
||||||
}, 0)
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
|
|
||||||
createEffect(() => {
|
createEffect(() => {
|
||||||
if (!input || input.isDestroyed) return
|
if (!input || input.isDestroyed) return
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,6 @@ import {
|
||||||
For,
|
For,
|
||||||
Match,
|
Match,
|
||||||
on,
|
on,
|
||||||
onCleanup,
|
|
||||||
onMount,
|
onMount,
|
||||||
Show,
|
Show,
|
||||||
Switch,
|
Switch,
|
||||||
|
|
@ -292,23 +291,21 @@ export function Session() {
|
||||||
})
|
})
|
||||||
|
|
||||||
let lastSwitch: string | undefined = undefined
|
let lastSwitch: string | undefined = undefined
|
||||||
onCleanup(
|
event.on("message.part.updated", (evt) => {
|
||||||
event.on("message.part.updated", (evt) => {
|
const part = evt.properties.part
|
||||||
const part = evt.properties.part
|
if (part.type !== "tool") return
|
||||||
if (part.type !== "tool") return
|
if (part.sessionID !== route.sessionID) return
|
||||||
if (part.sessionID !== route.sessionID) return
|
if (part.state.status !== "completed") return
|
||||||
if (part.state.status !== "completed") return
|
if (part.id === lastSwitch) return
|
||||||
if (part.id === lastSwitch) return
|
|
||||||
|
|
||||||
if (part.tool === "plan_exit") {
|
if (part.tool === "plan_exit") {
|
||||||
local.agent.set("build")
|
local.agent.set("build")
|
||||||
lastSwitch = part.id
|
lastSwitch = part.id
|
||||||
} else if (part.tool === "plan_enter") {
|
} else if (part.tool === "plan_enter") {
|
||||||
local.agent.set("plan")
|
local.agent.set("plan")
|
||||||
lastSwitch = part.id
|
lastSwitch = part.id
|
||||||
}
|
}
|
||||||
}),
|
})
|
||||||
)
|
|
||||||
|
|
||||||
let seeded = false
|
let seeded = false
|
||||||
let scroll: ScrollBoxRenderable
|
let scroll: ScrollBoxRenderable
|
||||||
|
|
@ -324,27 +321,25 @@ export function Session() {
|
||||||
const dialog = useDialog()
|
const dialog = useDialog()
|
||||||
const renderer = useRenderer()
|
const renderer = useRenderer()
|
||||||
|
|
||||||
onCleanup(
|
event.on("session.status", (evt) => {
|
||||||
event.on("session.status", (evt) => {
|
if (evt.properties.sessionID !== route.sessionID) return
|
||||||
if (evt.properties.sessionID !== route.sessionID) return
|
if (evt.properties.status.type !== "retry") return
|
||||||
if (evt.properties.status.type !== "retry") return
|
if (!evt.properties.status.action) return
|
||||||
if (!evt.properties.status.action) return
|
if (dialog.stack.length > 0) return
|
||||||
if (dialog.stack.length > 0) return
|
|
||||||
|
|
||||||
const keys = goUpsellKeys(evt.properties.status.action)
|
const keys = goUpsellKeys(evt.properties.status.action)
|
||||||
if (!keys) return
|
if (!keys) return
|
||||||
|
|
||||||
const seen = kv.get(keys.lastSeenAt)
|
const seen = kv.get(keys.lastSeenAt)
|
||||||
if (typeof seen === "number" && Date.now() - seen < GO_UPSELL_WINDOW) return
|
if (typeof seen === "number" && Date.now() - seen < GO_UPSELL_WINDOW) return
|
||||||
|
|
||||||
if (kv.get(keys.dontShow)) return
|
if (kv.get(keys.dontShow)) return
|
||||||
|
|
||||||
void DialogRetryAction.show(dialog, evt.properties.status.action).then((dontShowAgain) => {
|
void DialogRetryAction.show(dialog, evt.properties.status.action).then((dontShowAgain) => {
|
||||||
if (dontShowAgain) kv.set(keys.dontShow, true)
|
if (dontShowAgain) kv.set(keys.dontShow, true)
|
||||||
kv.set(keys.lastSeenAt, Date.now())
|
kv.set(keys.lastSeenAt, Date.now())
|
||||||
})
|
})
|
||||||
}),
|
})
|
||||||
)
|
|
||||||
|
|
||||||
const exit = useExit()
|
const exit = useExit()
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -170,16 +170,14 @@ export const layer = Layer.effect(
|
||||||
def: D,
|
def: D,
|
||||||
fn: (data: EventV2.Data<D>) => Effect.Effect<void, unknown>,
|
fn: (data: EventV2.Data<D>) => Effect.Effect<void, unknown>,
|
||||||
) =>
|
) =>
|
||||||
events
|
events.listen((event) => {
|
||||||
.listen((event) => {
|
if (event.type !== def.type || event.location?.directory !== _ctx.directory) return Effect.void
|
||||||
if (event.type !== def.type || event.location?.directory !== _ctx.directory) return Effect.void
|
return fn(event.data as EventV2.Data<D>).pipe(
|
||||||
return fn(event.data as EventV2.Data<D>).pipe(
|
Effect.catchCause((cause) =>
|
||||||
Effect.catchCause((cause) =>
|
Effect.sync(() => log.error("share subscriber failed", { type: def.type, cause })),
|
||||||
Effect.sync(() => log.error("share subscriber failed", { type: def.type, cause })),
|
),
|
||||||
),
|
)
|
||||||
)
|
})
|
||||||
})
|
|
||||||
.pipe(Effect.tap((unsubscribe) => Effect.addFinalizer(() => unsubscribe)))
|
|
||||||
|
|
||||||
yield* watch(Session.Event.Updated, (data) =>
|
yield* watch(Session.Event.Updated, (data) =>
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
|
|
|
||||||
|
|
@ -762,6 +762,17 @@ describe("session.compaction.prune", () => {
|
||||||
expect(Object.getOwnPropertyDescriptor(small.state, "metadata")?.get).toBeUndefined()
|
expect(Object.getOwnPropertyDescriptor(small.state, "metadata")?.get).toBeUndefined()
|
||||||
expect(small.state.metadata).toEqual({ description: "small" })
|
expect(small.state.metadata).toEqual({ description: "small" })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
part.state.metadata = { output: "x".repeat(200_000), description: "large again" }
|
||||||
|
yield* ssn.updatePart(part)
|
||||||
|
const largeAgain = (yield* MessageV2.filterCompactedEffect(info.id))
|
||||||
|
.flatMap((msg) => msg.parts)
|
||||||
|
.find((item) => item.id === part.id)
|
||||||
|
expect(largeAgain?.type).toBe("tool")
|
||||||
|
if (largeAgain?.type === "tool" && largeAgain.state.status === "completed") {
|
||||||
|
expect(Object.getOwnPropertyDescriptor(largeAgain.state, "metadata")?.get).toBeFunction()
|
||||||
|
expect(largeAgain.state.metadata.output).toHaveLength(200_000)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,6 @@ import { eq } from "drizzle-orm"
|
||||||
import { provideTmpdirInstance } from "../fixture/fixture"
|
import { provideTmpdirInstance } from "../fixture/fixture"
|
||||||
import { resetDatabase } from "../fixture/db"
|
import { resetDatabase } from "../fixture/db"
|
||||||
import { testEffect } from "../lib/effect"
|
import { testEffect } from "../lib/effect"
|
||||||
import { disposeInstance } from "@/effect/instance-registry"
|
|
||||||
|
|
||||||
const env = Layer.mergeAll(
|
const env = Layer.mergeAll(
|
||||||
Session.defaultLayer,
|
Session.defaultLayer,
|
||||||
|
|
@ -41,10 +40,10 @@ const json = (req: Parameters<typeof HttpClientResponse.fromWeb>[0], body: unkno
|
||||||
|
|
||||||
const none = HttpClient.make(() => Effect.die("unexpected http call"))
|
const none = HttpClient.make(() => Effect.die("unexpected http call"))
|
||||||
|
|
||||||
function live(client: HttpClient.HttpClient, events = EventV2Bridge.defaultLayer) {
|
function live(client: HttpClient.HttpClient) {
|
||||||
const http = Layer.succeed(HttpClient.HttpClient, client)
|
const http = Layer.succeed(HttpClient.HttpClient, client)
|
||||||
return ShareNext.layer.pipe(
|
return ShareNext.layer.pipe(
|
||||||
Layer.provide(events),
|
Layer.provide(EventV2Bridge.defaultLayer),
|
||||||
Layer.provide(Account.layer.pipe(Layer.provide(AccountRepo.defaultLayer), Layer.provide(http))),
|
Layer.provide(Account.layer.pipe(Layer.provide(AccountRepo.defaultLayer), Layer.provide(http))),
|
||||||
Layer.provide(Config.defaultLayer),
|
Layer.provide(Config.defaultLayer),
|
||||||
Layer.provide(Database.defaultLayer),
|
Layer.provide(Database.defaultLayer),
|
||||||
|
|
@ -102,34 +101,6 @@ beforeEach(async () => {
|
||||||
})
|
})
|
||||||
|
|
||||||
describe("ShareNext", () => {
|
describe("ShareNext", () => {
|
||||||
it.live("unsubscribes event listeners when the instance is disposed", () =>
|
|
||||||
provideTmpdirInstance((directory) => {
|
|
||||||
let active = 0
|
|
||||||
const events = Layer.mock(EventV2Bridge.Service, {
|
|
||||||
listen: () =>
|
|
||||||
Effect.sync(() => {
|
|
||||||
active++
|
|
||||||
return Effect.sync(() => {
|
|
||||||
active--
|
|
||||||
})
|
|
||||||
}),
|
|
||||||
})
|
|
||||||
|
|
||||||
return Effect.gen(function* () {
|
|
||||||
const share = yield* ShareNext.Service
|
|
||||||
let peak = 0
|
|
||||||
for (let index = 0; index < 20; index++) {
|
|
||||||
yield* share.init()
|
|
||||||
peak = Math.max(peak, active)
|
|
||||||
yield* Effect.promise(() => disposeInstance(directory))
|
|
||||||
}
|
|
||||||
|
|
||||||
expect(peak).toBe(5)
|
|
||||||
expect(active).toBe(0)
|
|
||||||
}).pipe(Effect.provide(live(none, events)))
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
|
|
||||||
it.live("request uses legacy share API without active org account", () =>
|
it.live("request uses legacy share API without active org account", () =>
|
||||||
provideTmpdirInstance(
|
provideTmpdirInstance(
|
||||||
() =>
|
() =>
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue