ceos/opencode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(core): add durable compaction barrier (#35371)

Browse source
This commit is contained in:
Kit Langton 2026-07-06 21:26:05 -04:00 • committed by GitHub
commit 04b673432c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
39 changed files with 1486 additions and 396 deletions
Download patch file Download diff file Expand all files Collapse all files

1
packages/core/src/database/migration.gen.ts generated
View file

@ -47,5 +47,6 @@ export const migrations = (
import("./migration/20260703200000_reset_v2_session_events"),
import("./migration/20260705180000_rename_instructions"),
import("./migration/20260706223930_add-session-fork"),
import("./migration/20260707010146_durable_session_inbox"),
])
).map((module) => module.default) satisfies DatabaseMigration.Migration[]

43
packages/core/src/database/migration/20260707010146_durable_session_inbox.ts Normal file
View file

@ -0,0 +1,43 @@
import { Effect } from "effect"
import type { DatabaseMigration } from "../migration"
export default {
id: "20260707010146_durable_session_inbox",
up(tx) {
return Effect.gen(function* () {
yield* tx.run(`PRAGMA foreign_keys=OFF;`)
yield* tx.run(`
CREATE TABLE \`__new_session_input\` (
\`id\` text PRIMARY KEY,
\`session_id\` text NOT NULL,
\`type\` text NOT NULL,
\`prompt\` text,
\`delivery\` text,
\`admitted_seq\` integer NOT NULL,
\`promoted_seq\` integer,
\`time_created\` integer NOT NULL,
CONSTRAINT \`fk_session_input_session_id_session_id_fk\` FOREIGN KEY (\`session_id\`) REFERENCES \`session\`(\`id\`) ON DELETE CASCADE
);
`)
yield* tx.run(
`INSERT INTO \`__new_session_input\`(\`id\`, \`session_id\`, \`type\`, \`prompt\`, \`delivery\`, \`admitted_seq\`, \`promoted_seq\`, \`time_created\`) SELECT \`id\`, \`session_id\`, 'prompt', \`prompt\`, \`delivery\`, \`admitted_seq\`, \`promoted_seq\`, \`time_created\` FROM \`session_input\`;`,
)
yield* tx.run(`DROP TABLE \`session_input\`;`)
yield* tx.run(`ALTER TABLE \`__new_session_input\` RENAME TO \`session_input\`;`)
yield* tx.run(`PRAGMA foreign_keys=ON;`)
yield* tx.run(`DROP INDEX IF EXISTS \`session_input_session_pending_delivery_seq_idx\`;`)
yield* tx.run(
`CREATE INDEX \`session_input_session_pending_type_delivery_seq_idx\` ON \`session_input\` (\`session_id\`,\`promoted_seq\`,\`type\`,\`delivery\`,\`admitted_seq\`);`,
)
yield* tx.run(
`CREATE UNIQUE INDEX \`session_input_session_pending_compaction_idx\` ON \`session_input\` (\`session_id\`) WHERE "session_input"."type" = 'compaction' and "session_input"."promoted_seq" is null;`,
)
yield* tx.run(
`CREATE UNIQUE INDEX \`session_input_session_admitted_seq_idx\` ON \`session_input\` (\`session_id\`,\`admitted_seq\`);`,
)
yield* tx.run(
`CREATE UNIQUE INDEX \`session_input_session_promoted_seq_idx\` ON \`session_input\` (\`session_id\`,\`promoted_seq\`);`,
)
})
},
} satisfies DatabaseMigration.Migration

10
packages/core/src/database/schema.gen.ts
View file

@ -170,8 +170,9 @@ export default {
CREATE TABLE \`session_input\` (
\`id\` text PRIMARY KEY,
\`session_id\` text NOT NULL,
\`prompt\` text NOT NULL,
\`delivery\` text NOT NULL,
\`type\` text NOT NULL,
\`prompt\` text,
\`delivery\` text,
\`admitted_seq\` integer NOT NULL,
\`promoted_seq\` integer,
\`time_created\` integer NOT NULL,
@ -261,7 +262,10 @@ export default {
yield* tx.run(`CREATE INDEX \`part_message_id_id_idx\` ON \`part\` (\`message_id\`,\`id\`);`)
yield* tx.run(`CREATE INDEX \`part_session_idx\` ON \`part\` (\`session_id\`);`)
yield* tx.run(
`CREATE INDEX \`session_input_session_pending_delivery_seq_idx\` ON \`session_input\` (\`session_id\`,\`promoted_seq\`,\`delivery\`,\`admitted_seq\`);`,
`CREATE INDEX \`session_input_session_pending_type_delivery_seq_idx\` ON \`session_input\` (\`session_id\`,\`promoted_seq\`,\`type\`,\`delivery\`,\`admitted_seq\`);`,
)
yield* tx.run(
`CREATE UNIQUE INDEX \`session_input_session_pending_compaction_idx\` ON \`session_input\` (\`session_id\`) WHERE "session_input"."type" = 'compaction' and "session_input"."promoted_seq" is null;`,
)
yield* tx.run(
`CREATE UNIQUE INDEX \`session_input_session_admitted_seq_idx\` ON \`session_input\` (\`session_id\`,\`admitted_seq\`);`,

65
packages/core/src/session.ts
View file

@ -33,7 +33,6 @@ import { MessageDecodeError } from "./session/error"
import { SessionEvent } from "./session/event"
import { SessionInput } from "./session/input"
import { Snapshot } from "./snapshot"
import { SessionCompaction } from "./session/compaction"
import { SessionRevert } from "./session/revert"
import { Revert } from "@opencode-ai/schema/revert"
import { FSUtil } from "./fs-util"
@ -96,6 +95,7 @@ type CreateInput = CreateBaseInput &
({ location: Location.Ref; parentID?: never } | { parentID: SessionSchema.ID; location?: never })
type CompactInput = {
id?: SessionMessage.ID
sessionID: SessionSchema.ID
}
@ -125,6 +125,13 @@ export class AttachmentError extends Schema.TaggedErrorClass<AttachmentError>()(
uri: Schema.String,
message: Schema.String,
}) {}
export class CompactionConflictError extends Schema.TaggedErrorClass<CompactionConflictError>()(
"Session.CompactionConflictError",
{
sessionID: SessionSchema.ID,
inputID: SessionMessage.ID,
},
) {}
export class BusyError extends Schema.TaggedErrorClass<BusyError>()("Session.BusyError", {
sessionID: SessionSchema.ID,
}) {}
@ -140,6 +147,7 @@ export type Error =
| OperationUnavailableError
| PromptConflictError
| AttachmentError
| CompactionConflictError
| BusyError
| SkillNotFoundError
| CommandV2.NotFoundError
@ -224,7 +232,7 @@ export interface Interface {
}) => Effect.Effect<void, NotFoundError | SkillNotFoundError>
readonly compact: (
input: CompactInput,
) => Effect.Effect<void, NotFoundError | BusyError | MessageDecodeError | OperationUnavailableError>
) => Effect.Effect<SessionInput.Compaction, NotFoundError | CompactionConflictError>
readonly wait: (id: SessionSchema.ID) => Effect.Effect<void, NotFoundError>
readonly active: Effect.Effect<ReadonlySet<SessionSchema.ID>>
readonly background: (sessionID: SessionSchema.ID) => Effect.Effect<void, NotFoundError>
@ -626,19 +634,20 @@ const layer = Layer.effect(
})
}),
compact: Effect.fn("V2Session.compact")(function* (input) {
const session = yield* result.get(input.sessionID)
// TODO: admit manual compaction as durable pending work, like prompt input, instead of rejecting active sessions.
if ((yield* execution.active).has(input.sessionID)) return yield* new BusyError({ sessionID: input.sessionID })
const context = yield* store.context(input.sessionID)
const compacted = yield* Effect.gen(function* () {
const compaction = yield* SessionCompaction.Service
return yield* compaction.compactManual({ session, messages: context })
yield* result.get(input.sessionID)
const inputID = input.id ?? SessionMessage.ID.create()
const admitted = yield* SessionInput.admitCompaction(db, events, {
id: inputID,
sessionID: input.sessionID,
}).pipe(
Effect.provide(locations.get(session.location)),
Effect.catch(() => Effect.succeed(false)),
Effect.catchDefect((defect) =>
defect instanceof SessionInput.LifecycleConflict
? new CompactionConflictError({ sessionID: input.sessionID, inputID })
: Effect.die(defect),
),
)
if (!compacted) return yield* new OperationUnavailableError({ operation: "compact" })
return undefined
yield* execution.wake(input.sessionID)
return admitted
}),
wait: Effect.fn("V2Session.wait")(function* (sessionID) {
yield* result.get(sessionID)
@ -734,11 +743,7 @@ function synthesizeTerminalShellInfo(started: ShellSchema.Info): ShellSchema.Inf
const resolvePrompt = Effect.fn("V2Session.resolvePrompt")(function* (input: PromptInput.Prompt) {
const fs = yield* FSUtil.Service
const files = input.files
? yield* Effect.forEach(
input.files,
(file) => materializeAttachment(fs, file),
{ concurrency: 8 },
)
? yield* Effect.forEach(input.files, (file) => materializeAttachment(fs, file), { concurrency: 8 })
: undefined
return Prompt.make({ text: input.text, agents: input.agents, files })
})
@ -769,7 +774,11 @@ const materializeAttachment = Effect.fn("V2Session.materializeAttachment")(funct
const content =
mime === "text/plain" && resolved.start !== undefined
? Buffer.from(
Buffer.from(resolved.bytes).toString("utf8").split("\n").slice(resolved.start - 1, resolved.end).join("\n"),
Buffer.from(resolved.bytes)
.toString("utf8")
.split("\n")
.slice(resolved.start - 1, resolved.end)
.join("\n"),
)
: resolved.bytes
return FileAttachment.create({
@ -799,13 +808,13 @@ const readFileAttachment = Effect.fn("V2Session.readFileAttachment")(function* (
},
catch: () => new AttachmentError({ uri, message: `Invalid file URI: ${uri}` }),
})
const info = yield* fs.stat(target).pipe(
Effect.mapError(() => new AttachmentError({ uri, message: `Unable to read attachment: ${uri}` })),
)
const info = yield* fs
.stat(target)
.pipe(Effect.mapError(() => new AttachmentError({ uri, message: `Unable to read attachment: ${uri}` })))
if (info.type === "Directory") {
const entries = yield* fs.readDirectoryEntries(target).pipe(
Effect.mapError(() => new AttachmentError({ uri, message: `Unable to read attachment: ${uri}` })),
)
const entries = yield* fs
.readDirectoryEntries(target)
.pipe(Effect.mapError(() => new AttachmentError({ uri, message: `Unable to read attachment: ${uri}` })))
return {
bytes: Buffer.from(
entries
@ -827,9 +836,9 @@ const readFileAttachment = Effect.fn("V2Session.readFileAttachment")(function* (
uri,
message: `Attachment exceeds the ${MAX_ATTACHMENT_BYTES} byte limit: ${uri}`,
})
const bytes = yield* fs.readFile(target).pipe(
Effect.mapError(() => new AttachmentError({ uri, message: `Unable to read attachment: ${uri}` })),
)
const bytes = yield* fs
.readFile(target)
.pipe(Effect.mapError(() => new AttachmentError({ uri, message: `Unable to read attachment: ${uri}` })))
return { bytes, source: { type: "uri" as const, uri }, start, end, name: path.basename(target), mime: undefined }
})

4
packages/core/src/session/compaction.ts
View file

@ -260,7 +260,9 @@ const make = (dependencies: Dependencies) => {
if (context === undefined || context <= 0) return false
const selected = select(input.messages, config.tokens)
if (!selected) return false
const previousSummary = input.messages.find((message) => message.type === "compaction")
const previousSummary = input.messages.find(
(message) => message.type === "compaction" && message.status === "completed",
)
const hasHead = selected.head.length > 0
if (!hasHead && previousSummary?.type !== "compaction" && !input.force) return false
const forcedShortContext = input.force && !hasHead

10
packages/core/src/session/history.ts
View file

@ -1,4 +1,4 @@
import { and, asc, desc, eq, gt, gte, ne, or } from "drizzle-orm"
import { and, asc, desc, eq, gt, gte, ne, or, sql } from "drizzle-orm"
import { Effect, Schema } from "effect"
import { Database } from "../database/database"
import { MessageDecodeError } from "./error"
@ -14,7 +14,13 @@ export const latestCompaction = Effect.fnUntraced(function* (db: DatabaseService
return yield* db
.select({ seq: SessionMessageTable.seq })
.from(SessionMessageTable)
.where(and(eq(SessionMessageTable.session_id, sessionID), eq(SessionMessageTable.type, "compaction")))
.where(
and(
eq(SessionMessageTable.session_id, sessionID),
eq(SessionMessageTable.type, "compaction"),
sql`json_extract(${SessionMessageTable.data}, '$.status') = 'completed'`,
),
)
.orderBy(desc(SessionMessageTable.seq))
.limit(1)
.get()

258
packages/core/src/session/input.ts
View file

@ -2,9 +2,10 @@ export * as SessionInput from "./input"
import { and, asc, eq, isNull } from "drizzle-orm"
import { DateTime, Effect, Schema } from "effect"
import { Admitted, Delivery } from "@opencode-ai/schema/session-input"
import { Admitted, Compaction, Delivery, Entry, PromptEntry } from "@opencode-ai/schema/session-input"
import type { Database } from "../database/database"
import type { EventV2 } from "../event"
import { KeyedMutex } from "../effect/keyed-mutex"
import { SessionEvent } from "./event"
import { SessionMessage } from "./message"
import { Prompt } from "@opencode-ai/schema/prompt"
@ -13,30 +14,77 @@ import { SessionInputTable, SessionMessageTable } from "./sql"
type DatabaseService = Database.Interface["db"]
export { Admitted, Delivery }
export { Admitted, Compaction, Delivery, Entry, PromptEntry }
const decodePrompt = Schema.decodeUnknownSync(Prompt)
const encodePrompt = Schema.encodeSync(Prompt)
const inboxLocks = KeyedMutex.makeUnsafe<SessionSchema.ID>()
const fromRow = (row: typeof SessionInputTable.$inferSelect): Admitted =>
Admitted.make({
export class LifecycleConflict extends Schema.TaggedErrorClass<LifecycleConflict>()("SessionInput.LifecycleConflict", {
id: SessionMessage.ID,
}) {}
const fromRow = (row: typeof SessionInputTable.$inferSelect): Entry => {
const base = {
admittedSeq: row.admitted_seq,
id: SessionMessage.ID.make(row.id),
sessionID: SessionSchema.ID.make(row.session_id),
timeCreated: DateTime.makeUnsafe(row.time_created),
}
if (row.type === "compaction")
return Compaction.make({
...base,
type: "compaction",
...(row.promoted_seq === null ? {} : { handledSeq: row.promoted_seq }),
})
if (!row.prompt || !row.delivery) throw new LifecycleConflict({ id: base.id })
return PromptEntry.make({
...base,
type: "prompt",
prompt: decodePrompt(row.prompt),
delivery: row.delivery,
timeCreated: DateTime.makeUnsafe(row.time_created),
...(row.promoted_seq === null ? {} : { promotedSeq: row.promoted_seq }),
})
}
const toAdmitted = (entry: PromptEntry): Admitted =>
Admitted.make({
admittedSeq: entry.admittedSeq,
id: entry.id,
sessionID: entry.sessionID,
prompt: entry.prompt,
delivery: entry.delivery,
timeCreated: entry.timeCreated,
...(entry.promotedSeq === undefined ? {} : { promotedSeq: entry.promotedSeq }),
})
export const find = Effect.fn("SessionInput.find")(function* (db: DatabaseService, id: SessionMessage.ID) {
const row = yield* db.select().from(SessionInputTable).where(eq(SessionInputTable.id, id)).get().pipe(Effect.orDie)
return row === undefined ? undefined : fromRow(row)
})
export class LifecycleConflict extends Schema.TaggedErrorClass<LifecycleConflict>()("SessionInput.LifecycleConflict", {
id: SessionMessage.ID,
}) {}
export const pendingCompaction = Effect.fn("SessionInput.pendingCompaction")(function* (
db: DatabaseService,
sessionID: SessionSchema.ID,
) {
const row = yield* db
.select()
.from(SessionInputTable)
.where(
and(
eq(SessionInputTable.session_id, sessionID),
eq(SessionInputTable.type, "compaction"),
isNull(SessionInputTable.promoted_seq),
),
)
.orderBy(asc(SessionInputTable.admitted_seq))
.limit(1)
.get()
.pipe(Effect.orDie)
if (!row) return
const entry = fromRow(row)
return entry.type === "compaction" ? entry : undefined
})
export const admit = Effect.fn("SessionInput.admit")(function* (
db: DatabaseService,
@ -49,7 +97,10 @@ export const admit = Effect.fn("SessionInput.admit")(function* (
},
) {
const existing = yield* find(db, input.id)
if (existing !== undefined) return existing
if (existing !== undefined) {
if (existing.type !== "prompt") return yield* Effect.die(new LifecycleConflict({ id: input.id }))
return toAdmitted(existing)
}
return yield* events
.publish(SessionEvent.PromptAdmitted, {
inputID: input.id,
@ -73,11 +124,54 @@ export const admit = Effect.fn("SessionInput.admit")(function* (
),
),
Effect.catchDefect((defect) =>
find(db, input.id).pipe(Effect.flatMap((stored) => (stored ? Effect.succeed(stored) : Effect.die(defect)))),
find(db, input.id).pipe(
Effect.flatMap((stored) =>
stored?.type === "prompt" ? Effect.succeed(toAdmitted(stored)) : Effect.die(defect),
),
),
),
)
})
export const admitCompaction = Effect.fn("SessionInput.admitCompaction")(function* (
db: DatabaseService,
events: EventV2.Interface,
input: { readonly id: SessionMessage.ID; readonly sessionID: SessionSchema.ID },
) {
return yield* inboxLocks.withLock(input.sessionID)(
Effect.gen(function* () {
const exact = yield* find(db, input.id)
if (exact) {
if (exact.type === "compaction" && exact.sessionID === input.sessionID) return exact
return yield* Effect.die(new LifecycleConflict({ id: input.id }))
}
const pending = yield* pendingCompaction(db, input.sessionID)
if (pending) return pending
return yield* events
.publish(SessionEvent.Compaction.Admitted, {
inputID: input.id,
sessionID: input.sessionID,
})
.pipe(
Effect.flatMap((event) => {
if (event.durable === undefined)
return Effect.die(new Error("Compaction admission event is missing aggregate sequence"))
return pendingCompaction(db, input.sessionID).pipe(
Effect.flatMap((stored) =>
stored ? Effect.succeed(stored) : Effect.die(new LifecycleConflict({ id: input.id })),
),
)
}),
Effect.catchDefect((defect) =>
pendingCompaction(db, input.sessionID).pipe(
Effect.flatMap((stored) => (stored ? Effect.succeed(stored) : Effect.die(defect))),
),
),
)
}),
)
})
export const projectAdmitted = Effect.fn("SessionInput.projectAdmitted")(function* (
db: DatabaseService,
input: {
@ -101,6 +195,7 @@ export const projectAdmitted = Effect.fn("SessionInput.projectAdmitted")(functio
.values({
id: input.id,
session_id: input.sessionID,
type: "prompt",
admitted_seq: input.admittedSeq,
prompt: encodePrompt(input.prompt),
delivery: input.delivery,
@ -113,6 +208,44 @@ export const projectAdmitted = Effect.fn("SessionInput.projectAdmitted")(functio
if (!stored) return yield* Effect.die(new LifecycleConflict({ id: input.id }))
})
export const projectCompactionAdmitted = Effect.fn("SessionInput.projectCompactionAdmitted")(function* (
db: DatabaseService,
input: {
readonly admittedSeq: number
readonly id: SessionMessage.ID
readonly sessionID: SessionSchema.ID
readonly timeCreated: DateTime.Utc
},
) {
const message = yield* db
.select({ id: SessionMessageTable.id })
.from(SessionMessageTable)
.where(eq(SessionMessageTable.id, input.id))
.get()
.pipe(Effect.orDie)
if (message !== undefined) return yield* Effect.die(new LifecycleConflict({ id: input.id }))
const stored = yield* db
.insert(SessionInputTable)
.values({
id: input.id,
session_id: input.sessionID,
type: "compaction",
admitted_seq: input.admittedSeq,
time_created: DateTime.toEpochMillis(input.timeCreated),
})
.onConflictDoNothing()
.returning()
.get()
.pipe(Effect.orDie)
if (stored) {
const entry = fromRow(stored)
return entry.type === "compaction" ? entry : yield* Effect.die(new LifecycleConflict({ id: entry.id }))
}
const pending = yield* pendingCompaction(db, input.sessionID)
if (pending) return pending
return yield* Effect.die(new LifecycleConflict({ id: input.id }))
})
export const projectPromptPromoted = Effect.fn("SessionInput.projectPromptPromoted")(function* (
db: DatabaseService,
input: {
@ -121,6 +254,7 @@ export const projectPromptPromoted = Effect.fn("SessionInput.projectPromptPromot
readonly promotedSeq: number
},
) {
if (yield* pendingCompaction(db, input.sessionID)) return yield* Effect.die(new LifecycleConflict({ id: input.id }))
const updated = yield* db
.update(SessionInputTable)
.set({ promoted_seq: input.promotedSeq })
@ -128,6 +262,7 @@ export const projectPromptPromoted = Effect.fn("SessionInput.projectPromptPromot
and(
eq(SessionInputTable.id, input.id),
eq(SessionInputTable.session_id, input.sessionID),
eq(SessionInputTable.type, "prompt"),
isNull(SessionInputTable.promoted_seq),
),
)
@ -136,29 +271,58 @@ export const projectPromptPromoted = Effect.fn("SessionInput.projectPromptPromot
.pipe(Effect.orDie)
if (updated) {
const stored = fromRow(updated)
if (stored.sessionID !== input.sessionID) return yield* Effect.die(new LifecycleConflict({ id: input.id }))
if (stored.type !== "prompt" || stored.sessionID !== input.sessionID)
return yield* Effect.die(new LifecycleConflict({ id: input.id }))
return stored
}
// Every PromptPromoted event is published from an admitted inbox row, so a missing or
// divergent row on replay is an invariant violation.
const stored = yield* find(db, input.id)
if (!stored || stored.sessionID !== input.sessionID || stored.promotedSeq !== input.promotedSeq)
if (
!stored ||
stored.type !== "prompt" ||
stored.sessionID !== input.sessionID ||
stored.promotedSeq !== input.promotedSeq
)
return yield* Effect.die(new LifecycleConflict({ id: input.id }))
return stored
})
export const settleCompaction = Effect.fn("SessionInput.settleCompaction")(function* (
db: DatabaseService,
input: { readonly sessionID: SessionSchema.ID; readonly handledSeq: number },
) {
const updated = yield* db
.update(SessionInputTable)
.set({ promoted_seq: input.handledSeq })
.where(
and(
eq(SessionInputTable.session_id, input.sessionID),
eq(SessionInputTable.type, "compaction"),
isNull(SessionInputTable.promoted_seq),
),
)
.returning()
.get()
.pipe(Effect.orDie)
if (updated) {
const stored = fromRow(updated)
return stored.type === "compaction" ? stored : yield* Effect.die(new LifecycleConflict({ id: stored.id }))
}
return undefined
})
export const hasPending = Effect.fn("SessionInput.hasPending")(function* (
db: DatabaseService,
sessionID: SessionSchema.ID,
delivery: Delivery,
) {
if (yield* pendingCompaction(db, sessionID)) return false
const row = yield* db
.select({ id: SessionInputTable.id })
.from(SessionInputTable)
.where(
and(
eq(SessionInputTable.session_id, sessionID),
eq(SessionInputTable.type, "prompt"),
isNull(SessionInputTable.promoted_seq),
eq(SessionInputTable.delivery, delivery),
),
@ -181,42 +345,44 @@ export const equivalent = (
input.sessionID === expected.sessionID &&
JSON.stringify(encodePrompt(input.prompt)) === JSON.stringify(encodePrompt(expected.prompt))
const matchesProjection = (
input: Admitted,
expected: {
readonly sessionID: SessionSchema.ID
readonly prompt: Prompt
readonly delivery: Delivery
readonly timeCreated: DateTime.Utc
},
) =>
equivalent(input, expected) &&
DateTime.toEpochMillis(input.timeCreated) === DateTime.toEpochMillis(expected.timeCreated)
const publish = Effect.fn("SessionInput.publish")(function* (
db: DatabaseService,
events: EventV2.Interface,
sessionID: SessionSchema.ID,
rows: ReadonlyArray<typeof SessionInputTable.$inferSelect>,
) {
for (const row of rows) {
const id = SessionMessage.ID.make(row.id)
yield* events
.publish(SessionEvent.PromptPromoted, {
sessionID,
inputID: id,
})
.pipe(
Effect.catchDefect((defect) =>
defect instanceof LifecycleConflict
? find(db, id).pipe(
Effect.flatMap((stored) => (stored?.promotedSeq === undefined ? Effect.die(defect) : Effect.void)),
)
: Effect.die(defect),
),
return yield* inboxLocks.withLock(sessionID)(
Effect.gen(function* () {
if (yield* pendingCompaction(db, sessionID)) return 0
yield* Effect.forEach(
rows,
(row) => {
const entry = fromRow(row)
if (entry.type !== "prompt") return Effect.die(new LifecycleConflict({ id: entry.id }))
return events
.publish(SessionEvent.PromptPromoted, {
sessionID,
inputID: entry.id,
})
.pipe(
Effect.catchDefect((defect) =>
defect instanceof LifecycleConflict
? find(db, entry.id).pipe(
Effect.flatMap((stored) =>
stored?.type === "prompt" && stored.promotedSeq !== undefined
? Effect.void
: Effect.die(defect),
),
)
: Effect.die(defect),
),
)
},
{ discard: true },
)
}
return rows.length
return rows.length
}),
)
})
export const promoteSteers = Effect.fn("SessionInput.promoteSteers")(function* (
@ -224,12 +390,14 @@ export const promoteSteers = Effect.fn("SessionInput.promoteSteers")(function* (
events: EventV2.Interface,
sessionID: SessionSchema.ID,
) {
if (yield* pendingCompaction(db, sessionID)) return 0
const rows = yield* db
.select()
.from(SessionInputTable)
.where(
and(
eq(SessionInputTable.session_id, sessionID),
eq(SessionInputTable.type, "prompt"),
isNull(SessionInputTable.promoted_seq),
eq(SessionInputTable.delivery, "steer"),
),
@ -245,12 +413,14 @@ export const promoteNextQueued = Effect.fn("SessionInput.promoteNextQueued")(fun
events: EventV2.Interface,
sessionID: SessionSchema.ID,
) {
if (yield* pendingCompaction(db, sessionID)) return false
const row = yield* db
.select()
.from(SessionInputTable)
.where(
and(
eq(SessionInputTable.session_id, sessionID),
eq(SessionInputTable.type, "prompt"),
isNull(SessionInputTable.promoted_seq),
eq(SessionInputTable.delivery, "queue"),
),

76
packages/core/src/session/message-updater.ts
View file

@ -16,8 +16,10 @@ export interface Adapter {
readonly getShell: (
shellID: SessionMessage.Shell["shell"]["id"],
) => Effect.Effect<SessionMessage.Shell | undefined, never, never>
readonly getCompaction: () => Effect.Effect<SessionMessage.Compaction | undefined, never, never>
readonly updateAssistant: (assistant: SessionMessage.Assistant) => Effect.Effect<void, never, never>
readonly updateShell: (shell: SessionMessage.Shell) => Effect.Effect<void, never, never>
readonly updateCompaction: (compaction: SessionMessage.Compaction) => Effect.Effect<void, never, never>
readonly appendMessage: (message: SessionMessage.Message) => Effect.Effect<void, never, never>
}
@ -26,6 +28,10 @@ export function memory(state: MemoryState): Adapter {
state.messages.findLastIndex((message) => message.id === messageID)
const shellIndex = (messageID: SessionMessage.ID) =>
state.messages.findLastIndex((message) => message.id === messageID)
const compactionIndex = () =>
state.messages.findLastIndex(
(message) => message.type === "compaction" && (message.status === "queued" || message.status === "running"),
)
// A newer step supersedes stale incomplete rows; never resume an older assistant projection.
const latestAssistantIndex = () => state.messages.findLastIndex((message) => message.type === "assistant")
@ -62,6 +68,13 @@ export function memory(state: MemoryState): Adapter {
})
})
},
getCompaction() {
return Effect.sync(() => {
const index = compactionIndex()
const message = state.messages[index]
return message?.type === "compaction" ? message : undefined
})
},
updateAssistant(assistant) {
return Effect.sync(() => {
const index = assistantIndex(assistant.id)
@ -80,6 +93,12 @@ export function memory(state: MemoryState): Adapter {
state.messages[index] = shell
})
},
updateCompaction(compaction) {
return Effect.sync(() => {
const index = state.messages.findLastIndex((message) => message.id === compaction.id)
if (index >= 0) state.messages[index] = compaction
})
},
appendMessage(message) {
return Effect.sync(() => {
state.messages.push(message)
@ -424,21 +443,60 @@ export function update(adapter: Adapter, event: SessionEvent.Event) {
}
})
},
"session.compaction.started": () => Effect.void,
"session.compaction.delta": () => Effect.void,
"session.compaction.ended": (event) => {
return adapter.appendMessage(
"session.compaction.admitted": (event) =>
adapter.appendMessage(
SessionMessage.Compaction.make({
id: SessionMessage.ID.fromEvent(event.id),
id: event.data.inputID,
type: "compaction",
status: "queued",
metadata: event.metadata,
reason: event.data.reason,
summary: event.data.text,
recent: event.data.recent,
reason: "manual",
summary: "",
recent: "",
time: { created: event.created },
}),
)
),
"session.compaction.started": (event) =>
Effect.gen(function* () {
if (event.data.reason !== "manual") return
const current = yield* adapter.getCompaction()
if (!current) return
yield* adapter.updateCompaction({ ...current, status: "running" })
}),
"session.compaction.delta": () => Effect.void,
"session.compaction.ended": (event) => {
return Effect.gen(function* () {
const current = event.data.reason === "manual" ? yield* adapter.getCompaction() : undefined
if (current) {
yield* adapter.updateCompaction({
...current,
status: "completed",
reason: event.data.reason,
summary: event.data.text,
recent: event.data.recent,
})
return
}
yield* adapter.appendMessage(
SessionMessage.Compaction.make({
id: SessionMessage.ID.fromEvent(event.id),
type: "compaction",
status: "completed",
metadata: event.metadata,
reason: event.data.reason,
summary: event.data.text,
recent: event.data.recent,
time: { created: event.created },
}),
)
})
},
"session.compaction.failed": () =>
Effect.gen(function* () {
const current = yield* adapter.getCompaction()
if (!current) return
yield* adapter.updateCompaction({ ...current, status: "failed" })
}),
"session.revert.staged": () => Effect.void,
"session.revert.cleared": () => Effect.void,
"session.revert.committed": () => Effect.void,

65
packages/core/src/session/projector.ts
View file

@ -248,6 +248,7 @@ const projectFork = Effect.fn("SessionProjector.projectFork")(function* (
eq(SessionMessageTable.session_id, event.data.parentID),
gt(SessionMessageTable.seq, cursor),
copiedSeq === 0 ? undefined : lt(SessionMessageTable.seq, copiedSeq + 1),
sql`${SessionMessageTable.type} != 'compaction' or json_extract(${SessionMessageTable.data}, '$.status') not in ('queued', 'running')`,
),
)
.orderBy(asc(SessionMessageTable.seq))
@ -297,11 +298,12 @@ const projectFork = Effect.fn("SessionProjector.projectFork")(function* (
.values(
inputRows.flatMap((row) => {
const id = idMap.get(row.id)
return id
return id && row.type === "prompt"
? [
{
id,
session_id: event.data.sessionID,
type: "prompt" as const,
prompt: row.prompt,
delivery: row.delivery,
admitted_seq: row.admitted_seq,
@ -431,8 +433,30 @@ function run(db: DatabaseService, event: MessageEvent) {
return message.type === "shell" ? message : undefined
})
},
getCompaction() {
return Effect.gen(function* () {
const row = yield* db
.select()
.from(SessionMessageTable)
.where(
and(
eq(SessionMessageTable.session_id, event.data.sessionID),
eq(SessionMessageTable.type, "compaction"),
sql`json_extract(${SessionMessageTable.data}, '$.status') in ('queued', 'running')`,
),
)
.orderBy(desc(SessionMessageTable.seq))
.limit(1)
.get()
.pipe(Effect.orDie)
if (!row) return
const message = decodeRow(row)
return message.type === "compaction" ? message : undefined
})
},
updateAssistant: updateMessage,
updateShell: updateMessage,
updateCompaction: updateMessage,
appendMessage,
}
yield* SessionMessageUpdater.update(adapter, event)
@ -642,6 +666,20 @@ const layer = Layer.effectDiscard(
})
}),
)
yield* events.project(SessionEvent.Compaction.Admitted, (event) =>
Effect.gen(function* () {
if (event.durable === undefined)
return yield* Effect.die(new Error("Durable Session event is missing aggregate sequence"))
const admitted = yield* SessionInput.projectCompactionAdmitted(db, {
admittedSeq: event.durable.seq,
id: event.data.inputID,
sessionID: event.data.sessionID,
timeCreated: event.created,
})
if (admitted.id !== event.data.inputID) return
yield* run(db, event)
}),
)
yield* events.project(SessionEvent.Execution.Succeeded, (event) => run(db, event))
yield* events.project(SessionEvent.Execution.Failed, (event) => run(db, event))
yield* events.project(SessionEvent.Execution.Interrupted, (event) => run(db, event))
@ -672,7 +710,30 @@ const layer = Layer.effectDiscard(
yield* events.project(SessionEvent.Reasoning.Started, (event) => run(db, event))
yield* events.project(SessionEvent.Reasoning.Ended, (event) => run(db, event))
yield* events.project(SessionEvent.RetryScheduled, (event) => run(db, event))
yield* events.project(SessionEvent.Compaction.Ended, (event) => run(db, event))
yield* events.project(SessionEvent.Compaction.Started, (event) => run(db, event))
yield* events.project(SessionEvent.Compaction.Ended, (event) =>
Effect.gen(function* () {
yield* run(db, event)
if (event.durable === undefined)
return yield* Effect.die(new Error("Durable Session event is missing aggregate sequence"))
if (event.data.reason === "manual")
yield* SessionInput.settleCompaction(db, {
sessionID: event.data.sessionID,
handledSeq: event.durable.seq,
})
}),
)
yield* events.project(SessionEvent.Compaction.Failed, (event) =>
Effect.gen(function* () {
yield* run(db, event)
if (event.durable === undefined)
return yield* Effect.die(new Error("Durable Session event is missing aggregate sequence"))
yield* SessionInput.settleCompaction(db, {
sessionID: event.data.sessionID,
handledSeq: event.durable.seq,
})
}),
)
yield* events.project(SessionEvent.RevertEvent.Staged, (event) =>
db
.update(SessionTable)

44
packages/core/src/session/runner/llm.ts
View file

@ -228,7 +228,10 @@ const layer = Layer.effect(
toolChoice: isLastStep ? "none" : undefined,
})
// Automatic compaction completed; rebuild the request from compacted history.
if (yield* compaction.compactIfNeeded({ sessionID: session.id, messages: context, request }))
if (
!(yield* SessionInput.pendingCompaction(db, session.id)) &&
(yield* compaction.compactIfNeeded({ sessionID: session.id, messages: context, request }))
)
return { _tag: "RestartAfterCompaction", step: currentStep } as const
const startSnapshot = yield* snapshots.capture()
const publisher = createLLMEventPublisher(events, {
@ -511,11 +514,36 @@ const layer = Layer.effect(
}
})
const runPendingCompaction = Effect.fn("SessionRunner.runPendingCompaction")(function* (
sessionID: SessionSchema.ID,
) {
const pending = yield* SessionInput.pendingCompaction(db, sessionID)
if (!pending) return false
const session = yield* getSession(sessionID)
return yield* Effect.uninterruptibleMask((restore) =>
Effect.gen(function* () {
const compacted = yield* restore(
Effect.gen(function* () {
return yield* compaction.compactManual({
session,
messages: yield* store.context(sessionID),
})
}),
).pipe(Effect.exit)
if (Exit.isSuccess(compacted) && compacted.value) return true
yield* events.publish(SessionEvent.Compaction.Failed, { sessionID })
if (Exit.isFailure(compacted)) return yield* Effect.failCause(compacted.cause)
return true
}),
)
})
// Execution lifecycle is published per busy period by SessionExecution, not per drain here.
const drain = Effect.fn("SessionRunner.drain")(function* (input: {
readonly sessionID: SessionSchema.ID
readonly force: boolean
}) {
yield* runPendingCompaction(input.sessionID)
const hasSteer = yield* SessionInput.hasPending(db, input.sessionID, "steer")
const hasQueue = hasSteer ? false : yield* SessionInput.hasPending(db, input.sessionID, "queue")
if (!input.force && !hasSteer && !hasQueue) return
@ -539,11 +567,19 @@ const layer = Layer.effect(
}
needsContinuation = result.needsContinuation
step = result.step + 1
if (needsContinuation) {
promotion = (yield* SessionInput.pendingCompaction(db, input.sessionID)) ? undefined : "steer"
continue
}
yield* runPendingCompaction(input.sessionID)
promotion = "steer"
if (!needsContinuation) needsContinuation = yield* SessionInput.hasPending(db, input.sessionID, "steer")
needsContinuation = yield* SessionInput.hasPending(db, input.sessionID, "steer")
}
shouldRun = yield* SessionInput.hasPending(db, input.sessionID, "queue")
promotion = shouldRun ? "queue" : undefined
yield* runPendingCompaction(input.sessionID)
const hasSteer = yield* SessionInput.hasPending(db, input.sessionID, "steer")
const hasQueue = hasSteer ? false : yield* SessionInput.hasPending(db, input.sessionID, "queue")
shouldRun = hasSteer || hasQueue
promotion = hasSteer ? "steer" : hasQueue ? "queue" : undefined
}
})

1
packages/core/src/session/runner/to-llm-message.ts
View file

@ -209,6 +209,7 @@ function toLLMMessage(message: SessionMessage.Message, model: ModelV2.Ref): Mess
case "assistant":
return assistant(message, model)
case "compaction":
if (message.status !== "completed") return []
return [
Message.make({
id: message.id,

12
packages/core/src/session/sql.ts
View file

@ -1,4 +1,5 @@
import { sqliteTable, text, integer, index, primaryKey, real, uniqueIndex } from "drizzle-orm/sqlite-core"
import { sql } from "drizzle-orm"
import { directoryColumn, pathColumn } from "../database/path"
import { ProjectTable } from "../project/sql"
import type { SessionMessage } from "./message"
@ -147,8 +148,9 @@ export const SessionInputTable = sqliteTable(
.$type<SessionSchema.ID>()
.notNull()
.references(() => SessionTable.id, { onDelete: "cascade" }),
prompt: text({ mode: "json" }).notNull().$type<Prompt>(),
delivery: text().$type<SessionInput.Delivery>().notNull(),
type: text().$type<SessionInput.Entry["type"]>().notNull(),
prompt: text({ mode: "json" }).$type<Prompt>(),
delivery: text().$type<SessionInput.Delivery>(),
admitted_seq: integer().notNull(),
promoted_seq: integer(),
time_created: integer()
@ -156,12 +158,16 @@ export const SessionInputTable = sqliteTable(
.$default(() => Date.now()),
},
(table) => [
index("session_input_session_pending_delivery_seq_idx").on(
index("session_input_session_pending_type_delivery_seq_idx").on(
table.session_id,
table.promoted_seq,
table.type,
table.delivery,
table.admitted_seq,
),
uniqueIndex("session_input_session_pending_compaction_idx")
.on(table.session_id)
.where(sql`${table.type} = 'compaction' and ${table.promoted_seq} is null`),
uniqueIndex("session_input_session_admitted_seq_idx").on(table.session_id, table.admitted_seq),
uniqueIndex("session_input_session_promoted_seq_idx").on(table.session_id, table.promoted_seq),
],