sync
This commit is contained in:
parent
c2e234ec4d
commit
fcd5ff7ebe
12 changed files with 279 additions and 237 deletions
|
|
@ -14,6 +14,7 @@ CREATE TABLE `project` (
|
||||||
CREATE TABLE `message` (
|
CREATE TABLE `message` (
|
||||||
`id` text PRIMARY KEY NOT NULL,
|
`id` text PRIMARY KEY NOT NULL,
|
||||||
`session_id` text NOT NULL,
|
`session_id` text NOT NULL,
|
||||||
|
`role` text NOT NULL,
|
||||||
`data` text NOT NULL,
|
`data` text NOT NULL,
|
||||||
FOREIGN KEY (`session_id`) REFERENCES `session`(`id`) ON UPDATE no action ON DELETE cascade
|
FOREIGN KEY (`session_id`) REFERENCES `session`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
);
|
);
|
||||||
|
|
@ -22,6 +23,7 @@ CREATE INDEX `message_session_idx` ON `message` (`session_id`);--> statement-bre
|
||||||
CREATE TABLE `part` (
|
CREATE TABLE `part` (
|
||||||
`id` text PRIMARY KEY NOT NULL,
|
`id` text PRIMARY KEY NOT NULL,
|
||||||
`message_id` text NOT NULL,
|
`message_id` text NOT NULL,
|
||||||
|
`type` text NOT NULL,
|
||||||
`data` text NOT NULL,
|
`data` text NOT NULL,
|
||||||
FOREIGN KEY (`message_id`) REFERENCES `message`(`id`) ON UPDATE no action ON DELETE cascade
|
FOREIGN KEY (`message_id`) REFERENCES `message`(`id`) ON UPDATE no action ON DELETE cascade
|
||||||
);
|
);
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
{
|
{
|
||||||
"version": "6",
|
"version": "6",
|
||||||
"dialect": "sqlite",
|
"dialect": "sqlite",
|
||||||
"id": "9970ec30-1179-4dd7-a0ab-6d0cf0f42219",
|
"id": "f7bf061b-aa6c-4b68-a29f-c210c54f109d",
|
||||||
"prevId": "00000000-0000-0000-0000-000000000000",
|
"prevId": "00000000-0000-0000-0000-000000000000",
|
||||||
"tables": {
|
"tables": {
|
||||||
"project": {
|
"project": {
|
||||||
|
|
@ -101,6 +101,13 @@
|
||||||
"notNull": true,
|
"notNull": true,
|
||||||
"autoincrement": false
|
"autoincrement": false
|
||||||
},
|
},
|
||||||
|
"role": {
|
||||||
|
"name": "role",
|
||||||
|
"type": "text",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
"data": {
|
"data": {
|
||||||
"name": "data",
|
"name": "data",
|
||||||
"type": "text",
|
"type": "text",
|
||||||
|
|
@ -154,6 +161,13 @@
|
||||||
"notNull": true,
|
"notNull": true,
|
||||||
"autoincrement": false
|
"autoincrement": false
|
||||||
},
|
},
|
||||||
|
"type": {
|
||||||
|
"name": "type",
|
||||||
|
"type": "text",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
"data": {
|
"data": {
|
||||||
"name": "data",
|
"name": "data",
|
||||||
"type": "text",
|
"type": "text",
|
||||||
|
|
|
||||||
|
|
@ -5,8 +5,8 @@
|
||||||
{
|
{
|
||||||
"idx": 0,
|
"idx": 0,
|
||||||
"version": "6",
|
"version": "6",
|
||||||
"when": 1768609466939,
|
"when": 1768625754197,
|
||||||
"tag": "0000_watery_shinobi_shaw",
|
"tag": "0000_normal_wind_dancer",
|
||||||
"breakpoints": true
|
"breakpoints": true
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -111,30 +111,32 @@ export const ImportCommand = cmd({
|
||||||
)
|
)
|
||||||
|
|
||||||
for (const msg of exportData.messages) {
|
for (const msg of exportData.messages) {
|
||||||
const { id: msgId, sessionID: msgSessionID, ...msgData } = msg.info
|
const { id: msgId, sessionID: msgSessionID, role: msgRole, ...msgData } = msg.info
|
||||||
Database.use((db) =>
|
Database.use((db) =>
|
||||||
db
|
db
|
||||||
.insert(MessageTable)
|
.insert(MessageTable)
|
||||||
.values({
|
.values({
|
||||||
id: msgId,
|
id: msgId,
|
||||||
sessionID: exportData.info.id,
|
sessionID: exportData.info.id,
|
||||||
|
role: msgRole,
|
||||||
data: msgData,
|
data: msgData,
|
||||||
})
|
})
|
||||||
.onConflictDoUpdate({ target: MessageTable.id, set: { data: msgData } })
|
.onConflictDoUpdate({ target: MessageTable.id, set: { role: msgRole, data: msgData } })
|
||||||
.run(),
|
.run(),
|
||||||
)
|
)
|
||||||
|
|
||||||
for (const part of msg.parts) {
|
for (const part of msg.parts) {
|
||||||
const { id: partId, messageID: _, sessionID: __, ...partData } = part
|
const { id: partId, messageID: _, sessionID: __, type: partType, ...partData } = part
|
||||||
Database.use((db) =>
|
Database.use((db) =>
|
||||||
db
|
db
|
||||||
.insert(PartTable)
|
.insert(PartTable)
|
||||||
.values({
|
.values({
|
||||||
id: partId,
|
id: partId,
|
||||||
messageID: msg.info.id,
|
messageID: msg.info.id,
|
||||||
|
type: partType,
|
||||||
data: partData,
|
data: partData,
|
||||||
})
|
})
|
||||||
.onConflictDoUpdate({ target: PartTable.id, set: { data: partData } })
|
.onConflictDoUpdate({ target: PartTable.id, set: { type: partType, data: partData } })
|
||||||
.run(),
|
.run(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,6 @@ import { BusEvent } from "@/bus/bus-event"
|
||||||
import { Bus } from "@/bus"
|
import { Bus } from "@/bus"
|
||||||
import { Session } from "."
|
import { Session } from "."
|
||||||
import { Identifier } from "../id/id"
|
import { Identifier } from "../id/id"
|
||||||
import { Instance } from "../project/instance"
|
|
||||||
import { Provider } from "../provider/provider"
|
import { Provider } from "../provider/provider"
|
||||||
import { MessageV2 } from "./message-v2"
|
import { MessageV2 } from "./message-v2"
|
||||||
import z from "zod"
|
import z from "zod"
|
||||||
|
|
@ -109,10 +108,6 @@ export namespace SessionCompaction {
|
||||||
mode: "compaction",
|
mode: "compaction",
|
||||||
agent: "compaction",
|
agent: "compaction",
|
||||||
summary: true,
|
summary: true,
|
||||||
path: {
|
|
||||||
cwd: Instance.directory,
|
|
||||||
root: Instance.worktree,
|
|
||||||
},
|
|
||||||
cost: 0,
|
cost: 0,
|
||||||
tokens: {
|
tokens: {
|
||||||
output: 0,
|
output: 0,
|
||||||
|
|
|
||||||
|
|
@ -372,10 +372,6 @@ export namespace MessageV2 {
|
||||||
*/
|
*/
|
||||||
mode: z.string(),
|
mode: z.string(),
|
||||||
agent: z.string(),
|
agent: z.string(),
|
||||||
path: z.object({
|
|
||||||
cwd: z.string(),
|
|
||||||
root: z.string(),
|
|
||||||
}),
|
|
||||||
summary: z.boolean().optional(),
|
summary: z.boolean().optional(),
|
||||||
cost: z.number(),
|
cost: z.number(),
|
||||||
tokens: z.object({
|
tokens: z.object({
|
||||||
|
|
@ -633,7 +629,7 @@ export namespace MessageV2 {
|
||||||
part: PartTable,
|
part: PartTable,
|
||||||
})
|
})
|
||||||
.from(MessageTable)
|
.from(MessageTable)
|
||||||
.leftJoin(PartTable, eq(PartTable.messageID, MessageTable.id))
|
.leftJoin(PartTable, eq(PartTable.message_id, MessageTable.id))
|
||||||
.where(
|
.where(
|
||||||
inArray(
|
inArray(
|
||||||
MessageTable.id,
|
MessageTable.id,
|
||||||
|
|
@ -650,13 +646,19 @@ export namespace MessageV2 {
|
||||||
const first = group[0]
|
const first = group[0]
|
||||||
if (!first) continue
|
if (!first) continue
|
||||||
yield {
|
yield {
|
||||||
info: { ...first.message.data, id: first.message.id, sessionID: first.message.sessionID } as Info,
|
info: {
|
||||||
|
...first.message.data,
|
||||||
|
role: first.message.role,
|
||||||
|
id: first.message.id,
|
||||||
|
sessionID: first.message.sessionID,
|
||||||
|
} as Info,
|
||||||
parts: group
|
parts: group
|
||||||
.filter((row) => row.part)
|
.filter((row) => row.part)
|
||||||
.map((row) => ({
|
.map((row) => ({
|
||||||
...row.part!.data,
|
...row.part!.data,
|
||||||
|
type: row.part!.type,
|
||||||
id: row.part!.id,
|
id: row.part!.id,
|
||||||
messageID: row.part!.messageID,
|
messageID: row.part!.message_id,
|
||||||
sessionID: first.message.sessionID,
|
sessionID: first.message.sessionID,
|
||||||
})) as Part[],
|
})) as Part[],
|
||||||
}
|
}
|
||||||
|
|
@ -672,17 +674,19 @@ export namespace MessageV2 {
|
||||||
db
|
db
|
||||||
.select({
|
.select({
|
||||||
id: PartTable.id,
|
id: PartTable.id,
|
||||||
messageID: PartTable.messageID,
|
messageID: PartTable.message_id,
|
||||||
sessionID: MessageTable.sessionID,
|
sessionID: MessageTable.sessionID,
|
||||||
|
type: PartTable.type,
|
||||||
data: PartTable.data,
|
data: PartTable.data,
|
||||||
})
|
})
|
||||||
.from(PartTable)
|
.from(PartTable)
|
||||||
.innerJoin(MessageTable, eq(PartTable.messageID, MessageTable.id))
|
.innerJoin(MessageTable, eq(PartTable.message_id, MessageTable.id))
|
||||||
.where(eq(PartTable.messageID, messageID))
|
.where(eq(PartTable.message_id, messageID))
|
||||||
.all(),
|
.all(),
|
||||||
)
|
)
|
||||||
const result = rows.map((row) => ({
|
const result = rows.map((row) => ({
|
||||||
...row.data,
|
...row.data,
|
||||||
|
type: row.type,
|
||||||
id: row.id,
|
id: row.id,
|
||||||
messageID: row.messageID,
|
messageID: row.messageID,
|
||||||
sessionID: row.sessionID,
|
sessionID: row.sessionID,
|
||||||
|
|
@ -704,7 +708,7 @@ export namespace MessageV2 {
|
||||||
part: PartTable,
|
part: PartTable,
|
||||||
})
|
})
|
||||||
.from(MessageTable)
|
.from(MessageTable)
|
||||||
.leftJoin(PartTable, eq(PartTable.messageID, MessageTable.id))
|
.leftJoin(PartTable, eq(PartTable.message_id, MessageTable.id))
|
||||||
.where(eq(MessageTable.id, input.messageID))
|
.where(eq(MessageTable.id, input.messageID))
|
||||||
.orderBy(PartTable.id)
|
.orderBy(PartTable.id)
|
||||||
.all(),
|
.all(),
|
||||||
|
|
@ -712,13 +716,19 @@ export namespace MessageV2 {
|
||||||
const first = rows[0]
|
const first = rows[0]
|
||||||
if (!first) throw new Error(`Message not found: ${input.messageID}`)
|
if (!first) throw new Error(`Message not found: ${input.messageID}`)
|
||||||
return {
|
return {
|
||||||
info: { ...first.message.data, id: first.message.id, sessionID: first.message.sessionID } as Info,
|
info: {
|
||||||
|
...first.message.data,
|
||||||
|
role: first.message.role,
|
||||||
|
id: first.message.id,
|
||||||
|
sessionID: first.message.sessionID,
|
||||||
|
} as Info,
|
||||||
parts: rows
|
parts: rows
|
||||||
.filter((row) => row.part)
|
.filter((row) => row.part)
|
||||||
.map((row) => ({
|
.map((row) => ({
|
||||||
...row.part!.data,
|
...row.part!.data,
|
||||||
|
type: row.part!.type,
|
||||||
id: row.part!.id,
|
id: row.part!.id,
|
||||||
messageID: row.part!.messageID,
|
messageID: row.part!.message_id,
|
||||||
sessionID: first.message.sessionID,
|
sessionID: first.message.sessionID,
|
||||||
})) as Part[],
|
})) as Part[],
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -328,10 +328,6 @@ export namespace SessionPrompt {
|
||||||
sessionID,
|
sessionID,
|
||||||
mode: task.agent,
|
mode: task.agent,
|
||||||
agent: task.agent,
|
agent: task.agent,
|
||||||
path: {
|
|
||||||
cwd: Instance.directory,
|
|
||||||
root: Instance.worktree,
|
|
||||||
},
|
|
||||||
cost: 0,
|
cost: 0,
|
||||||
tokens: {
|
tokens: {
|
||||||
input: 0,
|
input: 0,
|
||||||
|
|
@ -530,10 +526,6 @@ export namespace SessionPrompt {
|
||||||
role: "assistant",
|
role: "assistant",
|
||||||
mode: agent.name,
|
mode: agent.name,
|
||||||
agent: agent.name,
|
agent: agent.name,
|
||||||
path: {
|
|
||||||
cwd: Instance.directory,
|
|
||||||
root: Instance.worktree,
|
|
||||||
},
|
|
||||||
cost: 0,
|
cost: 0,
|
||||||
tokens: {
|
tokens: {
|
||||||
input: 0,
|
input: 0,
|
||||||
|
|
@ -1387,10 +1379,6 @@ NOTE: At any point in time through this workflow you should feel free to ask the
|
||||||
mode: input.agent,
|
mode: input.agent,
|
||||||
agent: input.agent,
|
agent: input.agent,
|
||||||
cost: 0,
|
cost: 0,
|
||||||
path: {
|
|
||||||
cwd: Instance.directory,
|
|
||||||
root: Instance.worktree,
|
|
||||||
},
|
|
||||||
time: {
|
time: {
|
||||||
created: Date.now(),
|
created: Date.now(),
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,5 @@
|
||||||
import { Database } from "bun:sqlite"
|
import { Database } from "bun:sqlite"
|
||||||
import { drizzle } from "drizzle-orm/bun-sqlite"
|
import { drizzle } from "drizzle-orm/bun-sqlite"
|
||||||
import { eq } from "drizzle-orm"
|
|
||||||
import { Global } from "../global"
|
import { Global } from "../global"
|
||||||
import { Log } from "../util/log"
|
import { Log } from "../util/log"
|
||||||
import { ProjectTable } from "../project/project.sql"
|
import { ProjectTable } from "../project/project.sql"
|
||||||
|
|
@ -41,17 +40,29 @@ export function migrateFromJson(sqlite: Database, customStorageDir?: string) {
|
||||||
errors: [] as string[],
|
errors: [] as string[],
|
||||||
}
|
}
|
||||||
|
|
||||||
// Migrate projects first (no FK deps)
|
// Run entire migration in a single transaction for performance
|
||||||
const projectGlob = new Bun.Glob("project/*.json")
|
sqlite.run("BEGIN TRANSACTION")
|
||||||
for (const file of projectGlob.scanSync({ cwd: storageDir, absolute: true })) {
|
|
||||||
try {
|
try {
|
||||||
const data = JSON.parse(fs.readFileSync(file, "utf-8"))
|
// Track existing IDs to avoid repeated DB lookups
|
||||||
if (!data.id) {
|
const projectIDs = new Set<string>()
|
||||||
stats.errors.push(`project missing id: ${file}`)
|
const sessionIDs = new Set<string>()
|
||||||
continue
|
const messageIDs = new Set<string>()
|
||||||
}
|
|
||||||
db.insert(ProjectTable)
|
// Migrate projects first (no FK deps)
|
||||||
.values({
|
const projectGlob = new Bun.Glob("project/*.json")
|
||||||
|
const projectFiles = Array.from(projectGlob.scanSync({ cwd: storageDir, absolute: true }))
|
||||||
|
const projectValues: (typeof ProjectTable.$inferInsert)[] = []
|
||||||
|
|
||||||
|
for (const file of projectFiles) {
|
||||||
|
try {
|
||||||
|
const data = JSON.parse(fs.readFileSync(file, "utf-8"))
|
||||||
|
if (!data.id) {
|
||||||
|
stats.errors.push(`project missing id: ${file}`)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
projectIDs.add(data.id)
|
||||||
|
projectValues.push({
|
||||||
id: data.id,
|
id: data.id,
|
||||||
worktree: data.worktree ?? "/",
|
worktree: data.worktree ?? "/",
|
||||||
vcs: data.vcs,
|
vcs: data.vcs,
|
||||||
|
|
@ -63,32 +74,35 @@ export function migrateFromJson(sqlite: Database, customStorageDir?: string) {
|
||||||
time_initialized: data.time?.initialized,
|
time_initialized: data.time?.initialized,
|
||||||
sandboxes: data.sandboxes ?? [],
|
sandboxes: data.sandboxes ?? [],
|
||||||
})
|
})
|
||||||
.onConflictDoNothing()
|
} catch (e) {
|
||||||
.run()
|
stats.errors.push(`failed to migrate project ${file}: ${e}`)
|
||||||
stats.projects++
|
}
|
||||||
} catch (e) {
|
|
||||||
stats.errors.push(`failed to migrate project ${file}: ${e}`)
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
log.info("migrated projects", { count: stats.projects })
|
|
||||||
|
|
||||||
// Migrate sessions (depends on projects)
|
if (projectValues.length > 0) {
|
||||||
const sessionGlob = new Bun.Glob("session/*/*.json")
|
db.insert(ProjectTable).values(projectValues).onConflictDoNothing().run()
|
||||||
for (const file of sessionGlob.scanSync({ cwd: storageDir, absolute: true })) {
|
stats.projects = projectValues.length
|
||||||
try {
|
}
|
||||||
const data = JSON.parse(fs.readFileSync(file, "utf-8"))
|
log.info("migrated projects", { count: stats.projects })
|
||||||
if (!data.id || !data.projectID) {
|
|
||||||
stats.errors.push(`session missing id or projectID: ${file}`)
|
// Migrate sessions (depends on projects)
|
||||||
continue
|
const sessionGlob = new Bun.Glob("session/*/*.json")
|
||||||
}
|
const sessionFiles = Array.from(sessionGlob.scanSync({ cwd: storageDir, absolute: true }))
|
||||||
// Check if project exists (skip orphaned sessions)
|
const sessionValues: (typeof SessionTable.$inferInsert)[] = []
|
||||||
const project = db.select().from(ProjectTable).where(eq(ProjectTable.id, data.projectID)).get()
|
|
||||||
if (!project) {
|
for (const file of sessionFiles) {
|
||||||
log.warn("skipping orphaned session", { sessionID: data.id, projectID: data.projectID })
|
try {
|
||||||
continue
|
const data = JSON.parse(fs.readFileSync(file, "utf-8"))
|
||||||
}
|
if (!data.id || !data.projectID) {
|
||||||
db.insert(SessionTable)
|
stats.errors.push(`session missing id or projectID: ${file}`)
|
||||||
.values({
|
continue
|
||||||
|
}
|
||||||
|
if (!projectIDs.has(data.projectID)) {
|
||||||
|
log.warn("skipping orphaned session", { sessionID: data.id, projectID: data.projectID })
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sessionIDs.add(data.id)
|
||||||
|
sessionValues.push({
|
||||||
id: data.id,
|
id: data.id,
|
||||||
projectID: data.projectID,
|
projectID: data.projectID,
|
||||||
parentID: data.parentID ?? null,
|
parentID: data.parentID ?? null,
|
||||||
|
|
@ -111,167 +125,195 @@ export function migrateFromJson(sqlite: Database, customStorageDir?: string) {
|
||||||
time_compacting: data.time?.compacting ?? null,
|
time_compacting: data.time?.compacting ?? null,
|
||||||
time_archived: data.time?.archived ?? null,
|
time_archived: data.time?.archived ?? null,
|
||||||
})
|
})
|
||||||
.onConflictDoNothing()
|
} catch (e) {
|
||||||
.run()
|
stats.errors.push(`failed to migrate session ${file}: ${e}`)
|
||||||
stats.sessions++
|
}
|
||||||
} catch (e) {
|
|
||||||
stats.errors.push(`failed to migrate session ${file}: ${e}`)
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
log.info("migrated sessions", { count: stats.sessions })
|
|
||||||
|
|
||||||
// Migrate messages (depends on sessions)
|
if (sessionValues.length > 0) {
|
||||||
const messageGlob = new Bun.Glob("message/*/*.json")
|
db.insert(SessionTable).values(sessionValues).onConflictDoNothing().run()
|
||||||
for (const file of messageGlob.scanSync({ cwd: storageDir, absolute: true })) {
|
stats.sessions = sessionValues.length
|
||||||
try {
|
|
||||||
const data = JSON.parse(fs.readFileSync(file, "utf-8"))
|
|
||||||
if (!data.id || !data.sessionID) {
|
|
||||||
stats.errors.push(`message missing id or sessionID: ${file}`)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// Check if session exists
|
|
||||||
const session = db.select().from(SessionTable).where(eq(SessionTable.id, data.sessionID)).get()
|
|
||||||
if (!session) {
|
|
||||||
log.warn("skipping orphaned message", { messageID: data.id, sessionID: data.sessionID })
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
const { id, sessionID, ...rest } = data
|
|
||||||
db.insert(MessageTable)
|
|
||||||
.values({
|
|
||||||
id,
|
|
||||||
sessionID,
|
|
||||||
data: rest,
|
|
||||||
})
|
|
||||||
.onConflictDoNothing()
|
|
||||||
.run()
|
|
||||||
stats.messages++
|
|
||||||
} catch (e) {
|
|
||||||
stats.errors.push(`failed to migrate message ${file}: ${e}`)
|
|
||||||
}
|
}
|
||||||
}
|
log.info("migrated sessions", { count: stats.sessions })
|
||||||
log.info("migrated messages", { count: stats.messages })
|
|
||||||
|
|
||||||
// Migrate parts (depends on messages)
|
// Migrate messages (depends on sessions)
|
||||||
const partGlob = new Bun.Glob("part/*/*.json")
|
const messageGlob = new Bun.Glob("message/*/*.json")
|
||||||
for (const file of partGlob.scanSync({ cwd: storageDir, absolute: true })) {
|
const messageFiles = Array.from(messageGlob.scanSync({ cwd: storageDir, absolute: true }))
|
||||||
try {
|
const messageValues: (typeof MessageTable.$inferInsert)[] = []
|
||||||
const data = JSON.parse(fs.readFileSync(file, "utf-8"))
|
|
||||||
if (!data.id || !data.messageID) {
|
|
||||||
stats.errors.push(`part missing id or messageID: ${file}`)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// Check if message exists
|
|
||||||
const message = db.select().from(MessageTable).where(eq(MessageTable.id, data.messageID)).get()
|
|
||||||
if (!message) {
|
|
||||||
log.warn("skipping orphaned part", { partID: data.id, messageID: data.messageID })
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
const { id, messageID, sessionID: _, ...rest } = data
|
|
||||||
db.insert(PartTable)
|
|
||||||
.values({
|
|
||||||
id,
|
|
||||||
messageID,
|
|
||||||
data: rest,
|
|
||||||
})
|
|
||||||
.onConflictDoNothing()
|
|
||||||
.run()
|
|
||||||
stats.parts++
|
|
||||||
} catch (e) {
|
|
||||||
stats.errors.push(`failed to migrate part ${file}: ${e}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
log.info("migrated parts", { count: stats.parts })
|
|
||||||
|
|
||||||
// Migrate session diffs (use raw SQL since TypeScript schema doesn't match migration)
|
for (const file of messageFiles) {
|
||||||
const diffGlob = new Bun.Glob("session_diff/*.json")
|
try {
|
||||||
for (const file of diffGlob.scanSync({ cwd: storageDir, absolute: true })) {
|
const data = JSON.parse(fs.readFileSync(file, "utf-8"))
|
||||||
try {
|
if (!data.id || !data.sessionID) {
|
||||||
const data = JSON.parse(fs.readFileSync(file, "utf-8"))
|
stats.errors.push(`message missing id or sessionID: ${file}`)
|
||||||
const sessionID = path.basename(file, ".json")
|
continue
|
||||||
// Check if session exists
|
}
|
||||||
const session = db.select().from(SessionTable).where(eq(SessionTable.id, sessionID)).get()
|
if (!sessionIDs.has(data.sessionID)) {
|
||||||
if (!session) {
|
log.warn("skipping orphaned message", { messageID: data.id, sessionID: data.sessionID })
|
||||||
log.warn("skipping orphaned session_diff", { sessionID })
|
continue
|
||||||
continue
|
}
|
||||||
|
messageIDs.add(data.id)
|
||||||
|
const { id, sessionID, role, ...rest } = data
|
||||||
|
messageValues.push({ id, sessionID, role, data: rest })
|
||||||
|
} catch (e) {
|
||||||
|
stats.errors.push(`failed to migrate message ${file}: ${e}`)
|
||||||
}
|
}
|
||||||
sqlite.run("INSERT OR IGNORE INTO session_diff (session_id, data) VALUES (?, ?)", [
|
|
||||||
sessionID,
|
|
||||||
JSON.stringify(data),
|
|
||||||
])
|
|
||||||
stats.diffs++
|
|
||||||
} catch (e) {
|
|
||||||
stats.errors.push(`failed to migrate session_diff ${file}: ${e}`)
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
log.info("migrated session diffs", { count: stats.diffs })
|
|
||||||
|
|
||||||
// Migrate todos
|
if (messageValues.length > 0) {
|
||||||
const todoGlob = new Bun.Glob("todo/*.json")
|
db.insert(MessageTable).values(messageValues).onConflictDoNothing().run()
|
||||||
for (const file of todoGlob.scanSync({ cwd: storageDir, absolute: true })) {
|
stats.messages = messageValues.length
|
||||||
try {
|
}
|
||||||
const data = JSON.parse(fs.readFileSync(file, "utf-8"))
|
log.info("migrated messages", { count: stats.messages })
|
||||||
const sessionID = path.basename(file, ".json")
|
|
||||||
const session = db.select().from(SessionTable).where(eq(SessionTable.id, sessionID)).get()
|
// Migrate parts (depends on messages)
|
||||||
if (!session) {
|
const partGlob = new Bun.Glob("part/*/*.json")
|
||||||
log.warn("skipping orphaned todo", { sessionID })
|
const partFiles = Array.from(partGlob.scanSync({ cwd: storageDir, absolute: true }))
|
||||||
continue
|
const partValues: (typeof PartTable.$inferInsert)[] = []
|
||||||
|
|
||||||
|
for (const file of partFiles) {
|
||||||
|
try {
|
||||||
|
const data = JSON.parse(fs.readFileSync(file, "utf-8"))
|
||||||
|
if (!data.id || !data.messageID) {
|
||||||
|
stats.errors.push(`part missing id or messageID: ${file}`)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (!messageIDs.has(data.messageID)) {
|
||||||
|
log.warn("skipping orphaned part", { partID: data.id, messageID: data.messageID })
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
const { id, messageID, sessionID: _, type, ...rest } = data
|
||||||
|
partValues.push({ id, message_id: messageID, type, data: rest })
|
||||||
|
} catch (e) {
|
||||||
|
stats.errors.push(`failed to migrate part ${file}: ${e}`)
|
||||||
}
|
}
|
||||||
db.insert(TodoTable).values({ sessionID, data }).onConflictDoNothing().run()
|
|
||||||
stats.todos++
|
|
||||||
} catch (e) {
|
|
||||||
stats.errors.push(`failed to migrate todo ${file}: ${e}`)
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
log.info("migrated todos", { count: stats.todos })
|
|
||||||
|
|
||||||
// Migrate permissions
|
if (partValues.length > 0) {
|
||||||
const permGlob = new Bun.Glob("permission/*.json")
|
db.insert(PartTable).values(partValues).onConflictDoNothing().run()
|
||||||
for (const file of permGlob.scanSync({ cwd: storageDir, absolute: true })) {
|
stats.parts = partValues.length
|
||||||
try {
|
}
|
||||||
const data = JSON.parse(fs.readFileSync(file, "utf-8"))
|
log.info("migrated parts", { count: stats.parts })
|
||||||
const projectID = path.basename(file, ".json")
|
|
||||||
const project = db.select().from(ProjectTable).where(eq(ProjectTable.id, projectID)).get()
|
// Migrate session diffs (use prepared statement for batch insert)
|
||||||
if (!project) {
|
const diffGlob = new Bun.Glob("session_diff/*.json")
|
||||||
log.warn("skipping orphaned permission", { projectID })
|
const diffFiles = Array.from(diffGlob.scanSync({ cwd: storageDir, absolute: true }))
|
||||||
continue
|
const diffStmt = sqlite.prepare("INSERT OR IGNORE INTO session_diff (session_id, data) VALUES (?, ?)")
|
||||||
|
|
||||||
|
for (const file of diffFiles) {
|
||||||
|
try {
|
||||||
|
const data = JSON.parse(fs.readFileSync(file, "utf-8"))
|
||||||
|
const sessionID = path.basename(file, ".json")
|
||||||
|
if (!sessionIDs.has(sessionID)) {
|
||||||
|
log.warn("skipping orphaned session_diff", { sessionID })
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
diffStmt.run(sessionID, JSON.stringify(data))
|
||||||
|
stats.diffs++
|
||||||
|
} catch (e) {
|
||||||
|
stats.errors.push(`failed to migrate session_diff ${file}: ${e}`)
|
||||||
}
|
}
|
||||||
db.insert(PermissionTable).values({ projectID, data }).onConflictDoNothing().run()
|
|
||||||
stats.permissions++
|
|
||||||
} catch (e) {
|
|
||||||
stats.errors.push(`failed to migrate permission ${file}: ${e}`)
|
|
||||||
}
|
}
|
||||||
}
|
log.info("migrated session diffs", { count: stats.diffs })
|
||||||
log.info("migrated permissions", { count: stats.permissions })
|
|
||||||
|
|
||||||
// Migrate session shares
|
// Migrate todos
|
||||||
const shareGlob = new Bun.Glob("session_share/*.json")
|
const todoGlob = new Bun.Glob("todo/*.json")
|
||||||
for (const file of shareGlob.scanSync({ cwd: storageDir, absolute: true })) {
|
const todoFiles = Array.from(todoGlob.scanSync({ cwd: storageDir, absolute: true }))
|
||||||
try {
|
const todoValues: (typeof TodoTable.$inferInsert)[] = []
|
||||||
const data = JSON.parse(fs.readFileSync(file, "utf-8"))
|
|
||||||
const sessionID = path.basename(file, ".json")
|
for (const file of todoFiles) {
|
||||||
const session = db.select().from(SessionTable).where(eq(SessionTable.id, sessionID)).get()
|
try {
|
||||||
if (!session) {
|
const data = JSON.parse(fs.readFileSync(file, "utf-8"))
|
||||||
log.warn("skipping orphaned session_share", { sessionID })
|
const sessionID = path.basename(file, ".json")
|
||||||
continue
|
if (!sessionIDs.has(sessionID)) {
|
||||||
|
log.warn("skipping orphaned todo", { sessionID })
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
todoValues.push({ sessionID, data })
|
||||||
|
} catch (e) {
|
||||||
|
stats.errors.push(`failed to migrate todo ${file}: ${e}`)
|
||||||
}
|
}
|
||||||
db.insert(SessionShareTable).values({ sessionID, data }).onConflictDoNothing().run()
|
|
||||||
stats.shares++
|
|
||||||
} catch (e) {
|
|
||||||
stats.errors.push(`failed to migrate session_share ${file}: ${e}`)
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
log.info("migrated session shares", { count: stats.shares })
|
|
||||||
|
|
||||||
// Migrate shares (downloaded shared sessions, no FK)
|
if (todoValues.length > 0) {
|
||||||
const share2Glob = new Bun.Glob("share/*.json")
|
db.insert(TodoTable).values(todoValues).onConflictDoNothing().run()
|
||||||
for (const file of share2Glob.scanSync({ cwd: storageDir, absolute: true })) {
|
stats.todos = todoValues.length
|
||||||
try {
|
|
||||||
const data = JSON.parse(fs.readFileSync(file, "utf-8"))
|
|
||||||
const sessionID = path.basename(file, ".json")
|
|
||||||
db.insert(ShareTable).values({ sessionID, data }).onConflictDoNothing().run()
|
|
||||||
} catch (e) {
|
|
||||||
stats.errors.push(`failed to migrate share ${file}: ${e}`)
|
|
||||||
}
|
}
|
||||||
|
log.info("migrated todos", { count: stats.todos })
|
||||||
|
|
||||||
|
// Migrate permissions
|
||||||
|
const permGlob = new Bun.Glob("permission/*.json")
|
||||||
|
const permFiles = Array.from(permGlob.scanSync({ cwd: storageDir, absolute: true }))
|
||||||
|
const permValues: (typeof PermissionTable.$inferInsert)[] = []
|
||||||
|
|
||||||
|
for (const file of permFiles) {
|
||||||
|
try {
|
||||||
|
const data = JSON.parse(fs.readFileSync(file, "utf-8"))
|
||||||
|
const projectID = path.basename(file, ".json")
|
||||||
|
if (!projectIDs.has(projectID)) {
|
||||||
|
log.warn("skipping orphaned permission", { projectID })
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
permValues.push({ projectID, data })
|
||||||
|
} catch (e) {
|
||||||
|
stats.errors.push(`failed to migrate permission ${file}: ${e}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (permValues.length > 0) {
|
||||||
|
db.insert(PermissionTable).values(permValues).onConflictDoNothing().run()
|
||||||
|
stats.permissions = permValues.length
|
||||||
|
}
|
||||||
|
log.info("migrated permissions", { count: stats.permissions })
|
||||||
|
|
||||||
|
// Migrate session shares
|
||||||
|
const shareGlob = new Bun.Glob("session_share/*.json")
|
||||||
|
const shareFiles = Array.from(shareGlob.scanSync({ cwd: storageDir, absolute: true }))
|
||||||
|
const shareValues: (typeof SessionShareTable.$inferInsert)[] = []
|
||||||
|
|
||||||
|
for (const file of shareFiles) {
|
||||||
|
try {
|
||||||
|
const data = JSON.parse(fs.readFileSync(file, "utf-8"))
|
||||||
|
const sessionID = path.basename(file, ".json")
|
||||||
|
if (!sessionIDs.has(sessionID)) {
|
||||||
|
log.warn("skipping orphaned session_share", { sessionID })
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
shareValues.push({ sessionID, data })
|
||||||
|
} catch (e) {
|
||||||
|
stats.errors.push(`failed to migrate session_share ${file}: ${e}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (shareValues.length > 0) {
|
||||||
|
db.insert(SessionShareTable).values(shareValues).onConflictDoNothing().run()
|
||||||
|
stats.shares = shareValues.length
|
||||||
|
}
|
||||||
|
log.info("migrated session shares", { count: stats.shares })
|
||||||
|
|
||||||
|
// Migrate shares (downloaded shared sessions, no FK)
|
||||||
|
const share2Glob = new Bun.Glob("share/*.json")
|
||||||
|
const share2Files = Array.from(share2Glob.scanSync({ cwd: storageDir, absolute: true }))
|
||||||
|
const share2Values: (typeof ShareTable.$inferInsert)[] = []
|
||||||
|
|
||||||
|
for (const file of share2Files) {
|
||||||
|
try {
|
||||||
|
const data = JSON.parse(fs.readFileSync(file, "utf-8"))
|
||||||
|
const sessionID = path.basename(file, ".json")
|
||||||
|
share2Values.push({ sessionID, data })
|
||||||
|
} catch (e) {
|
||||||
|
stats.errors.push(`failed to migrate share ${file}: ${e}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (share2Values.length > 0) {
|
||||||
|
db.insert(ShareTable).values(share2Values).onConflictDoNothing().run()
|
||||||
|
}
|
||||||
|
|
||||||
|
sqlite.run("COMMIT")
|
||||||
|
} catch (e) {
|
||||||
|
sqlite.run("ROLLBACK")
|
||||||
|
throw e
|
||||||
}
|
}
|
||||||
|
|
||||||
// Mark migration complete
|
// Mark migration complete
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
// Auto-generated - do not edit
|
// Auto-generated - do not edit
|
||||||
import m0 from "../../migration/0000_watery_shinobi_shaw.sql" with { type: "text" }
|
import m0 from "../../migration/0000_normal_wind_dancer.sql" with { type: "text" }
|
||||||
|
|
||||||
export const migrations = [
|
export const migrations = [
|
||||||
{ name: "0000_watery_shinobi_shaw", sql: m0 },
|
{ name: "0000_normal_wind_dancer", sql: m0 },
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,7 @@
|
||||||
import { describe, expect, test, beforeEach, afterEach } from "bun:test"
|
import { describe, expect, test } from "bun:test"
|
||||||
import path from "path"
|
import path from "path"
|
||||||
import { Session } from "../../src/session"
|
import { Session } from "../../src/session"
|
||||||
import { SessionRevert } from "../../src/session/revert"
|
import { SessionRevert } from "../../src/session/revert"
|
||||||
import { SessionCompaction } from "../../src/session/compaction"
|
|
||||||
import { MessageV2 } from "../../src/session/message-v2"
|
import { MessageV2 } from "../../src/session/message-v2"
|
||||||
import { Log } from "../../src/util/log"
|
import { Log } from "../../src/util/log"
|
||||||
import { Instance } from "../../src/project/instance"
|
import { Instance } from "../../src/project/instance"
|
||||||
|
|
@ -53,10 +52,6 @@ describe("revert + compact workflow", () => {
|
||||||
sessionID,
|
sessionID,
|
||||||
mode: "default",
|
mode: "default",
|
||||||
agent: "default",
|
agent: "default",
|
||||||
path: {
|
|
||||||
cwd: tmp.path,
|
|
||||||
root: tmp.path,
|
|
||||||
},
|
|
||||||
cost: 0,
|
cost: 0,
|
||||||
tokens: {
|
tokens: {
|
||||||
output: 0,
|
output: 0,
|
||||||
|
|
@ -113,10 +108,6 @@ describe("revert + compact workflow", () => {
|
||||||
sessionID,
|
sessionID,
|
||||||
mode: "default",
|
mode: "default",
|
||||||
agent: "default",
|
agent: "default",
|
||||||
path: {
|
|
||||||
cwd: tmp.path,
|
|
||||||
root: tmp.path,
|
|
||||||
},
|
|
||||||
cost: 0,
|
cost: 0,
|
||||||
tokens: {
|
tokens: {
|
||||||
output: 0,
|
output: 0,
|
||||||
|
|
@ -227,10 +218,6 @@ describe("revert + compact workflow", () => {
|
||||||
sessionID,
|
sessionID,
|
||||||
mode: "default",
|
mode: "default",
|
||||||
agent: "default",
|
agent: "default",
|
||||||
path: {
|
|
||||||
cwd: tmp.path,
|
|
||||||
root: tmp.path,
|
|
||||||
},
|
|
||||||
cost: 0,
|
cost: 0,
|
||||||
tokens: {
|
tokens: {
|
||||||
output: 0,
|
output: 0,
|
||||||
|
|
|
||||||
|
|
@ -384,7 +384,7 @@ describe("JSON to SQLite migration", () => {
|
||||||
const db = drizzle(sqlite)
|
const db = drizzle(sqlite)
|
||||||
const row = db.select().from(PartTable).where(eq(PartTable.id, fixtures.part.id)).get()
|
const row = db.select().from(PartTable).where(eq(PartTable.id, fixtures.part.id)).get()
|
||||||
expect(row?.id).toBe(fixtures.part.id)
|
expect(row?.id).toBe(fixtures.part.id)
|
||||||
expect(row?.messageID).toBe(fixtures.message.id)
|
expect(row?.message_id).toBe(fixtures.message.id)
|
||||||
})
|
})
|
||||||
|
|
||||||
test("skips orphaned part (missing message)", async () => {
|
test("skips orphaned part (missing message)", async () => {
|
||||||
|
|
|
||||||
2
src/storage/migrations.generated.ts
Normal file
2
src/storage/migrations.generated.ts
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
// Auto-generated - do not edit
|
||||||
|
export const migrations: { name: string; sql: string }[] = []
|
||||||
Loading…
Add table
Add a link
Reference in a new issue