ceos/opencode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(opencode): remove JSON storage migration (#30461)

Browse source
This commit is contained in:
Dax 2026-06-02 19:05:14 -04:00 • committed by GitHub
commit ca2acc4f8d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 17 additions and 1590 deletions
Download patch file Download diff file Expand all files Collapse all files

43
packages/opencode/src/index.ts
View file

@ -15,7 +15,6 @@ import { InstallationVersion } from "@opencode-ai/core/installation/version"
import { NamedError } from "@opencode-ai/core/util/error"
import { FormatError } from "./cli/error"
import { ServeCommand } from "./cli/cmd/serve"
import { Filesystem } from "@/util/filesystem"
import { DebugCommand } from "./cli/cmd/debug"
import { StatsCommand } from "./cli/cmd/stats"
import { McpCommand } from "./cli/cmd/mcp"
@ -30,13 +29,9 @@ import { WebCommand } from "./cli/cmd/web"
import { PrCommand } from "./cli/cmd/pr"
import { SessionCommand } from "./cli/cmd/session"
import { DbCommand } from "./cli/cmd/db"
import { Global } from "@opencode-ai/core/global"
import { JsonMigration } from "@/storage/json-migration"
import { Database } from "@opencode-ai/core/database/database"
import { errorMessage } from "./util/error"
import { PluginCommand } from "./cli/cmd/plug"
import { Heap } from "./cli/heap"
import { drizzle } from "drizzle-orm/bun-sqlite"
import { ensureProcessMetadata } from "@opencode-ai/core/util/opencode-process"
import { isRecord } from "@/util/record"
@ -115,44 +110,6 @@ const cli = yargs(args)
run_id: processMetadata.runID,
})
const marker = Database.path()
if (!(await Filesystem.exists(marker))) {
const tty = process.stderr.isTTY
process.stderr.write("Performing one time database migration, may take a few minutes..." + EOL)
const width = 36
const orange = "\x1b[38;5;214m"
const muted = "\x1b[0;2m"
const reset = "\x1b[0m"
let last = -1
if (tty) process.stderr.write("\x1b[?25l")
const sqlite = new (await import("bun:sqlite")).Database(marker)
try {
await JsonMigration.run(drizzle({ client: sqlite }), {
progress: (event) => {
const percent = Math.floor((event.current / event.total) * 100)
if (percent === last && event.current !== event.total) return
last = percent
if (tty) {
const fill = Math.round((percent / 100) * width)
const bar = `${"■".repeat(fill)}${"･".repeat(width - fill)}`
process.stderr.write(
`\r${orange}${bar} ${percent.toString().padStart(3)}%${reset} ${muted}${event.label.padEnd(12)} ${event.current}/${event.total}${reset}`,
)
if (event.current === event.total) process.stderr.write("\n")
} else {
process.stderr.write(`sqlite-migration:${percent}${EOL}`)
}
},
})
} finally {
sqlite.close()
if (tty) process.stderr.write("\x1b[?25h")
else {
process.stderr.write(`sqlite-migration:done${EOL}`)
}
}
process.stderr.write("Database migration complete." + EOL)
}
})
.usage("")
.completion("completion", "generate shell completion script")

1
packages/opencode/src/node.ts
View file

@ -3,4 +3,3 @@ export { Server } from "./server/server"
export { bootstrap } from "./cli/bootstrap"
export * as Log from "@opencode-ai/core/util/log"
export { Database } from "@opencode-ai/core/database/database"
export { JsonMigration } from "@/storage/json-migration"

409
packages/opencode/src/storage/json-migration.ts
View file

@ -1,409 +0,0 @@
import type { SQLiteBunDatabase } from "drizzle-orm/bun-sqlite"
import type { NodeSQLiteDatabase } from "drizzle-orm/node-sqlite"
import { Global } from "@opencode-ai/core/global"
import * as Log from "@opencode-ai/core/util/log"
import { ProjectTable } from "@opencode-ai/core/project/sql"
import { SessionTable, MessageTable, PartTable, TodoTable } from "@opencode-ai/core/session/sql"
import { SessionShareTable } from "@opencode-ai/core/share/sql"
import path from "path"
import { existsSync } from "fs"
import { Filesystem } from "@/util/filesystem"
import { Glob } from "@opencode-ai/core/util/glob"
const log = Log.create({ service: "json-migration" })
export type Progress = {
current: number
total: number
label: string
}
type Options = {
progress?: (event: Progress) => void
}
export async function run(db: SQLiteBunDatabase<any, any> | NodeSQLiteDatabase<any, any>, options?: Options) {
const storageDir = path.join(Global.Path.data, "storage")
if (!existsSync(storageDir)) {
log.info("storage directory does not exist, skipping migration")
return {
projects: 0,
sessions: 0,
messages: 0,
parts: 0,
todos: 0,
permissions: 0,
shares: 0,
errors: [] as string[],
}
}
log.info("starting json to sqlite migration", { storageDir })
const start = performance.now()
// const db = drizzle({ client: sqlite })
// Optimize SQLite for bulk inserts
db.run("PRAGMA journal_mode = WAL")
db.run("PRAGMA synchronous = OFF")
db.run("PRAGMA cache_size = 10000")
db.run("PRAGMA temp_store = MEMORY")
const stats = {
projects: 0,
sessions: 0,
messages: 0,
parts: 0,
todos: 0,
permissions: 0,
shares: 0,
errors: [] as string[],
}
const orphans = {
sessions: 0,
todos: 0,
permissions: 0,
shares: 0,
}
const errs = stats.errors
const batchSize = 1000
const now = Date.now()
async function list(pattern: string) {
return Glob.scan(pattern, { cwd: storageDir, absolute: true })
}
async function read(files: string[], start: number, end: number) {
const count = end - start
// oxlint-disable-next-line unicorn/no-new-array -- pre-allocated for index-based batch fill
const tasks = new Array(count)
for (let i = 0; i < count; i++) {
tasks[i] = Filesystem.readJson(files[start + i])
}
const results = await Promise.allSettled(tasks)
// oxlint-disable-next-line unicorn/no-new-array -- pre-allocated for index-based batch fill
const items = new Array(count)
for (let i = 0; i < results.length; i++) {
const result = results[i]
if (result.status === "fulfilled") {
items[i] = result.value
continue
}
errs.push(`failed to read ${files[start + i]}: ${result.reason}`)
}
return items
}
function insert(values: unknown[], table: Parameters<typeof db.insert>[0], label: string) {
if (values.length === 0) return 0
try {
db.insert(table).values(values).onConflictDoNothing().run()
return values.length
} catch (e) {
errs.push(`failed to migrate ${label} batch: ${e}`)
return 0
}
}
// Pre-scan all files upfront to avoid repeated glob operations
log.info("scanning files...")
const [projectFiles, sessionFiles, messageFiles, partFiles, todoFiles, shareFiles] = await Promise.all([
list("project/*.json"),
list("session/*/*.json"),
list("message/*/*.json"),
list("part/*/*.json"),
list("todo/*.json"),
list("session_share/*.json"),
])
log.info("file scan complete", {
projects: projectFiles.length,
sessions: sessionFiles.length,
messages: messageFiles.length,
parts: partFiles.length,
todos: todoFiles.length,
shares: shareFiles.length,
})
const total = Math.max(
1,
projectFiles.length +
sessionFiles.length +
messageFiles.length +
partFiles.length +
todoFiles.length +
shareFiles.length,
)
const progress = options?.progress
let current = 0
const step = (label: string, count: number) => {
current = Math.min(total, current + count)
progress?.({ current, total, label })
}
progress?.({ current, total, label: "starting" })
db.run("BEGIN TRANSACTION")
// Migrate projects first (no FK deps)
// Derive all IDs from file paths, not JSON content
const projectIds = new Set<string>()
const projectValues: unknown[] = []
for (let i = 0; i < projectFiles.length; i += batchSize) {
const end = Math.min(i + batchSize, projectFiles.length)
const batch = await read(projectFiles, i, end)
projectValues.length = 0
for (let j = 0; j < batch.length; j++) {
const data = batch[j]
if (!data) continue
const id = path.basename(projectFiles[i + j], ".json")
projectIds.add(id)
projectValues.push({
id,
worktree: data.worktree ?? "/",
vcs: data.vcs,
name: data.name ?? undefined,
icon_url: data.icon?.url,
icon_url_override: data.icon?.override,
icon_color: data.icon?.color,
time_created: data.time?.created ?? now,
time_updated: data.time?.updated ?? now,
time_initialized: data.time?.initialized,
sandboxes: data.sandboxes ?? [],
commands: data.commands,
})
}
stats.projects += insert(projectValues, ProjectTable, "project")
step("projects", end - i)
}
log.info("migrated projects", { count: stats.projects, duration: Math.round(performance.now() - start) })
// Migrate sessions (depends on projects)
// Derive all IDs from directory/file paths, not JSON content, since earlier
// migrations may have moved sessions to new directories without updating the JSON
const sessionProjects = sessionFiles.map((file) => path.basename(path.dirname(file)))
const sessionIds = new Set<string>()
const sessionValues: unknown[] = []
for (let i = 0; i < sessionFiles.length; i += batchSize) {
const end = Math.min(i + batchSize, sessionFiles.length)
const batch = await read(sessionFiles, i, end)
sessionValues.length = 0
for (let j = 0; j < batch.length; j++) {
const data = batch[j]
if (!data) continue
const id = path.basename(sessionFiles[i + j], ".json")
const projectID = sessionProjects[i + j]
if (!projectIds.has(projectID)) {
orphans.sessions++
continue
}
sessionIds.add(id)
sessionValues.push({
id,
project_id: projectID,
parent_id: data.parentID ?? null,
slug: data.slug ?? "",
directory: data.directory ?? "",
path: data.path ?? null,
title: data.title ?? "",
version: data.version ?? "",
share_url: data.share?.url ?? null,
summary_additions: data.summary?.additions ?? null,
summary_deletions: data.summary?.deletions ?? null,
summary_files: data.summary?.files ?? null,
summary_diffs: data.summary?.diffs ?? null,
cost: 0,
tokens_input: 0,
tokens_output: 0,
tokens_reasoning: 0,
tokens_cache_read: 0,
tokens_cache_write: 0,
revert: data.revert ?? null,
permission: data.permission ?? null,
time_created: data.time?.created ?? now,
time_updated: data.time?.updated ?? now,
time_compacting: data.time?.compacting ?? null,
time_archived: data.time?.archived ?? null,
})
}
stats.sessions += insert(sessionValues, SessionTable, "session")
step("sessions", end - i)
}
log.info("migrated sessions", { count: stats.sessions })
if (orphans.sessions > 0) {
log.warn("skipped orphaned sessions", { count: orphans.sessions })
}
// Migrate messages using pre-scanned file map
const allMessageFiles = [] as string[]
const allMessageSessions = [] as string[]
const messageSessions = new Map<string, string>()
for (const file of messageFiles) {
const sessionID = path.basename(path.dirname(file))
if (!sessionIds.has(sessionID)) continue
allMessageFiles.push(file)
allMessageSessions.push(sessionID)
}
for (let i = 0; i < allMessageFiles.length; i += batchSize) {
const end = Math.min(i + batchSize, allMessageFiles.length)
const batch = await read(allMessageFiles, i, end)
// oxlint-disable-next-line unicorn/no-new-array -- pre-allocated for index-based batch fill
const values = new Array(batch.length)
let count = 0
for (let j = 0; j < batch.length; j++) {
const data = batch[j]
if (!data) continue
const file = allMessageFiles[i + j]
const id = path.basename(file, ".json")
const sessionID = allMessageSessions[i + j]
messageSessions.set(id, sessionID)
const rest = data
delete rest.id
delete rest.sessionID
values[count++] = {
id,
session_id: sessionID,
time_created: data.time?.created ?? now,
time_updated: data.time?.updated ?? now,
data: rest,
}
}
values.length = count
stats.messages += insert(values, MessageTable, "message")
step("messages", end - i)
}
log.info("migrated messages", { count: stats.messages })
// Migrate parts using pre-scanned file map
for (let i = 0; i < partFiles.length; i += batchSize) {
const end = Math.min(i + batchSize, partFiles.length)
const batch = await read(partFiles, i, end)
// oxlint-disable-next-line unicorn/no-new-array -- pre-allocated for index-based batch fill
const values = new Array(batch.length)
let count = 0
for (let j = 0; j < batch.length; j++) {
const data = batch[j]
if (!data) continue
const file = partFiles[i + j]
const id = path.basename(file, ".json")
const messageID = path.basename(path.dirname(file))
const sessionID = messageSessions.get(messageID)
if (!sessionID) {
errs.push(`part missing message session: ${file}`)
continue
}
if (!sessionIds.has(sessionID)) continue
const rest = data
delete rest.id
delete rest.messageID
delete rest.sessionID
values[count++] = {
id,
message_id: messageID,
session_id: sessionID,
time_created: data.time?.created ?? now,
time_updated: data.time?.updated ?? now,
data: rest,
}
}
values.length = count
stats.parts += insert(values, PartTable, "part")
step("parts", end - i)
}
log.info("migrated parts", { count: stats.parts })
// Migrate todos
const todoSessions = todoFiles.map((file) => path.basename(file, ".json"))
for (let i = 0; i < todoFiles.length; i += batchSize) {
const end = Math.min(i + batchSize, todoFiles.length)
const batch = await read(todoFiles, i, end)
const values: unknown[] = []
for (let j = 0; j < batch.length; j++) {
const data = batch[j]
if (!data) continue
const sessionID = todoSessions[i + j]
if (!sessionIds.has(sessionID)) {
orphans.todos++
continue
}
if (!Array.isArray(data)) {
errs.push(`todo not an array: ${todoFiles[i + j]}`)
continue
}
for (let position = 0; position < data.length; position++) {
const todo = data[position]
if (!todo?.content || !todo?.status || !todo?.priority) continue
values.push({
session_id: sessionID,
content: todo.content,
status: todo.status,
priority: todo.priority,
position,
time_created: now,
time_updated: now,
})
}
}
stats.todos += insert(values, TodoTable, "todo")
step("todos", end - i)
}
log.info("migrated todos", { count: stats.todos })
if (orphans.todos > 0) {
log.warn("skipped orphaned todos", { count: orphans.todos })
}
// Migrate session shares
const shareSessions = shareFiles.map((file) => path.basename(file, ".json"))
const shareValues: unknown[] = []
for (let i = 0; i < shareFiles.length; i += batchSize) {
const end = Math.min(i + batchSize, shareFiles.length)
const batch = await read(shareFiles, i, end)
shareValues.length = 0
for (let j = 0; j < batch.length; j++) {
const data = batch[j]
if (!data) continue
const sessionID = shareSessions[i + j]
if (!sessionIds.has(sessionID)) {
orphans.shares++
continue
}
if (!data?.id || !data?.secret || !data?.url) {
errs.push(`session_share missing id/secret/url: ${shareFiles[i + j]}`)
continue
}
shareValues.push({ session_id: sessionID, id: data.id, secret: data.secret, url: data.url })
}
stats.shares += insert(shareValues, SessionShareTable, "session_share")
step("shares", end - i)
}
log.info("migrated session shares", { count: stats.shares })
if (orphans.shares > 0) {
log.warn("skipped orphaned session shares", { count: orphans.shares })
}
db.run("COMMIT")
log.info("json migration complete", {
projects: stats.projects,
sessions: stats.sessions,
messages: stats.messages,
parts: stats.parts,
todos: stats.todos,
permissions: stats.permissions,
shares: stats.shares,
errorCount: stats.errors.length,
duration: Math.round(performance.now() - start),
})
if (stats.errors.length > 0) {
log.warn("migration errors", { errors: stats.errors.slice(0, 20) })
}
progress?.({ current: total, total, label: "complete" })
return stats
}
export * as JsonMigration from "./json-migration"

856
packages/opencode/test/storage/json-migration.test.ts
View file

@ -1,856 +0,0 @@
import { describe, test, expect, beforeEach, afterEach } from "bun:test"
import { Database } from "bun:sqlite"
import { drizzle, SQLiteBunDatabase } from "drizzle-orm/bun-sqlite"
import { migrate } from "drizzle-orm/bun-sqlite/migrator"
import path from "path"
import fs from "fs/promises"
import { existsSync, readFileSync, readdirSync } from "fs"
import { JsonMigration } from "@/storage/json-migration"
import { Global } from "@opencode-ai/core/global"
import { ProjectTable } from "@opencode-ai/core/project/sql"
import { ProjectV2 } from "@opencode-ai/core/project"
import { AbsolutePath } from "@opencode-ai/core/schema"
import { SessionTable, MessageTable, PartTable, TodoTable } from "@opencode-ai/core/session/sql"
import { SessionShareTable } from "@opencode-ai/core/share/sql"
import { SessionID, MessageID, PartID } from "../../src/session/schema"
// Test fixtures
const fixtures = {
project: {
id: "proj_test123abc",
name: "Test Project",
worktree: "/test/path",
vcs: "git" as const,
sandboxes: [],
},
session: {
id: "ses_test456def",
projectID: "proj_test123abc",
slug: "test-session",
directory: "/test/path",
title: "Test Session",
version: "1.0.0",
time: { created: 1700000000000, updated: 1700000001000 },
},
message: {
id: "msg_test789ghi",
sessionID: "ses_test456def",
role: "user" as const,
agent: "default",
model: { providerID: "openai", modelID: "gpt-4" },
time: { created: 1700000000000 },
},
part: {
id: "prt_testabc123",
messageID: "msg_test789ghi",
sessionID: "ses_test456def",
type: "text" as const,
text: "Hello, world!",
},
}
// Helper to create test storage directory structure
async function setupStorageDir() {
const storageDir = path.join(Global.Path.data, "storage")
await fs.rm(storageDir, { recursive: true, force: true })
await fs.mkdir(path.join(storageDir, "project"), { recursive: true })
await fs.mkdir(path.join(storageDir, "session", "proj_test123abc"), { recursive: true })
await fs.mkdir(path.join(storageDir, "message", "ses_test456def"), { recursive: true })
await fs.mkdir(path.join(storageDir, "part", "msg_test789ghi"), { recursive: true })
await fs.mkdir(path.join(storageDir, "session_diff"), { recursive: true })
await fs.mkdir(path.join(storageDir, "todo"), { recursive: true })
await fs.mkdir(path.join(storageDir, "permission"), { recursive: true })
await fs.mkdir(path.join(storageDir, "session_share"), { recursive: true })
// Create legacy marker to indicate JSON storage exists
await Bun.write(path.join(storageDir, "migration"), "1")
return storageDir
}
async function writeProject(storageDir: string, project: Record<string, unknown>) {
await Bun.write(path.join(storageDir, "project", `${project.id}.json`), JSON.stringify(project))
}
async function writeSession(storageDir: string, projectID: string, session: Record<string, unknown>) {
await Bun.write(path.join(storageDir, "session", projectID, `${session.id}.json`), JSON.stringify(session))
}
// Helper to create in-memory test database with schema
function createTestDb() {
const sqlite = new Database(":memory:")
sqlite.exec("PRAGMA foreign_keys = ON")
// Apply schema migrations using drizzle migrate
const dir = path.join(import.meta.dirname, "../../../core/migration")
const entries = readdirSync(dir, { withFileTypes: true })
const migrations = entries
.filter((entry) => entry.isDirectory() && existsSync(path.join(dir, entry.name, "migration.sql")))
.map((entry) => ({
sql: readFileSync(path.join(dir, entry.name, "migration.sql"), "utf-8"),
timestamp: Number(entry.name.split("_")[0]),
name: entry.name,
}))
.sort((a, b) => a.timestamp - b.timestamp)
const db = drizzle({ client: sqlite })
migrate(db, migrations)
return [sqlite, db] as const
}
describe("JSON to SQLite migration", () => {
let storageDir: string
let sqlite: Database
let db: SQLiteBunDatabase
beforeEach(async () => {
storageDir = await setupStorageDir()
;[sqlite, db] = createTestDb()
})
afterEach(async () => {
sqlite.close()
await fs.rm(storageDir, { recursive: true, force: true })
})
test("migrates project", async () => {
await writeProject(storageDir, {
id: "proj_test123abc",
worktree: "/test/path",
vcs: "git",
name: "Test Project",
time: { created: 1700000000000, updated: 1700000001000 },
sandboxes: ["/test/sandbox"],
})
const stats = await JsonMigration.run(db)
expect(stats?.projects).toBe(1)
const projects = db.select().from(ProjectTable).all()
expect(projects.length).toBe(1)
expect(projects[0].id).toBe(ProjectV2.ID.make("proj_test123abc"))
expect(projects[0].worktree).toBe(AbsolutePath.make("/test/path"))
expect(projects[0].name).toBe("Test Project")
expect(projects[0].sandboxes).toEqual([AbsolutePath.make("/test/sandbox")])
})
test("stores imported Windows project and session paths in storage form", async () => {
if (process.platform !== "win32") return
await writeProject(storageDir, {
id: "proj_test123abc",
worktree: "C:\\Repo\\Thing",
vcs: "git",
sandboxes: ["C:\\Repo\\Thing\\sandbox"],
})
await writeSession(storageDir, "proj_test123abc", {
id: "ses_test456def",
slug: "storage-path",
directory: "C:\\Repo\\Thing\\packages\\api",
path: "packages\\api",
title: "Storage Path",
version: "test",
})
await JsonMigration.run(db)
expect(sqlite.query("SELECT worktree, sandboxes FROM project WHERE id = ?").get("proj_test123abc")).toEqual({
worktree: "C:/Repo/Thing",
sandboxes: JSON.stringify(["C:/Repo/Thing/sandbox"]),
})
expect(sqlite.query("SELECT directory, path FROM session WHERE id = ?").get("ses_test456def")).toEqual({
directory: "C:/Repo/Thing/packages/api",
path: "packages/api",
})
})
test("uses filename for project id when JSON has different value", async () => {
await Bun.write(
path.join(storageDir, "project", "proj_filename.json"),
JSON.stringify({
id: "proj_different_in_json", // Stale! Should be ignored
worktree: "/test/path",
vcs: "git",
name: "Test Project",
sandboxes: [],
}),
)
const stats = await JsonMigration.run(db)
expect(stats?.projects).toBe(1)
const projects = db.select().from(ProjectTable).all()
expect(projects.length).toBe(1)
expect(projects[0].id).toBe(ProjectV2.ID.make("proj_filename")) // Uses filename, not JSON id
})
test("migrates project with commands", async () => {
await writeProject(storageDir, {
id: "proj_with_commands",
worktree: "/test/path",
vcs: "git",
name: "Project With Commands",
time: { created: 1700000000000, updated: 1700000001000 },
sandboxes: ["/test/sandbox"],
commands: { start: "npm run dev" },
})
const stats = await JsonMigration.run(db)
expect(stats?.projects).toBe(1)
const projects = db.select().from(ProjectTable).all()
expect(projects.length).toBe(1)
expect(projects[0].id).toBe(ProjectV2.ID.make("proj_with_commands"))
expect(projects[0].commands).toEqual({ start: "npm run dev" })
})
test("migrates project without commands field", async () => {
await writeProject(storageDir, {
id: "proj_no_commands",
worktree: "/test/path",
vcs: "git",
name: "Project Without Commands",
time: { created: 1700000000000, updated: 1700000001000 },
sandboxes: [],
})
const stats = await JsonMigration.run(db)
expect(stats?.projects).toBe(1)
const projects = db.select().from(ProjectTable).all()
expect(projects.length).toBe(1)
expect(projects[0].id).toBe(ProjectV2.ID.make("proj_no_commands"))
expect(projects[0].commands).toBeNull()
})
test("migrates session with individual columns", async () => {
await writeProject(storageDir, {
id: "proj_test123abc",
worktree: "/test/path",
time: { created: Date.now(), updated: Date.now() },
sandboxes: [],
})
await writeSession(storageDir, "proj_test123abc", {
id: "ses_test456def",
projectID: "proj_test123abc",
slug: "test-session",
directory: "/test/dir",
title: "Test Session Title",
version: "1.0.0",
time: { created: 1700000000000, updated: 1700000001000 },
summary: { additions: 10, deletions: 5, files: 3 },
share: { url: "https://example.com/share" },
})
await JsonMigration.run(db)
const sessions = db.select().from(SessionTable).all()
expect(sessions.length).toBe(1)
expect(sessions[0].id).toBe(SessionID.make("ses_test456def"))
expect(sessions[0].project_id).toBe(ProjectV2.ID.make("proj_test123abc"))
expect(sessions[0].slug).toBe("test-session")
expect(sessions[0].title).toBe("Test Session Title")
expect(sessions[0].summary_additions).toBe(10)
expect(sessions[0].summary_deletions).toBe(5)
expect(sessions[0].share_url).toBe("https://example.com/share")
})
test("migrates messages and parts", async () => {
await writeProject(storageDir, {
id: "proj_test123abc",
worktree: "/",
time: { created: Date.now(), updated: Date.now() },
sandboxes: [],
})
await writeSession(storageDir, "proj_test123abc", { ...fixtures.session })
await Bun.write(
path.join(storageDir, "message", "ses_test456def", "msg_test789ghi.json"),
JSON.stringify({ ...fixtures.message }),
)
await Bun.write(
path.join(storageDir, "part", "msg_test789ghi", "prt_testabc123.json"),
JSON.stringify({ ...fixtures.part }),
)
const stats = await JsonMigration.run(db)
expect(stats?.messages).toBe(1)
expect(stats?.parts).toBe(1)
const messages = db.select().from(MessageTable).all()
expect(messages.length).toBe(1)
expect(messages[0].id).toBe(MessageID.make("msg_test789ghi"))
const parts = db.select().from(PartTable).all()
expect(parts.length).toBe(1)
expect(parts[0].id).toBe(PartID.make("prt_testabc123"))
})
test("migrates legacy parts without ids in body", async () => {
await writeProject(storageDir, {
id: "proj_test123abc",
worktree: "/",
time: { created: Date.now(), updated: Date.now() },
sandboxes: [],
})
await writeSession(storageDir, "proj_test123abc", { ...fixtures.session })
await Bun.write(
path.join(storageDir, "message", "ses_test456def", "msg_test789ghi.json"),
JSON.stringify({
role: "user",
agent: "default",
model: { providerID: "openai", modelID: "gpt-4" },
time: { created: 1700000000000 },
}),
)
await Bun.write(
path.join(storageDir, "part", "msg_test789ghi", "prt_testabc123.json"),
JSON.stringify({
type: "text",
text: "Hello, world!",
}),
)
const stats = await JsonMigration.run(db)
expect(stats?.messages).toBe(1)
expect(stats?.parts).toBe(1)
const messages = db.select().from(MessageTable).all()
expect(messages.length).toBe(1)
expect(messages[0].id).toBe(MessageID.make("msg_test789ghi"))
expect(messages[0].session_id).toBe(SessionID.make("ses_test456def"))
expect(messages[0].data).not.toHaveProperty("id")
expect(messages[0].data).not.toHaveProperty("sessionID")
const parts = db.select().from(PartTable).all()
expect(parts.length).toBe(1)
expect(parts[0].id).toBe(PartID.make("prt_testabc123"))
expect(parts[0].message_id).toBe(MessageID.make("msg_test789ghi"))
expect(parts[0].session_id).toBe(SessionID.make("ses_test456def"))
expect(parts[0].data).not.toHaveProperty("id")
expect(parts[0].data).not.toHaveProperty("messageID")
expect(parts[0].data).not.toHaveProperty("sessionID")
})
test("uses filename for message id when JSON has different value", async () => {
await writeProject(storageDir, {
id: "proj_test123abc",
worktree: "/",
time: { created: Date.now(), updated: Date.now() },
sandboxes: [],
})
await writeSession(storageDir, "proj_test123abc", { ...fixtures.session })
await Bun.write(
path.join(storageDir, "message", "ses_test456def", "msg_from_filename.json"),
JSON.stringify({
id: "msg_different_in_json", // Stale! Should be ignored
sessionID: "ses_test456def",
role: "user",
agent: "default",
time: { created: 1700000000000 },
}),
)
const stats = await JsonMigration.run(db)
expect(stats?.messages).toBe(1)
const messages = db.select().from(MessageTable).all()
expect(messages.length).toBe(1)
expect(messages[0].id).toBe(MessageID.make("msg_from_filename")) // Uses filename, not JSON id
expect(messages[0].session_id).toBe(SessionID.make("ses_test456def"))
})
test("uses paths for part id and messageID when JSON has different values", async () => {
await writeProject(storageDir, {
id: "proj_test123abc",
worktree: "/",
time: { created: Date.now(), updated: Date.now() },
sandboxes: [],
})
await writeSession(storageDir, "proj_test123abc", { ...fixtures.session })
await Bun.write(
path.join(storageDir, "message", "ses_test456def", "msg_realmsgid.json"),
JSON.stringify({
role: "user",
agent: "default",
time: { created: 1700000000000 },
}),
)
await Bun.write(
path.join(storageDir, "part", "msg_realmsgid", "prt_from_filename.json"),
JSON.stringify({
id: "prt_different_in_json", // Stale! Should be ignored
messageID: "msg_different_in_json", // Stale! Should be ignored
sessionID: "ses_test456def",
type: "text",
text: "Hello",
}),
)
const stats = await JsonMigration.run(db)
expect(stats?.parts).toBe(1)
const parts = db.select().from(PartTable).all()
expect(parts.length).toBe(1)
expect(parts[0].id).toBe(PartID.make("prt_from_filename")) // Uses filename, not JSON id
expect(parts[0].message_id).toBe(MessageID.make("msg_realmsgid")) // Uses parent dir, not JSON messageID
})
test("skips orphaned sessions (no parent project)", async () => {
await Bun.write(
path.join(storageDir, "session", "proj_test123abc", "ses_orphan.json"),
JSON.stringify({
id: "ses_orphan",
projectID: "proj_nonexistent",
slug: "orphan",
directory: "/",
title: "Orphan",
version: "1.0.0",
time: { created: Date.now(), updated: Date.now() },
}),
)
const stats = await JsonMigration.run(db)
expect(stats?.sessions).toBe(0)
})
test("uses directory path for projectID when JSON has stale value", async () => {
// Simulates the scenario where earlier migration moved sessions to new
// git-based project directories but didn't update the projectID field
const gitBasedProjectID = "abc123gitcommit"
await writeProject(storageDir, {
id: gitBasedProjectID,
worktree: "/test/path",
vcs: "git",
time: { created: Date.now(), updated: Date.now() },
sandboxes: [],
})
// Session is in the git-based directory but JSON still has old projectID
await writeSession(storageDir, gitBasedProjectID, {
id: "ses_migrated",
projectID: "old-project-name", // Stale! Should be ignored
slug: "migrated-session",
directory: "/test/path",
title: "Migrated Session",
version: "1.0.0",
time: { created: 1700000000000, updated: 1700000001000 },
})
const stats = await JsonMigration.run(db)
expect(stats?.sessions).toBe(1)
const sessions = db.select().from(SessionTable).all()
expect(sessions.length).toBe(1)
expect(sessions[0].id).toBe(SessionID.make("ses_migrated"))
expect(sessions[0].project_id).toBe(ProjectV2.ID.make(gitBasedProjectID)) // Uses directory, not stale JSON
})
test("uses filename for session id when JSON has different value", async () => {
await writeProject(storageDir, {
id: "proj_test123abc",
worktree: "/test/path",
time: { created: Date.now(), updated: Date.now() },
sandboxes: [],
})
await Bun.write(
path.join(storageDir, "session", "proj_test123abc", "ses_from_filename.json"),
JSON.stringify({
id: "ses_different_in_json", // Stale! Should be ignored
projectID: "proj_test123abc",
slug: "test-session",
directory: "/test/path",
title: "Test Session",
version: "1.0.0",
time: { created: 1700000000000, updated: 1700000001000 },
}),
)
const stats = await JsonMigration.run(db)
expect(stats?.sessions).toBe(1)
const sessions = db.select().from(SessionTable).all()
expect(sessions.length).toBe(1)
expect(sessions[0].id).toBe(SessionID.make("ses_from_filename")) // Uses filename, not JSON id
expect(sessions[0].project_id).toBe(ProjectV2.ID.make("proj_test123abc"))
})
test("is idempotent (running twice doesn't duplicate)", async () => {
await writeProject(storageDir, {
id: "proj_test123abc",
worktree: "/",
time: { created: Date.now(), updated: Date.now() },
sandboxes: [],
})
await JsonMigration.run(db)
await JsonMigration.run(db)
const projects = db.select().from(ProjectTable).all()
expect(projects.length).toBe(1) // Still only 1 due to onConflictDoNothing
})
test("migrates todos", async () => {
await writeProject(storageDir, {
id: "proj_test123abc",
worktree: "/",
time: { created: Date.now(), updated: Date.now() },
sandboxes: [],
})
await writeSession(storageDir, "proj_test123abc", { ...fixtures.session })
// Create todo file (named by sessionID, contains array of todos)
await Bun.write(
path.join(storageDir, "todo", "ses_test456def.json"),
JSON.stringify([
{
id: "todo_1",
content: "First todo",
status: "pending",
priority: "high",
},
{
id: "todo_2",
content: "Second todo",
status: "completed",
priority: "medium",
},
]),
)
const stats = await JsonMigration.run(db)
expect(stats?.todos).toBe(2)
const todos = db.select().from(TodoTable).orderBy(TodoTable.position).all()
expect(todos.length).toBe(2)
expect(todos[0].content).toBe("First todo")
expect(todos[0].status).toBe("pending")
expect(todos[0].priority).toBe("high")
expect(todos[0].position).toBe(0)
expect(todos[1].content).toBe("Second todo")
expect(todos[1].position).toBe(1)
})
test("todos are ordered by position", async () => {
await writeProject(storageDir, {
id: "proj_test123abc",
worktree: "/",
time: { created: Date.now(), updated: Date.now() },
sandboxes: [],
})
await writeSession(storageDir, "proj_test123abc", { ...fixtures.session })
await Bun.write(
path.join(storageDir, "todo", "ses_test456def.json"),
JSON.stringify([
{ content: "Third", status: "pending", priority: "low" },
{ content: "First", status: "pending", priority: "high" },
{ content: "Second", status: "in_progress", priority: "medium" },
]),
)
await JsonMigration.run(db)
const todos = db.select().from(TodoTable).orderBy(TodoTable.position).all()
expect(todos.length).toBe(3)
expect(todos[0].content).toBe("Third")
expect(todos[0].position).toBe(0)
expect(todos[1].content).toBe("First")
expect(todos[1].position).toBe(1)
expect(todos[2].content).toBe("Second")
expect(todos[2].position).toBe(2)
})
test("does not migrate legacy permissions", async () => {
await writeProject(storageDir, {
id: "proj_test123abc",
worktree: "/",
time: { created: Date.now(), updated: Date.now() },
sandboxes: [],
})
// Create permission file (named by projectID, contains array of rules)
const permissionData = [
{ permission: "file.read", pattern: "/test/file1.ts", action: "allow" as const },
{ permission: "file.write", pattern: "/test/file2.ts", action: "ask" as const },
{ permission: "command.run", pattern: "npm install", action: "deny" as const },
]
await Bun.write(path.join(storageDir, "permission", "proj_test123abc.json"), JSON.stringify(permissionData))
const stats = await JsonMigration.run(db)
expect(stats?.permissions).toBe(0)
})
test("migrates session shares", async () => {
await writeProject(storageDir, {
id: "proj_test123abc",
worktree: "/",
time: { created: Date.now(), updated: Date.now() },
sandboxes: [],
})
await writeSession(storageDir, "proj_test123abc", { ...fixtures.session })
// Create session share file (named by sessionID)
await Bun.write(
path.join(storageDir, "session_share", "ses_test456def.json"),
JSON.stringify({
id: "share_123",
secret: "supersecretkey",
url: "https://share.example.com/ses_test456def",
}),
)
const stats = await JsonMigration.run(db)
expect(stats?.shares).toBe(1)
const shares = db.select().from(SessionShareTable).all()
expect(shares.length).toBe(1)
expect(shares[0].session_id).toBe("ses_test456def")
expect(shares[0].id).toBe("share_123")
expect(shares[0].secret).toBe("supersecretkey")
expect(shares[0].url).toBe("https://share.example.com/ses_test456def")
})
test("returns empty stats when storage directory does not exist", async () => {
await fs.rm(storageDir, { recursive: true, force: true })
const stats = await JsonMigration.run(db)
expect(stats.projects).toBe(0)
expect(stats.sessions).toBe(0)
expect(stats.messages).toBe(0)
expect(stats.parts).toBe(0)
expect(stats.todos).toBe(0)
expect(stats.permissions).toBe(0)
expect(stats.shares).toBe(0)
expect(stats.errors).toEqual([])
})
test("continues when a JSON file is unreadable and records an error", async () => {
await writeProject(storageDir, {
id: "proj_test123abc",
worktree: "/",
time: { created: Date.now(), updated: Date.now() },
sandboxes: [],
})
await Bun.write(path.join(storageDir, "project", "broken.json"), "{ invalid json")
const stats = await JsonMigration.run(db)
expect(stats.projects).toBe(1)
expect(stats.errors.some((x) => x.includes("failed to read") && x.includes("broken.json"))).toBe(true)
const projects = db.select().from(ProjectTable).all()
expect(projects.length).toBe(1)
expect(projects[0].id).toBe(ProjectV2.ID.make("proj_test123abc"))
})
test("skips invalid todo entries while preserving source positions", async () => {
await writeProject(storageDir, {
id: "proj_test123abc",
worktree: "/",
time: { created: Date.now(), updated: Date.now() },
sandboxes: [],
})
await writeSession(storageDir, "proj_test123abc", { ...fixtures.session })
await Bun.write(
path.join(storageDir, "todo", "ses_test456def.json"),
JSON.stringify([
{ content: "keep-0", status: "pending", priority: "high" },
{ content: "drop-1", priority: "low" },
{ content: "keep-2", status: "completed", priority: "medium" },
]),
)
const stats = await JsonMigration.run(db)
expect(stats.todos).toBe(2)
const todos = db.select().from(TodoTable).orderBy(TodoTable.position).all()
expect(todos.length).toBe(2)
expect(todos[0].content).toBe("keep-0")
expect(todos[0].position).toBe(0)
expect(todos[1].content).toBe("keep-2")
expect(todos[1].position).toBe(2)
})
test("skips orphaned todos and shares", async () => {
await writeProject(storageDir, {
id: "proj_test123abc",
worktree: "/",
time: { created: Date.now(), updated: Date.now() },
sandboxes: [],
})
await writeSession(storageDir, "proj_test123abc", { ...fixtures.session })
await Bun.write(
path.join(storageDir, "todo", "ses_test456def.json"),
JSON.stringify([{ content: "valid", status: "pending", priority: "high" }]),
)
await Bun.write(
path.join(storageDir, "todo", "ses_missing.json"),
JSON.stringify([{ content: "orphan", status: "pending", priority: "high" }]),
)
await Bun.write(
path.join(storageDir, "permission", "proj_test123abc.json"),
JSON.stringify([{ permission: "file.read" }]),
)
await Bun.write(
path.join(storageDir, "permission", "proj_missing.json"),
JSON.stringify([{ permission: "file.write" }]),
)
await Bun.write(
path.join(storageDir, "session_share", "ses_test456def.json"),
JSON.stringify({ id: "share_ok", secret: "secret", url: "https://ok.example.com" }),
)
await Bun.write(
path.join(storageDir, "session_share", "ses_missing.json"),
JSON.stringify({ id: "share_missing", secret: "secret", url: "https://missing.example.com" }),
)
const stats = await JsonMigration.run(db)
expect(stats.todos).toBe(1)
expect(stats.permissions).toBe(0)
expect(stats.shares).toBe(1)
expect(db.select().from(TodoTable).all().length).toBe(1)
expect(db.select().from(SessionShareTable).all().length).toBe(1)
})
test("handles mixed corruption and partial validity in one migration run", async () => {
await writeProject(storageDir, {
id: "proj_test123abc",
worktree: "/ok",
time: { created: 1700000000000, updated: 1700000001000 },
sandboxes: [],
})
await Bun.write(
path.join(storageDir, "project", "proj_missing_id.json"),
JSON.stringify({ worktree: "/bad", sandboxes: [] }),
)
await Bun.write(path.join(storageDir, "project", "proj_broken.json"), "{ nope")
await writeSession(storageDir, "proj_test123abc", {
id: "ses_test456def",
projectID: "proj_test123abc",
slug: "ok",
directory: "/ok",
title: "Ok",
version: "1",
time: { created: 1700000000000, updated: 1700000001000 },
})
await Bun.write(
path.join(storageDir, "session", "proj_test123abc", "ses_missing_project.json"),
JSON.stringify({
id: "ses_missing_project",
slug: "bad",
directory: "/bad",
title: "Bad",
version: "1",
}),
)
await Bun.write(
path.join(storageDir, "session", "proj_test123abc", "ses_orphan.json"),
JSON.stringify({
id: "ses_orphan",
projectID: "proj_missing",
slug: "orphan",
directory: "/bad",
title: "Orphan",
version: "1",
}),
)
await Bun.write(
path.join(storageDir, "message", "ses_test456def", "msg_ok.json"),
JSON.stringify({ role: "user", time: { created: 1700000000000 } }),
)
await Bun.write(path.join(storageDir, "message", "ses_test456def", "msg_broken.json"), "{ nope")
await Bun.write(
path.join(storageDir, "message", "ses_missing", "msg_orphan.json"),
JSON.stringify({ role: "user", time: { created: 1700000000000 } }),
)
await Bun.write(
path.join(storageDir, "part", "msg_ok", "part_ok.json"),
JSON.stringify({ type: "text", text: "ok" }),
)
await Bun.write(
path.join(storageDir, "part", "msg_missing", "part_missing_message.json"),
JSON.stringify({ type: "text", text: "bad" }),
)
await Bun.write(path.join(storageDir, "part", "msg_ok", "part_broken.json"), "{ nope")
await Bun.write(
path.join(storageDir, "todo", "ses_test456def.json"),
JSON.stringify([
{ content: "ok", status: "pending", priority: "high" },
{ content: "skip", status: "pending" },
]),
)
await Bun.write(
path.join(storageDir, "todo", "ses_missing.json"),
JSON.stringify([{ content: "orphan", status: "pending", priority: "high" }]),
)
await Bun.write(path.join(storageDir, "todo", "ses_broken.json"), "{ nope")
await Bun.write(
path.join(storageDir, "permission", "proj_test123abc.json"),
JSON.stringify([{ permission: "file.read" }]),
)
await Bun.write(
path.join(storageDir, "permission", "proj_missing.json"),
JSON.stringify([{ permission: "file.write" }]),
)
await Bun.write(path.join(storageDir, "permission", "proj_broken.json"), "{ nope")
await Bun.write(
path.join(storageDir, "session_share", "ses_test456def.json"),
JSON.stringify({ id: "share_ok", secret: "secret", url: "https://ok.example.com" }),
)
await Bun.write(
path.join(storageDir, "session_share", "ses_missing.json"),
JSON.stringify({ id: "share_orphan", secret: "secret", url: "https://missing.example.com" }),
)
await Bun.write(path.join(storageDir, "session_share", "ses_broken.json"), "{ nope")
const stats = await JsonMigration.run(db)
// Projects: proj_test123abc (valid), proj_missing_id (now derives id from filename)
// Sessions: ses_test456def (valid), ses_missing_project (now uses dir path),
// ses_orphan (now uses dir path, ignores stale projectID)
expect(stats.projects).toBe(2)
expect(stats.sessions).toBe(3)
expect(stats.messages).toBe(1)
expect(stats.parts).toBe(1)
expect(stats.todos).toBe(1)
expect(stats.permissions).toBe(0)
expect(stats.shares).toBe(1)
expect(stats.errors.length).toBeGreaterThanOrEqual(6)
expect(db.select().from(ProjectTable).all().length).toBe(2)
expect(db.select().from(SessionTable).all().length).toBe(3)
expect(db.select().from(MessageTable).all().length).toBe(1)
expect(db.select().from(PartTable).all().length).toBe(1)
expect(db.select().from(TodoTable).all().length).toBe(1)
expect(db.select().from(SessionShareTable).all().length).toBe(1)
})
})