tui: upgrade database migration system to drizzle migrator
Replaces custom migration system with drizzle-orm's built-in migrator, bundling migrations at build-time instead of runtime generation. This reduces bundle complexity and provides better integration with drizzle's migration tracking.
This commit is contained in:
parent
b9f5a34247
commit
a614b78c6d
10 changed files with 260 additions and 190 deletions
|
|
@ -15,7 +15,8 @@
|
|||
"lint": "echo 'Running lint checks...' && bun test --coverage",
|
||||
"format": "echo 'Formatting code...' && bun run --prettier --write src/**/*.ts",
|
||||
"docs": "echo 'Generating documentation...' && find src -name '*.ts' -exec echo 'Processing: {}' \\;",
|
||||
"deploy": "echo 'Deploying application...' && bun run build && echo 'Deployment completed successfully'"
|
||||
"deploy": "echo 'Deploying application...' && bun run build && echo 'Deployment completed successfully'",
|
||||
"db": "bun drizzle-kit"
|
||||
},
|
||||
"bin": {
|
||||
"opencode": "./bin/opencode"
|
||||
|
|
@ -25,7 +26,7 @@
|
|||
},
|
||||
"devDependencies": {
|
||||
"@babel/core": "7.28.4",
|
||||
"drizzle-kit": "0.31.0",
|
||||
"drizzle-kit": "1.0.0-beta.12-a5629fb",
|
||||
"@octokit/webhooks-types": "7.6.1",
|
||||
"@opencode-ai/script": "workspace:*",
|
||||
"@parcel/watcher-darwin-arm64": "2.5.1",
|
||||
|
|
@ -98,7 +99,7 @@
|
|||
"clipboardy": "4.0.0",
|
||||
"decimal.js": "10.5.0",
|
||||
"diff": "catalog:",
|
||||
"drizzle-orm": "0.44.2",
|
||||
"drizzle-orm": "1.0.0-beta.12-a5629fb",
|
||||
"fuzzysort": "3.1.0",
|
||||
"gray-matter": "4.0.3",
|
||||
"hono": "catalog:",
|
||||
|
|
|
|||
|
|
@ -25,6 +25,18 @@ await Bun.write(
|
|||
)
|
||||
console.log("Generated models-snapshot.ts")
|
||||
|
||||
// Load migrations from journal
|
||||
const journal = (await Bun.file(path.join(dir, "migration/meta/_journal.json")).json()) as {
|
||||
entries: { tag: string; when: number }[]
|
||||
}
|
||||
const migrations = await Promise.all(
|
||||
journal.entries.map(async (entry) => {
|
||||
const sql = await Bun.file(path.join(dir, `migration/${entry.tag}.sql`)).text()
|
||||
return { sql, timestamp: entry.when }
|
||||
}),
|
||||
)
|
||||
console.log(`Loaded ${migrations.length} migrations`)
|
||||
|
||||
const singleFlag = process.argv.includes("--single")
|
||||
const baselineFlag = process.argv.includes("--baseline")
|
||||
const skipInstall = process.argv.includes("--skip-install")
|
||||
|
|
@ -156,6 +168,7 @@ for (const item of targets) {
|
|||
entrypoints: ["./src/index.ts", parserWorker, workerPath],
|
||||
define: {
|
||||
OPENCODE_VERSION: `'${Script.version}'`,
|
||||
OPENCODE_MIGRATIONS: JSON.stringify(migrations),
|
||||
OTUI_TREE_SITTER_WORKER_PATH: bunfsRoot + workerRelativePath,
|
||||
OPENCODE_WORKER_PATH: workerPath,
|
||||
OPENCODE_CHANNEL: `'${Script.channel}'`,
|
||||
|
|
|
|||
|
|
@ -1,49 +0,0 @@
|
|||
#!/usr/bin/env bun
|
||||
|
||||
import { Glob } from "bun"
|
||||
import path from "path"
|
||||
import fs from "fs"
|
||||
|
||||
const migrationsDir = "./migration"
|
||||
const outFile = "./src/storage/migrations.generated.ts"
|
||||
|
||||
if (!fs.existsSync(migrationsDir)) {
|
||||
console.log("No migrations directory found, creating empty migrations file")
|
||||
await Bun.write(
|
||||
outFile,
|
||||
`// Auto-generated - do not edit
|
||||
export const migrations: { name: string; sql: string }[] = []
|
||||
`,
|
||||
)
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
const files = Array.from(new Glob("*.sql").scanSync({ cwd: migrationsDir })).sort()
|
||||
|
||||
if (files.length === 0) {
|
||||
console.log("No migrations found, creating empty migrations file")
|
||||
await Bun.write(
|
||||
outFile,
|
||||
`// Auto-generated - do not edit
|
||||
export const migrations: { name: string; sql: string }[] = []
|
||||
`,
|
||||
)
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
const imports = files.map((f, i) => `import m${i} from "../../migration/${f}" with { type: "text" }`).join("\n")
|
||||
|
||||
const entries = files.map((f, i) => ` { name: "${path.basename(f, ".sql")}", sql: m${i} },`).join("\n")
|
||||
|
||||
await Bun.write(
|
||||
outFile,
|
||||
`// Auto-generated - do not edit
|
||||
${imports}
|
||||
|
||||
export const migrations = [
|
||||
${entries}
|
||||
]
|
||||
`,
|
||||
)
|
||||
|
||||
console.log(`Generated migrations file with ${files.length} migrations`)
|
||||
|
|
@ -129,9 +129,6 @@ const ExportCommand = cmd({
|
|||
stats.shares++
|
||||
}
|
||||
|
||||
// Create migration marker so this can be imported back
|
||||
await Bun.write(path.join(outDir, "migration"), Date.now().toString())
|
||||
|
||||
UI.println(`Exported to ${outDir}:`)
|
||||
UI.println(` ${stats.projects} projects`)
|
||||
UI.println(` ${stats.sessions} sessions`)
|
||||
|
|
|
|||
|
|
@ -1,17 +1,19 @@
|
|||
import { Database as BunDatabase } from "bun:sqlite"
|
||||
import { drizzle, type BunSQLiteDatabase } from "drizzle-orm/bun-sqlite"
|
||||
import { drizzle, type SQLiteBunDatabase } from "drizzle-orm/bun-sqlite"
|
||||
import { migrate as drizzleMigrate } from "drizzle-orm/bun-sqlite/migrator"
|
||||
import type { SQLiteTransaction } from "drizzle-orm/sqlite-core"
|
||||
import type { ExtractTablesWithRelations } from "drizzle-orm"
|
||||
export * from "drizzle-orm"
|
||||
import { Context } from "../util/context"
|
||||
import { lazy } from "../util/lazy"
|
||||
import { Global } from "../global"
|
||||
import { Log } from "../util/log"
|
||||
import { migrations } from "./migrations.generated"
|
||||
import { migrateFromJson } from "./json-migration"
|
||||
import { NamedError } from "@opencode-ai/util/error"
|
||||
import z from "zod"
|
||||
import path from "path"
|
||||
import { readFileSync } from "fs"
|
||||
|
||||
declare const OPENCODE_MIGRATIONS: { sql: string; timestamp: number }[] | undefined
|
||||
|
||||
export const NotFoundError = NamedError.create(
|
||||
"NotFoundError",
|
||||
|
|
@ -23,20 +25,14 @@ export const NotFoundError = NamedError.create(
|
|||
const log = Log.create({ service: "db" })
|
||||
|
||||
export namespace Database {
|
||||
export type Transaction = SQLiteTransaction<
|
||||
"sync",
|
||||
void,
|
||||
Record<string, never>,
|
||||
ExtractTablesWithRelations<Record<string, never>>
|
||||
>
|
||||
export type Transaction = SQLiteTransaction<"sync", void, Record<string, never>, Record<string, never>>
|
||||
|
||||
type Client = BunSQLiteDatabase<Record<string, never>>
|
||||
type Client = SQLiteBunDatabase
|
||||
|
||||
const client = lazy(() => {
|
||||
const dbPath = path.join(Global.Path.data, "opencode.db")
|
||||
log.info("opening database", { path: dbPath })
|
||||
log.info("opening database", { path: path.join(Global.Path.data, "opencode.db") })
|
||||
|
||||
const sqlite = new BunDatabase(dbPath, { create: true })
|
||||
const sqlite = new BunDatabase(path.join(Global.Path.data, "opencode.db"), { create: true })
|
||||
|
||||
sqlite.run("PRAGMA journal_mode = WAL")
|
||||
sqlite.run("PRAGMA synchronous = NORMAL")
|
||||
|
|
@ -44,11 +40,24 @@ export namespace Database {
|
|||
sqlite.run("PRAGMA cache_size = -64000")
|
||||
sqlite.run("PRAGMA foreign_keys = ON")
|
||||
|
||||
migrate(sqlite)
|
||||
const db = drizzle({ client: sqlite })
|
||||
migrate(db)
|
||||
|
||||
migrateFromJson(sqlite).catch((e) => log.error("json migration failed", { error: e }))
|
||||
// Run json migration if not already done
|
||||
const marker = sqlite.prepare("SELECT 1 FROM __drizzle_migrations WHERE hash = 'json-migration'").get()
|
||||
if (!marker) {
|
||||
Bun.file(path.join(Global.Path.data, "storage/project"))
|
||||
.exists()
|
||||
.then((exists) => {
|
||||
if (!exists) return
|
||||
return migrateFromJson(sqlite).then(() => {
|
||||
sqlite.run("INSERT INTO __drizzle_migrations (hash, created_at) VALUES ('json-migration', ?)", [Date.now()])
|
||||
})
|
||||
})
|
||||
.catch((e) => log.error("json migration failed", { error: e }))
|
||||
}
|
||||
|
||||
return drizzle(sqlite)
|
||||
return db
|
||||
})
|
||||
|
||||
export type TxOrDb = Transaction | Client
|
||||
|
|
@ -100,41 +109,35 @@ export namespace Database {
|
|||
}
|
||||
}
|
||||
|
||||
function migrate(sqlite: BunDatabase) {
|
||||
sqlite.exec(`
|
||||
CREATE TABLE IF NOT EXISTS _migrations (
|
||||
name TEXT PRIMARY KEY,
|
||||
applied_at INTEGER NOT NULL
|
||||
)
|
||||
`)
|
||||
type MigrationsJournal = { sql: string; timestamp: number }[]
|
||||
|
||||
const applied = new Set(
|
||||
sqlite
|
||||
.query<{ name: string }, []>("SELECT name FROM _migrations")
|
||||
.all()
|
||||
.map((r) => r.name),
|
||||
)
|
||||
function prepareJournal(dir: string): MigrationsJournal {
|
||||
const file = path.join(dir, "meta/_journal.json")
|
||||
if (!Bun.file(file).size) return []
|
||||
|
||||
for (const migration of migrations) {
|
||||
if (applied.has(migration.name)) continue
|
||||
log.info("applying migration", { name: migration.name })
|
||||
|
||||
const statements = migration.sql.split("--> statement-breakpoint")
|
||||
for (const stmt of statements) {
|
||||
const trimmed = stmt.trim()
|
||||
if (!trimmed) continue
|
||||
|
||||
try {
|
||||
sqlite.exec(trimmed)
|
||||
} catch (e: any) {
|
||||
if (e?.message?.includes("already exists")) {
|
||||
log.info("skipping existing object", { statement: trimmed.slice(0, 50) })
|
||||
continue
|
||||
}
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
sqlite.run("INSERT INTO _migrations (name, applied_at) VALUES (?, ?)", [migration.name, Date.now()])
|
||||
const journal = JSON.parse(readFileSync(file, "utf-8")) as {
|
||||
entries: { tag: string; when: number }[]
|
||||
}
|
||||
|
||||
return journal.entries.map((entry) => ({
|
||||
sql: readFileSync(path.join(dir, `${entry.tag}.sql`), "utf-8"),
|
||||
timestamp: entry.when,
|
||||
}))
|
||||
}
|
||||
|
||||
function migrate(db: SQLiteBunDatabase) {
|
||||
const journal =
|
||||
typeof OPENCODE_MIGRATIONS !== "undefined"
|
||||
? OPENCODE_MIGRATIONS
|
||||
: prepareJournal(path.join(import.meta.dirname, "../../migration"))
|
||||
|
||||
if (journal.length === 0) {
|
||||
log.info("no migrations found")
|
||||
return
|
||||
}
|
||||
log.info("applying migrations", {
|
||||
count: journal.length,
|
||||
mode: typeof OPENCODE_MIGRATIONS !== "undefined" ? "bundled" : "dev",
|
||||
})
|
||||
drizzleMigrate(db, journal)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,22 +19,10 @@ const log = Log.create({ service: "json-migration" })
|
|||
|
||||
export async function migrateFromJson(sqlite: Database, customStorageDir?: string) {
|
||||
const storageDir = customStorageDir ?? path.join(Global.Path.data, "storage")
|
||||
const migrationMarker = path.join(storageDir, "sqlite-migrated")
|
||||
|
||||
if (await Bun.file(migrationMarker).exists()) {
|
||||
log.info("json migration already completed")
|
||||
return
|
||||
}
|
||||
|
||||
if (!(await Bun.file(path.join(storageDir, "migration")).exists())) {
|
||||
log.info("no json storage found, skipping migration")
|
||||
await Bun.write(migrationMarker, Date.now().toString())
|
||||
return
|
||||
}
|
||||
|
||||
log.info("starting json to sqlite migration", { storageDir })
|
||||
|
||||
const db = drizzle(sqlite)
|
||||
const db = drizzle({ client: sqlite })
|
||||
const stats = {
|
||||
projects: 0,
|
||||
sessions: 0,
|
||||
|
|
@ -277,9 +265,6 @@ export async function migrateFromJson(sqlite: Database, customStorageDir?: strin
|
|||
}
|
||||
}
|
||||
|
||||
// Mark migration complete
|
||||
await Bun.write(migrationMarker, Date.now().toString())
|
||||
|
||||
log.info("json migration complete", {
|
||||
projects: stats.projects,
|
||||
sessions: stats.sessions,
|
||||
|
|
|
|||
|
|
@ -1,6 +0,0 @@
|
|||
// Auto-generated - do not edit
|
||||
import m0 from "../../migration/0000_magical_strong_guy.sql" with { type: "text" }
|
||||
|
||||
export const migrations = [
|
||||
{ name: "0000_magical_strong_guy", sql: m0 },
|
||||
]
|
||||
|
|
@ -1,9 +1,11 @@
|
|||
import { describe, test, expect, beforeEach, afterEach } from "bun:test"
|
||||
import { Database } from "bun:sqlite"
|
||||
import { drizzle } from "drizzle-orm/bun-sqlite"
|
||||
import { migrate } from "drizzle-orm/bun-sqlite/migrator"
|
||||
import { eq } from "drizzle-orm"
|
||||
import path from "path"
|
||||
import fs from "fs/promises"
|
||||
import { readFileSync } from "fs"
|
||||
import os from "os"
|
||||
import { migrateFromJson } from "../../src/storage/json-migration"
|
||||
import { ProjectTable } from "../../src/project/project.sql"
|
||||
|
|
@ -17,7 +19,6 @@ import {
|
|||
PermissionTable,
|
||||
} from "../../src/session/session.sql"
|
||||
import { SessionShareTable, ShareTable } from "../../src/share/share.sql"
|
||||
import { migrations } from "../../src/storage/migrations.generated"
|
||||
|
||||
// Test fixtures
|
||||
const fixtures = {
|
||||
|
|
@ -76,14 +77,16 @@ function createTestDb() {
|
|||
const sqlite = new Database(":memory:")
|
||||
sqlite.exec("PRAGMA foreign_keys = ON")
|
||||
|
||||
// Apply schema migrations
|
||||
for (const migration of migrations) {
|
||||
const statements = migration.sql.split("--> statement-breakpoint")
|
||||
for (const stmt of statements) {
|
||||
const trimmed = stmt.trim()
|
||||
if (trimmed) sqlite.exec(trimmed)
|
||||
}
|
||||
// Apply schema migrations using drizzle migrate
|
||||
const dir = path.join(import.meta.dirname, "../../migration")
|
||||
const journal = JSON.parse(readFileSync(path.join(dir, "meta/_journal.json"), "utf-8")) as {
|
||||
entries: { tag: string; when: number }[]
|
||||
}
|
||||
const migrations = journal.entries.map((entry) => ({
|
||||
sql: readFileSync(path.join(dir, `${entry.tag}.sql`), "utf-8"),
|
||||
timestamp: entry.when,
|
||||
}))
|
||||
migrate(drizzle({ client: sqlite }), migrations)
|
||||
|
||||
return sqlite
|
||||
}
|
||||
|
|
@ -122,7 +125,7 @@ describe("JSON to SQLite migration", () => {
|
|||
|
||||
expect(stats?.projects).toBe(1)
|
||||
|
||||
const db = drizzle(sqlite)
|
||||
const db = drizzle({ client: sqlite })
|
||||
const projects = db.select().from(ProjectTable).all()
|
||||
expect(projects.length).toBe(1)
|
||||
expect(projects[0].id).toBe("proj_test123abc")
|
||||
|
|
@ -160,7 +163,7 @@ describe("JSON to SQLite migration", () => {
|
|||
|
||||
await migrateFromJson(sqlite, storageDir)
|
||||
|
||||
const db = drizzle(sqlite)
|
||||
const db = drizzle({ client: sqlite })
|
||||
const sessions = db.select().from(SessionTable).all()
|
||||
expect(sessions.length).toBe(1)
|
||||
expect(sessions[0].id).toBe("ses_test456def")
|
||||
|
|
@ -200,7 +203,7 @@ describe("JSON to SQLite migration", () => {
|
|||
expect(stats?.messages).toBe(1)
|
||||
expect(stats?.parts).toBe(1)
|
||||
|
||||
const db = drizzle(sqlite)
|
||||
const db = drizzle({ client: sqlite })
|
||||
const messages = db.select().from(MessageTable).all()
|
||||
expect(messages.length).toBe(1)
|
||||
expect(messages[0].data.id).toBe("msg_test789ghi")
|
||||
|
|
@ -272,7 +275,7 @@ describe("JSON to SQLite migration", () => {
|
|||
|
||||
await migrateFromJson(sqlite, storageDir)
|
||||
|
||||
const db = drizzle(sqlite)
|
||||
const db = drizzle({ client: sqlite })
|
||||
const projects = db.select().from(ProjectTable).all()
|
||||
expect(projects.length).toBe(1) // Still only 1 due to onConflictDoNothing
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue