ceos/opencode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: extract shared util package (#37828)

Browse source
This commit is contained in:
Dax 2026-07-21 10:34:29 -04:00 • committed by GitHub
commit e0810753f2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
272 changed files with 590 additions and 565 deletions
Download patch file Download diff file Expand all files Collapse all files

312
packages/core/src/util/effect-flock.ts
View file

@ -1,312 +0,0 @@
import path from "path"
import os from "os"
import { randomUUID } from "crypto"
import { Context, Effect, Function, Layer, Option, Schedule, Schema } from "effect"
import type { FileSystem, Scope } from "effect"
import type { PlatformError } from "effect/PlatformError"
import { FSUtil } from "../fs-util"
import { Global } from "../global"
import { makeGlobalNode } from "../effect/app-node"
import { Hash } from "./hash"
export namespace EffectFlock {
// ---------------------------------------------------------------------------
// Errors
// ---------------------------------------------------------------------------
export class LockTimeoutError extends Schema.TaggedErrorClass<LockTimeoutError>()("LockTimeoutError", {
key: Schema.String,
}) {}
export class LockCompromisedError extends Schema.TaggedErrorClass<LockCompromisedError>()("LockCompromisedError", {
detail: Schema.String,
}) {}
class ReleaseError extends Schema.TaggedErrorClass<ReleaseError>()("ReleaseError", {
detail: Schema.String,
cause: Schema.optional(Schema.Defect()),
}) {
override get message() {
return this.detail
}
}
/** Internal: signals "lock is held, retry later". Never leaks to callers. */
class NotAcquired extends Schema.TaggedErrorClass<NotAcquired>()("NotAcquired", {}) {}
export type LockError = LockTimeoutError | LockCompromisedError
export interface Options {
readonly staleMs?: number
readonly timeoutMs?: number
}
// ---------------------------------------------------------------------------
// Timing defaults
// ---------------------------------------------------------------------------
const DEFAULT_STALE_MS = 60_000
const DEFAULT_TIMEOUT_MS = 5 * 60_000
const BASE_DELAY_MS = 100
const MAX_DELAY_MS = 2_000
const retrySchedule = (timeoutMs: number) =>
Schedule.min([
Schedule.exponential(BASE_DELAY_MS, 1.7),
Schedule.spaced(Math.min(MAX_DELAY_MS, Math.max(BASE_DELAY_MS, Math.floor(timeoutMs / 10)))),
]).pipe(
Schedule.jittered,
Schedule.while((meta) => meta.elapsed < timeoutMs),
)
// ---------------------------------------------------------------------------
// Lock metadata schema
// ---------------------------------------------------------------------------
const LockMetaJson = Schema.fromJsonString(
Schema.Struct({
token: Schema.String,
pid: Schema.Number,
hostname: Schema.String,
createdAt: Schema.String,
}),
)
const decodeMeta = Schema.decodeUnknownSync(LockMetaJson)
const encodeMeta = Schema.encodeSync(LockMetaJson)
// ---------------------------------------------------------------------------
// Service
// ---------------------------------------------------------------------------
export interface Interface {
readonly acquire: (key: string, dir?: string, options?: Options) => Effect.Effect<void, LockError, Scope.Scope>
readonly withLock: {
(key: string, dir?: string): <A, E, R>(body: Effect.Effect<A, E, R>) => Effect.Effect<A, E | LockError, R>
<A, E, R>(body: Effect.Effect<A, E, R>, key: string, dir?: string): Effect.Effect<A, E | LockError, R>
}
}
export class Service extends Context.Service<Service, Interface>()("EffectFlock") {}
// ---------------------------------------------------------------------------
// Layer
// ---------------------------------------------------------------------------
function wall() {
return performance.timeOrigin + performance.now()
}
const mtimeMs = (info: FileSystem.File.Info) => Option.getOrElse(info.mtime, () => new Date(0)).getTime()
const isPathGone = (e: PlatformError) => e.reason._tag === "NotFound" || e.reason._tag === "Unknown"
const layer: Layer.Layer<Service, never, Global.Service | FSUtil.Service> = Layer.effect(
Service,
Effect.gen(function* () {
const global = yield* Global.Service
const fs = yield* FSUtil.Service
const lockRoot = path.join(global.state, "locks")
const hostname = os.hostname()
const ensuredDirs = new Set<string>()
// -- helpers (close over fs) --
const safeStat = (file: string) =>
fs.stat(file).pipe(
Effect.catchIf(isPathGone, () => Effect.void),
Effect.orDie,
)
const forceRemove = (target: string) => fs.remove(target, { recursive: true }).pipe(Effect.ignore)
/** Atomic mkdir — returns true if created, false if already exists, dies on other errors. */
const atomicMkdir = (dir: string) =>
fs.makeDirectory(dir, { mode: 0o700 }).pipe(
Effect.as(true),
Effect.catchIf(
(e) => e.reason._tag === "AlreadyExists",
() => Effect.succeed(false),
),
Effect.orDie,
)
/** Write with exclusive create — compromised error if file already exists. */
const exclusiveWrite = (filePath: string, content: string, lockDir: string, detail: string) =>
fs.writeFileString(filePath, content, { flag: "wx" }).pipe(
Effect.catch(() =>
Effect.gen(function* () {
yield* forceRemove(lockDir)
return yield* new LockCompromisedError({ detail })
}),
),
)
const cleanStaleBreaker = Effect.fnUntraced(function* (breakerPath: string, staleMs: number) {
const bs = yield* safeStat(breakerPath)
if (bs && wall() - mtimeMs(bs) > staleMs) yield* forceRemove(breakerPath)
return false
})
const ensureDir = Effect.fnUntraced(function* (dir: string) {
if (ensuredDirs.has(dir)) return
yield* fs.makeDirectory(dir, { recursive: true }).pipe(Effect.orDie)
ensuredDirs.add(dir)
})
const isStale = Effect.fnUntraced(function* (
lockDir: string,
heartbeatPath: string,
metaPath: string,
staleMs: number,
) {
const now = wall()
const hb = yield* safeStat(heartbeatPath)
if (hb) return now - mtimeMs(hb) > staleMs
const meta = yield* safeStat(metaPath)
if (meta) return now - mtimeMs(meta) > staleMs
const dir = yield* safeStat(lockDir)
if (!dir) return false
return now - mtimeMs(dir) > staleMs
})
// -- single lock attempt --
type Handle = { token: string; metaPath: string; heartbeatPath: string; lockDir: string }
const tryAcquireLockDir = (lockDir: string, key: string, staleMs: number) =>
Effect.gen(function* () {
const token = randomUUID()
const metaPath = path.join(lockDir, "meta.json")
const heartbeatPath = path.join(lockDir, "heartbeat")
// Atomic mkdir — the POSIX lock primitive
const created = yield* atomicMkdir(lockDir)
if (!created) {
if (!(yield* isStale(lockDir, heartbeatPath, metaPath, staleMs))) return yield* new NotAcquired()
// Stale — race for breaker ownership
const breakerPath = lockDir + ".breaker"
const claimed = yield* fs.makeDirectory(breakerPath, { mode: 0o700 }).pipe(
Effect.as(true),
Effect.catchIf(
(e) => e.reason._tag === "AlreadyExists",
() => cleanStaleBreaker(breakerPath, staleMs),
),
Effect.catchIf(isPathGone, () => Effect.succeed(false)),
Effect.orDie,
)
if (!claimed) return yield* new NotAcquired()
// We own the breaker — double-check staleness, nuke, recreate
const recreated = yield* Effect.gen(function* () {
if (!(yield* isStale(lockDir, heartbeatPath, metaPath, staleMs))) return false
yield* forceRemove(lockDir)
return yield* atomicMkdir(lockDir)
}).pipe(Effect.ensuring(forceRemove(breakerPath)))
if (!recreated) return yield* new NotAcquired()
}
// We own the lock dir — write heartbeat + meta with exclusive create
yield* exclusiveWrite(heartbeatPath, "", lockDir, "heartbeat already existed")
const metaJson = encodeMeta({ token, pid: process.pid, hostname, createdAt: new Date().toISOString() })
yield* exclusiveWrite(metaPath, metaJson, lockDir, "meta.json already existed")
return { token, metaPath, heartbeatPath, lockDir } satisfies Handle
}).pipe(
Effect.withSpan("EffectFlock.tryAcquire", {
attributes: { key },
}),
)
// -- retry wrapper (preserves Handle type) --
const acquireHandle = (
lockfile: string,
key: string,
options: { staleMs: number; timeoutMs: number },
): Effect.Effect<Handle, LockError> =>
tryAcquireLockDir(lockfile, key, options.staleMs).pipe(
Effect.retry({
while: (err) => err._tag === "NotAcquired",
schedule: retrySchedule(options.timeoutMs),
}),
Effect.catchTag("NotAcquired", () => Effect.fail(new LockTimeoutError({ key }))),
Effect.timeoutOrElse({
duration: options.timeoutMs,
orElse: () => Effect.fail(new LockTimeoutError({ key })),
}),
)
// -- release --
const release = (handle: Handle) =>
Effect.gen(function* () {
const raw = yield* fs.readFileString(handle.metaPath).pipe(
Effect.catch((err) => {
if (isPathGone(err)) return Effect.die(new ReleaseError({ detail: "metadata missing" }))
return Effect.die(err)
}),
)
const parsed = yield* Effect.try({
try: () => decodeMeta(raw),
catch: (cause) => new ReleaseError({ detail: "metadata invalid", cause }),
}).pipe(Effect.orDie)
if (parsed.token !== handle.token) return yield* Effect.die(new ReleaseError({ detail: "token mismatch" }))
yield* forceRemove(handle.lockDir)
})
// -- build service --
const acquire = Effect.fn("EffectFlock.acquire")(function* (key: string, dir?: string, options: Options = {}) {
const lockDir = dir ?? lockRoot
const staleMs = options.staleMs ?? DEFAULT_STALE_MS
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS
yield* ensureDir(lockDir)
const lockfile = path.join(lockDir, Hash.fast(key) + ".lock")
// acquireRelease: acquire is uninterruptible, release is guaranteed
const handle = yield* Effect.acquireRelease(acquireHandle(lockfile, key, { staleMs, timeoutMs }), (handle) =>
release(handle),
)
// Heartbeat fiber — scoped, so it's interrupted before release runs
yield* fs
.utimes(handle.heartbeatPath, new Date(), new Date())
.pipe(
Effect.ignore,
Effect.repeat(Schedule.spaced(Math.max(100, Math.floor(staleMs / 3)))),
Effect.forkScoped,
)
})
const withLock: Interface["withLock"] = Function.dual(
(args) => Effect.isEffect(args[0]),
<A, E, R>(body: Effect.Effect<A, E, R>, key: string, dir?: string): Effect.Effect<A, E | LockError, R> =>
Effect.scoped(
Effect.gen(function* () {
yield* acquire(key, dir)
return yield* body
}),
),
)
return Service.of({ acquire, withLock })
}),
)
export const node = makeGlobalNode({ service: Service, layer: layer, deps: [Global.node, FSUtil.node] })
}

358
packages/core/src/util/flock.ts
View file

@ -1,358 +0,0 @@
import path from "path"
import os from "os"
import { randomBytes, randomUUID } from "crypto"
import { mkdir, readFile, rm, stat, utimes, writeFile } from "fs/promises"
import { Hash } from "./hash"
import { Effect } from "effect"
export type FlockGlobal = {
state: string
}
export namespace Flock {
let global: FlockGlobal | undefined
export function setGlobal(g: FlockGlobal) {
global = g
}
const root = () => {
if (!global) throw new Error("Flock global not set")
return path.join(global.state, "locks")
}
// Defaults for callers that do not provide timing options.
const defaultOpts = {
staleMs: 60_000,
timeoutMs: 5 * 60_000,
baseDelayMs: 100,
maxDelayMs: 2_000,
}
export interface WaitEvent {
key: string
attempt: number
delay: number
waited: number
}
export type Wait = (input: WaitEvent) => void | Promise<void>
export interface Options {
dir?: string
signal?: AbortSignal
staleMs?: number
timeoutMs?: number
baseDelayMs?: number
maxDelayMs?: number
onWait?: Wait
}
type Opts = {
staleMs: number
timeoutMs: number
baseDelayMs: number
maxDelayMs: number
}
type Owned = {
acquired: true
startHeartbeat: (intervalMs?: number) => void
release: () => Promise<void>
}
export interface Lease {
release: () => Promise<void>
[Symbol.asyncDispose]: () => Promise<void>
}
function code(err: unknown) {
if (typeof err !== "object" || err === null || !("code" in err)) return
const value = err.code
if (typeof value !== "string") return
return value
}
function sleep(ms: number, signal?: AbortSignal) {
return new Promise<void>((resolve, reject) => {
if (signal?.aborted) {
reject(signal.reason ?? new Error("Aborted"))
return
}
let timer: NodeJS.Timeout | undefined
const done = () => {
signal?.removeEventListener("abort", abort)
resolve()
}
const abort = () => {
if (timer) {
clearTimeout(timer)
}
signal?.removeEventListener("abort", abort)
reject(signal?.reason ?? new Error("Aborted"))
}
signal?.addEventListener("abort", abort, { once: true })
timer = setTimeout(done, ms)
})
}
function jitter(ms: number) {
const j = Math.floor(ms * 0.3)
const d = Math.floor(Math.random() * (2 * j + 1)) - j
return Math.max(0, ms + d)
}
function mono() {
return performance.now()
}
function wall() {
return performance.timeOrigin + mono()
}
async function stats(file: string) {
try {
return await stat(file)
} catch (err) {
const errCode = code(err)
if (errCode === "ENOENT" || errCode === "ENOTDIR") return
throw err
}
}
async function stale(lockDir: string, heartbeatPath: string, metaPath: string, staleMs: number) {
// Stale detection allows automatic recovery after crashed owners.
const now = wall()
const heartbeat = await stats(heartbeatPath)
if (heartbeat) {
return now - heartbeat.mtimeMs > staleMs
}
const meta = await stats(metaPath)
if (meta) {
return now - meta.mtimeMs > staleMs
}
const dir = await stats(lockDir)
if (!dir) {
return false
}
return now - dir.mtimeMs > staleMs
}
async function tryAcquireLockDir(lockDir: string, opts: Opts): Promise<Owned | { acquired: false }> {
const token = randomUUID?.() ?? randomBytes(16).toString("hex")
const metaPath = path.join(lockDir, "meta.json")
const heartbeatPath = path.join(lockDir, "heartbeat")
try {
await mkdir(lockDir, { mode: 0o700 })
} catch (err) {
if (code(err) !== "EEXIST") {
throw err
}
if (!(await stale(lockDir, heartbeatPath, metaPath, opts.staleMs))) {
return { acquired: false }
}
const breakerPath = lockDir + ".breaker"
try {
await mkdir(breakerPath, { mode: 0o700 })
} catch (claimErr) {
const errCode = code(claimErr)
if (errCode === "EEXIST") {
const breaker = await stats(breakerPath)
if (breaker && wall() - breaker.mtimeMs > opts.staleMs) {
await rm(breakerPath, { recursive: true, force: true }).catch(() => undefined)
}
return { acquired: false }
}
if (errCode === "ENOENT" || errCode === "ENOTDIR") {
return { acquired: false }
}
throw claimErr
}
try {
// Breaker ownership ensures only one contender performs stale cleanup.
if (!(await stale(lockDir, heartbeatPath, metaPath, opts.staleMs))) {
return { acquired: false }
}
await rm(lockDir, { recursive: true, force: true })
try {
await mkdir(lockDir, { mode: 0o700 })
} catch (retryErr) {
const errCode = code(retryErr)
if (errCode === "EEXIST" || errCode === "ENOTEMPTY") {
return { acquired: false }
}
throw retryErr
}
} finally {
await rm(breakerPath, { recursive: true, force: true }).catch(() => undefined)
}
}
const meta = {
token,
pid: process.pid,
hostname: os.hostname(),
createdAt: new Date().toISOString(),
}
await writeFile(heartbeatPath, "", { flag: "wx" }).catch(async () => {
await rm(lockDir, { recursive: true, force: true })
throw new Error("Lock acquired but heartbeat already existed (possible compromise).")
})
await writeFile(metaPath, JSON.stringify(meta, null, 2), { flag: "wx" }).catch(async () => {
await rm(lockDir, { recursive: true, force: true })
throw new Error("Lock acquired but meta.json already existed (possible compromise).")
})
let timer: NodeJS.Timeout | undefined
const startHeartbeat = (intervalMs = Math.max(100, Math.floor(opts.staleMs / 3))) => {
if (timer) return
// Heartbeat prevents long critical sections from being evicted as stale.
timer = setInterval(() => {
const t = new Date()
void utimes(heartbeatPath, t, t).catch(() => undefined)
}, intervalMs)
timer.unref?.()
}
const release = async () => {
if (timer) {
clearInterval(timer)
timer = undefined
}
const current = await readFile(metaPath, "utf8")
.then((raw) => {
const parsed = JSON.parse(raw)
if (!parsed || typeof parsed !== "object") return {}
return {
token: "token" in parsed && typeof parsed.token === "string" ? parsed.token : undefined,
}
})
.catch((err) => {
const errCode = code(err)
if (errCode === "ENOENT" || errCode === "ENOTDIR") {
throw new Error("Refusing to release: lock is compromised (metadata missing).")
}
if (err instanceof SyntaxError) {
throw new Error("Refusing to release: lock is compromised (metadata invalid).")
}
throw err
})
// Token check prevents deleting a lock that was re-acquired by another process.
if (current.token !== token) {
throw new Error("Refusing to release: lock token mismatch (not the owner).")
}
await rm(lockDir, { recursive: true, force: true })
}
return {
acquired: true,
startHeartbeat,
release,
}
}
async function acquireLockDir(
lockDir: string,
input: { key: string; onWait?: Wait; signal?: AbortSignal },
opts: Opts,
) {
const stop = mono() + opts.timeoutMs
let attempt = 0
let waited = 0
let delay = opts.baseDelayMs
while (true) {
input.signal?.throwIfAborted()
const res = await tryAcquireLockDir(lockDir, opts)
if (res.acquired) {
return res
}
if (mono() > stop) {
throw new Error(`Timed out waiting for lock: ${input.key}`)
}
attempt += 1
const ms = jitter(delay)
await input.onWait?.({
key: input.key,
attempt,
delay: ms,
waited,
})
await sleep(ms, input.signal)
waited += ms
delay = Math.min(opts.maxDelayMs, Math.floor(delay * 1.7))
}
}
export async function acquire(key: string, input: Options = {}): Promise<Lease> {
input.signal?.throwIfAborted()
const cfg: Opts = {
staleMs: input.staleMs ?? defaultOpts.staleMs,
timeoutMs: input.timeoutMs ?? defaultOpts.timeoutMs,
baseDelayMs: input.baseDelayMs ?? defaultOpts.baseDelayMs,
maxDelayMs: input.maxDelayMs ?? defaultOpts.maxDelayMs,
}
const dir = input.dir ?? root()
await mkdir(dir, { recursive: true })
const lockfile = path.join(dir, Hash.fast(key) + ".lock")
const lock = await acquireLockDir(
lockfile,
{
key,
onWait: input.onWait,
signal: input.signal,
},
cfg,
)
lock.startHeartbeat()
const release = () => lock.release()
return {
release,
[Symbol.asyncDispose]() {
return release()
},
}
}
export async function withLock<T>(key: string, fn: () => Promise<T>, input: Options = {}) {
await using _ = await acquire(key, input)
input.signal?.throwIfAborted()
return await fn()
}
export const effect = Effect.fn("Flock.effect")(function* (key: string, input: Options = {}) {
return yield* Effect.acquireRelease(
Effect.promise((signal) => Flock.acquire(key, { ...input, signal })).pipe(
Effect.withSpan("Flock.acquire", {
attributes: { key },
}),
),
(lock) => Effect.promise(() => lock.release()).pipe(Effect.withSpan("Flock.release")),
).pipe(Effect.asVoid)
})
}

34
packages/core/src/util/glob.ts
View file

@ -1,34 +0,0 @@
import { glob, globSync, type GlobOptions } from "glob"
import { minimatch } from "minimatch"
export namespace Glob {
export interface Options {
cwd?: string
absolute?: boolean
include?: "file" | "all"
dot?: boolean
symlink?: boolean
}
function toGlobOptions(options: Options): GlobOptions {
return {
cwd: options.cwd,
absolute: options.absolute,
dot: options.dot,
follow: options.symlink ?? false,
nodir: options.include !== "all",
}
}
export async function scan(pattern: string, options: Options = {}): Promise<string[]> {
return glob(pattern, toGlobOptions(options)) as Promise<string[]>
}
export function scanSync(pattern: string, options: Options = {}): string[] {
return globSync(pattern, toGlobOptions(options)) as string[]
}
export function match(pattern: string, filepath: string): boolean {
return minimatch(filepath, pattern, { dot: true })
}
}

11
packages/core/src/util/hash.ts
View file

@ -1,11 +0,0 @@
import { createHash } from "crypto"
export namespace Hash {
export function fast(input: string | Buffer): string {
return createHash("sha1").update(input).digest("hex")
}
export function sha256(input: string | Buffer): string {
return createHash("sha256").update(input).digest("hex")
}
}

2
packages/core/src/util/process-lock.ts
View file

@ -3,7 +3,7 @@ import { closeSync, mkdirSync, openSync } from "node:fs"
import { connect, createServer, type Server, type Socket } from "node:net"
import path from "node:path"
import { Effect, Schema } from "effect"
import { Hash } from "./hash"
import { Hash } from "@opencode-ai/util/hash"
export namespace ProcessLock {
export class HeldError extends Schema.TaggedErrorClass<HeldError>()("ProcessLockHeldError", {

2
packages/core/src/util/which.ts
View file

@ -1,6 +1,6 @@
import whichPkg from "which"
import path from "path"
import { Global } from "../global"
import { Global } from "@opencode-ai/util/global"
export function which(cmd: string, env?: NodeJS.ProcessEnv) {
const base = env?.PATH ?? env?.Path ?? process.env.PATH ?? process.env.Path ?? ""