refactor(core): unify filesystem search service (#31566)
This commit is contained in:
Dax
GitHub
parent
ce4e658e3f
commit
a0409e64d8
57 changed files with 962 additions and 2852 deletions
|
|
@ -1,231 +0,0 @@
|
|||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import * as Stream from "effect/Stream"
|
||||
import fs from "fs/promises"
|
||||
import os from "os"
|
||||
import path from "path"
|
||||
import { Ripgrep } from "@opencode-ai/core/filesystem/ripgrep"
|
||||
import { testEffect } from "../lib/effect"
|
||||
|
||||
const it = testEffect(Ripgrep.defaultLayer)
|
||||
|
||||
const tmpdir = (init?: (dir: string) => Effect.Effect<void>) =>
|
||||
Effect.acquireRelease(
|
||||
Effect.promise(async () => fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "opencode-test-")))),
|
||||
(dir) =>
|
||||
Effect.promise(() =>
|
||||
fs.rm(dir, {
|
||||
recursive: true,
|
||||
force: true,
|
||||
maxRetries: 5,
|
||||
retryDelay: 100,
|
||||
}),
|
||||
).pipe(Effect.ignore),
|
||||
).pipe(Effect.tap((dir) => init?.(dir) ?? Effect.void))
|
||||
|
||||
const write = (file: string, data: string) => Effect.promise(() => Bun.write(file, data))
|
||||
const mkdir = (dir: string) => Effect.promise(() => fs.mkdir(dir, { recursive: true }))
|
||||
const collectFiles = (input: Ripgrep.FilesInput) =>
|
||||
Ripgrep.Service.use((rg) =>
|
||||
rg.files(input).pipe(
|
||||
Stream.runCollect,
|
||||
Effect.map((c) => [...c]),
|
||||
),
|
||||
)
|
||||
|
||||
const withRipgrepConfig = <A, E, R>(value: string, effect: Effect.Effect<A, E, R>) =>
|
||||
Effect.acquireUseRelease(
|
||||
Effect.sync(() => {
|
||||
const prev = process.env["RIPGREP_CONFIG_PATH"]
|
||||
process.env["RIPGREP_CONFIG_PATH"] = value
|
||||
return prev
|
||||
}),
|
||||
() => effect,
|
||||
(prev) =>
|
||||
Effect.sync(() => {
|
||||
if (prev === undefined) delete process.env["RIPGREP_CONFIG_PATH"]
|
||||
else process.env["RIPGREP_CONFIG_PATH"] = prev
|
||||
}),
|
||||
)
|
||||
|
||||
describe("file.ripgrep", () => {
|
||||
it.live("exposes a cached managed executable filepath", () =>
|
||||
Effect.gen(function* () {
|
||||
const ripgrep = yield* Ripgrep.Service
|
||||
const first = yield* ripgrep.filepath
|
||||
const second = yield* ripgrep.filepath
|
||||
|
||||
expect(first).toBe(second)
|
||||
expect((yield* Effect.promise(() => fs.stat(first))).isFile()).toBe(true)
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("defaults to include hidden", () =>
|
||||
Effect.gen(function* () {
|
||||
const dir = yield* tmpdir((dir) =>
|
||||
Effect.gen(function* () {
|
||||
yield* write(path.join(dir, "visible.txt"), "hello")
|
||||
yield* mkdir(path.join(dir, ".opencode"))
|
||||
yield* write(path.join(dir, ".opencode", "thing.json"), "{}")
|
||||
}),
|
||||
)
|
||||
|
||||
const files = yield* collectFiles({ cwd: dir })
|
||||
expect(files.includes("visible.txt")).toBe(true)
|
||||
expect(files.includes(path.join(".opencode", "thing.json"))).toBe(true)
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("hidden false excludes hidden", () =>
|
||||
Effect.gen(function* () {
|
||||
const dir = yield* tmpdir((dir) =>
|
||||
Effect.gen(function* () {
|
||||
yield* write(path.join(dir, "visible.txt"), "hello")
|
||||
yield* mkdir(path.join(dir, ".opencode"))
|
||||
yield* write(path.join(dir, ".opencode", "thing.json"), "{}")
|
||||
}),
|
||||
)
|
||||
|
||||
const files = yield* collectFiles({ cwd: dir, hidden: false })
|
||||
expect(files.includes("visible.txt")).toBe(true)
|
||||
expect(files.includes(path.join(".opencode", "thing.json"))).toBe(false)
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("search returns empty when nothing matches", () =>
|
||||
Effect.gen(function* () {
|
||||
const dir = yield* tmpdir((dir) => write(path.join(dir, "match.ts"), "const value = 'other'\n"))
|
||||
|
||||
const result = yield* Ripgrep.use.search({ cwd: dir, pattern: "needle" })
|
||||
expect(result.partial).toBe(false)
|
||||
expect(result.items).toEqual([])
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("search returns match metadata with normalized path", () =>
|
||||
Effect.gen(function* () {
|
||||
const dir = yield* tmpdir((dir) =>
|
||||
Effect.gen(function* () {
|
||||
yield* mkdir(path.join(dir, "src"))
|
||||
yield* write(path.join(dir, "src", "match.ts"), "const needle = 1\n")
|
||||
}),
|
||||
)
|
||||
|
||||
const result = yield* Ripgrep.use.search({ cwd: dir, pattern: "needle" })
|
||||
expect(result.partial).toBe(false)
|
||||
expect(result.items).toHaveLength(1)
|
||||
expect(result.items[0]?.path.text).toBe(path.join("src", "match.ts"))
|
||||
expect(result.items[0]?.line_number).toBe(1)
|
||||
expect(result.items[0]?.lines.text).toContain("needle")
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("search returns matched rows with glob filter", () =>
|
||||
Effect.gen(function* () {
|
||||
const dir = yield* tmpdir((dir) =>
|
||||
Effect.gen(function* () {
|
||||
yield* write(path.join(dir, "match.ts"), "const value = 'needle'\n")
|
||||
yield* write(path.join(dir, "skip.txt"), "const value = 'other'\n")
|
||||
}),
|
||||
)
|
||||
|
||||
const result = yield* Ripgrep.use.search({ cwd: dir, pattern: "needle", glob: ["*.ts"] })
|
||||
expect(result.partial).toBe(false)
|
||||
expect(result.items).toHaveLength(1)
|
||||
expect(result.items[0]?.path.text).toContain("match.ts")
|
||||
expect(result.items[0]?.lines.text).toContain("needle")
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("search supports explicit file targets", () =>
|
||||
Effect.gen(function* () {
|
||||
const dir = yield* tmpdir((dir) =>
|
||||
Effect.gen(function* () {
|
||||
yield* write(path.join(dir, "match.ts"), "const value = 'needle'\n")
|
||||
yield* write(path.join(dir, "skip.ts"), "const value = 'needle'\n")
|
||||
}),
|
||||
)
|
||||
|
||||
const file = path.join(dir, "match.ts")
|
||||
const result = yield* Ripgrep.use.search({ cwd: dir, pattern: "needle", file: [file] })
|
||||
expect(result.partial).toBe(false)
|
||||
expect(result.items).toHaveLength(1)
|
||||
expect(result.items[0]?.path.text).toBe(file)
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("files returns empty when glob matches no files", () =>
|
||||
Effect.gen(function* () {
|
||||
const dir = yield* tmpdir((dir) =>
|
||||
Effect.gen(function* () {
|
||||
yield* mkdir(path.join(dir, "packages", "console"))
|
||||
yield* write(path.join(dir, "packages", "console", "package.json"), "{}")
|
||||
}),
|
||||
)
|
||||
|
||||
const files = yield* collectFiles({ cwd: dir, glob: ["packages/*"] })
|
||||
expect(files).toEqual([])
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("files returns stream of filenames", () =>
|
||||
Effect.gen(function* () {
|
||||
const dir = yield* tmpdir((dir) =>
|
||||
Effect.gen(function* () {
|
||||
yield* write(path.join(dir, "a.txt"), "hello")
|
||||
yield* write(path.join(dir, "b.txt"), "world")
|
||||
}),
|
||||
)
|
||||
|
||||
const files = yield* collectFiles({ cwd: dir }).pipe(Effect.map((files) => files.sort()))
|
||||
expect(files).toEqual(["a.txt", "b.txt"])
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("files respects glob filter", () =>
|
||||
Effect.gen(function* () {
|
||||
const dir = yield* tmpdir((dir) =>
|
||||
Effect.gen(function* () {
|
||||
yield* write(path.join(dir, "keep.ts"), "yes")
|
||||
yield* write(path.join(dir, "skip.txt"), "no")
|
||||
}),
|
||||
)
|
||||
|
||||
const files = yield* collectFiles({ cwd: dir, glob: ["*.ts"] })
|
||||
expect(files).toEqual(["keep.ts"])
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("files dies on nonexistent directory", () =>
|
||||
Effect.gen(function* () {
|
||||
const exit = yield* Ripgrep.Service.use((rg) =>
|
||||
rg.files({ cwd: "/tmp/nonexistent-dir-12345" }).pipe(Stream.runCollect),
|
||||
).pipe(Effect.exit)
|
||||
expect(exit._tag).toBe("Failure")
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("ignores RIPGREP_CONFIG_PATH in direct mode", () =>
|
||||
Effect.gen(function* () {
|
||||
const dir = yield* tmpdir((dir) => write(path.join(dir, "match.ts"), "const needle = 1\n"))
|
||||
|
||||
const result = yield* withRipgrepConfig(
|
||||
path.join(dir, "missing-ripgreprc"),
|
||||
Ripgrep.use.search({ cwd: dir, pattern: "needle" }),
|
||||
)
|
||||
expect(result.items).toHaveLength(1)
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("ignores RIPGREP_CONFIG_PATH in worker mode", () =>
|
||||
Effect.gen(function* () {
|
||||
const dir = yield* tmpdir((dir) => write(path.join(dir, "match.ts"), "const needle = 1\n"))
|
||||
|
||||
const result = yield* withRipgrepConfig(
|
||||
path.join(dir, "missing-ripgreprc"),
|
||||
Ripgrep.use.search({ cwd: dir, pattern: "needle" }),
|
||||
)
|
||||
expect(result.items).toHaveLength(1)
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
|
@ -1,176 +1,44 @@
|
|||
import { describe, expect } from "bun:test"
|
||||
import fs from "fs/promises"
|
||||
import os from "os"
|
||||
import path from "path"
|
||||
import { Effect } from "effect"
|
||||
import { Fff } from "#fff"
|
||||
import { Search } from "@opencode-ai/core/filesystem/search"
|
||||
import { Ripgrep } from "@opencode-ai/core/ripgrep"
|
||||
import { AbsolutePath, RelativePath } from "@opencode-ai/core/schema"
|
||||
import { tmpdir } from "../fixture/tmpdir"
|
||||
import { testEffect } from "../lib/effect"
|
||||
|
||||
const it = testEffect(Search.defaultLayer)
|
||||
const it = testEffect(Ripgrep.defaultLayer)
|
||||
|
||||
const tmpdir = (init?: (dir: string) => Effect.Effect<void>) =>
|
||||
const withTmp = <A, E, R>(f: (directory: AbsolutePath) => Effect.Effect<A, E, R>) =>
|
||||
Effect.acquireRelease(
|
||||
Effect.promise(async () => fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "opencode-test-")))),
|
||||
(dir) =>
|
||||
Effect.promise(() => fs.rm(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 })).pipe(
|
||||
Effect.ignore,
|
||||
),
|
||||
).pipe(Effect.tap((dir) => init?.(dir) ?? Effect.void))
|
||||
Effect.promise(() => tmpdir()),
|
||||
(tmp) => Effect.promise(() => tmp[Symbol.asyncDispose]()),
|
||||
).pipe(Effect.flatMap((tmp) => f(AbsolutePath.make(tmp.path))))
|
||||
|
||||
const write = (file: string, data: string) => Effect.promise(() => Bun.write(file, data))
|
||||
const waitForFileIndex = (search: Search.Interface, cwd: string) =>
|
||||
search.glob({ cwd, pattern: "**/*", limit: 1 }).pipe(Effect.ignore)
|
||||
|
||||
describe("file.search", () => {
|
||||
it.live("uses fff for Bun-backed grep", () =>
|
||||
Effect.gen(function* () {
|
||||
expect(Fff.available()).toBe(true)
|
||||
const dir = yield* tmpdir()
|
||||
yield* write(path.join(dir, "src", "match.ts"), "const needle = 1\n")
|
||||
|
||||
const search = yield* Search.Service
|
||||
const result = yield* search.search({ cwd: dir, pattern: "needle", limit: 10 })
|
||||
|
||||
expect(result.engine).toBe("fff")
|
||||
expect(result.items).toHaveLength(1)
|
||||
expect(result.items[0]?.path.text).toBe("src/match.ts")
|
||||
}),
|
||||
describe("Ripgrep", () => {
|
||||
it.live("globs files as an array", () =>
|
||||
withTmp((cwd) =>
|
||||
Effect.gen(function* () {
|
||||
yield* Effect.promise(() => fs.mkdir(path.join(cwd, "src")))
|
||||
yield* Effect.promise(() => fs.writeFile(path.join(cwd, "src", "match.ts"), "needle\n"))
|
||||
const result = yield* (yield* Ripgrep.Service).glob({ cwd, pattern: "**/*.ts", limit: 10 })
|
||||
expect(result.map((item) => item.path)).toEqual([RelativePath.make(path.join("src", "match.ts"))])
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
||||
it.live("keeps fuzzy file abbreviation matches", () =>
|
||||
Effect.gen(function* () {
|
||||
expect(Fff.available()).toBe(true)
|
||||
const dir = yield* tmpdir()
|
||||
yield* write(path.join(dir, "README.md"), "hello\n")
|
||||
|
||||
const search = yield* Search.Service
|
||||
yield* waitForFileIndex(search, dir)
|
||||
const results = yield* search.file({ cwd: dir, query: "rdme", limit: 10 })
|
||||
|
||||
expect(results).toContainEqual({ path: "README.md", type: "file" })
|
||||
}),
|
||||
it.live("greps files with include filtering", () =>
|
||||
withTmp((cwd) =>
|
||||
Effect.gen(function* () {
|
||||
yield* Effect.promise(() => fs.mkdir(path.join(cwd, "src")))
|
||||
yield* Effect.promise(() => fs.writeFile(path.join(cwd, "src", "match.ts"), "needle\n"))
|
||||
yield* Effect.promise(() => fs.writeFile(path.join(cwd, "src", "skip.txt"), "needle\n"))
|
||||
const result = yield* (yield* Ripgrep.Service).grep({ cwd, pattern: "needle", include: "*.ts", limit: 10 })
|
||||
expect(result).toHaveLength(1)
|
||||
expect(result[0]?.entry.path).toBe(RelativePath.make(path.join("src", "match.ts")))
|
||||
expect(result[0]?.submatches[0]?.text).toBe("needle")
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
||||
it.live("keeps empty file query candidates", () =>
|
||||
Effect.gen(function* () {
|
||||
expect(Fff.available()).toBe(true)
|
||||
const dir = yield* tmpdir()
|
||||
yield* write(path.join(dir, "README.md"), "hello\n")
|
||||
yield* write(path.join(dir, "src", "main.ts"), "export const main = true\n")
|
||||
|
||||
const search = yield* Search.Service
|
||||
yield* waitForFileIndex(search, dir)
|
||||
const results = yield* search.file({ cwd: dir, query: "", limit: 10, kind: "all" })
|
||||
|
||||
expect(results).toContainEqual({ path: "README.md", type: "file" })
|
||||
expect(results).toContainEqual({ path: "src/", type: "directory" })
|
||||
expect(results.map((item) => item.path)).not.toContain("")
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("stabilizes equal score file candidates by path length", () =>
|
||||
Effect.gen(function* () {
|
||||
expect(Fff.available()).toBe(true)
|
||||
const dir = yield* tmpdir()
|
||||
yield* write(path.join(dir, "src", "longer-name.ts"), "export const longer = true\n")
|
||||
yield* write(path.join(dir, "a.ts"), "export const shorter = true\n")
|
||||
|
||||
const search = yield* Search.Service
|
||||
yield* waitForFileIndex(search, dir)
|
||||
const results = yield* search.file({ cwd: dir, query: "", limit: 10 })
|
||||
|
||||
expect(results.slice(0, 2)).toEqual([
|
||||
{ path: "a.ts", type: "file" },
|
||||
{ path: "src/longer-name.ts", type: "file" },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("keeps paging grep results without an explicit limit", () =>
|
||||
Effect.gen(function* () {
|
||||
expect(Fff.available()).toBe(true)
|
||||
const dir = yield* tmpdir()
|
||||
yield* write(path.join(dir, "matches.txt"), Array.from({ length: 150 }, (_, idx) => `needle ${idx}\n`).join(""))
|
||||
|
||||
const search = yield* Search.Service
|
||||
const result = yield* search.search({ cwd: dir, pattern: "needle" })
|
||||
|
||||
expect(result.items).toHaveLength(150)
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("uses byte ranges for UTF-8 grep submatches", () =>
|
||||
Effect.gen(function* () {
|
||||
expect(Fff.available()).toBe(true)
|
||||
const dir = yield* tmpdir()
|
||||
yield* write(path.join(dir, "unicode.txt"), "éneedle\n")
|
||||
|
||||
const search = yield* Search.Service
|
||||
const result = yield* search.search({ cwd: dir, pattern: "needle", limit: 10 })
|
||||
|
||||
expect(result.items[0]?.submatches[0]?.match.text).toBe("needle")
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("post-filters fff grep include matches", () =>
|
||||
Effect.gen(function* () {
|
||||
expect(Fff.available()).toBe(true)
|
||||
const dir = yield* tmpdir()
|
||||
yield* write(path.join(dir, "src", "match.ts"), "needle\n")
|
||||
yield* write(path.join(dir, "src", "match.txt"), "needle\n")
|
||||
|
||||
const search = yield* Search.Service
|
||||
const result = yield* search.search({ cwd: dir, pattern: "needle", glob: ["*.ts"], limit: 10 })
|
||||
|
||||
expect(result.engine).toBe("fff")
|
||||
expect(result.items.map((entry) => entry.path.text)).toEqual(["src/match.ts"])
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("keeps fff grep include no-match results", () =>
|
||||
Effect.gen(function* () {
|
||||
expect(Fff.available()).toBe(true)
|
||||
const dir = yield* tmpdir()
|
||||
yield* write(path.join(dir, "src", "match.ts"), "needle\n")
|
||||
|
||||
const search = yield* Search.Service
|
||||
const result = yield* search.search({ cwd: dir, pattern: "missing", glob: ["*.ts"], limit: 10 })
|
||||
|
||||
expect(result.engine).toBe("fff")
|
||||
expect(result.items).toEqual([])
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("post-filters fff glob matches", () =>
|
||||
Effect.gen(function* () {
|
||||
expect(Fff.available()).toBe(true)
|
||||
const dir = yield* tmpdir()
|
||||
yield* write(path.join(dir, "src", "match.ts"), "export const value = 1\n")
|
||||
yield* write(path.join(dir, "src", "match.txt"), "hello\n")
|
||||
|
||||
const search = yield* Search.Service
|
||||
const result = yield* search.glob({ cwd: dir, pattern: "**/*.ts", limit: 10 })
|
||||
|
||||
expect(result.files).toEqual([path.join(dir, "src", "match.ts")])
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("tracks an opened file against its originating query", () =>
|
||||
Effect.gen(function* () {
|
||||
expect(Fff.available()).toBe(true)
|
||||
const dir = yield* tmpdir()
|
||||
yield* write(path.join(dir, "alpha-target-one.ts"), "export const one = 1\n")
|
||||
yield* write(path.join(dir, "alpha-target-two.ts"), "export const two = 2\n")
|
||||
|
||||
const search = yield* Search.Service
|
||||
yield* waitForFileIndex(search, dir)
|
||||
const results = yield* search.file({ cwd: dir, query: "alpha target two", limit: 10 })
|
||||
expect(results).toContainEqual({ path: "alpha-target-two.ts", type: "file" })
|
||||
|
||||
// open() records the query->file association in fff's history db via the
|
||||
// live picker. It must resolve a remembered file and run without error.
|
||||
yield* search.open({ cwd: dir, file: "alpha-target-two.ts" })
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue