refactor(core): unify filesystem search service (#31566)

This commit is contained in:
Dax 2026-06-09 20:38:02 -04:00 committed by GitHub
commit a0409e64d8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
57 changed files with 962 additions and 2852 deletions

View file

@ -78,6 +78,7 @@
"@clack/prompts": "1.0.0-alpha.1",
"@effect/opentelemetry": "catalog:",
"@effect/platform-node": "catalog:",
"@ff-labs/fff-bun": "0.9.3",
"@gitlab/opencode-gitlab-auth": "1.3.3",
"@modelcontextprotocol/sdk": "1.29.0",
"@octokit/graphql": "9.0.2",

View file

@ -1,10 +1,11 @@
import { Effect } from "effect"
import { Fff } from "@opencode-ai/core/filesystem/fff.bun"
import { AppRuntime } from "@/effect/app-runtime"
import { Search } from "@opencode-ai/core/filesystem/search"
import { FileSystem } from "@opencode-ai/core/filesystem"
import { AbsolutePath } from "@opencode-ai/core/schema"
import { InstanceStore } from "@/project/instance-store"
const dir = process.cwd()
const dir = AbsolutePath.make(process.cwd())
const FILE_QUERIES = ["fff", "package.json", "tools/ experiment"]
const GREP_QUERIES = ["FileFinder", "import", "grep", "autocomplete"]
@ -14,7 +15,7 @@ const FILE_LIMIT = 100
const GREP_LIMIT = 50
const GLOB_LIMIT = 50
const run = <A>(effect: Effect.Effect<A, unknown, Search.Service>) =>
const run = <A, R>(effect: Effect.Effect<A, unknown, R>) =>
AppRuntime.runPromise(
InstanceStore.Service.use((store) => store.provide({ directory: dir }, effect as never)),
) as Promise<A>
@ -57,32 +58,16 @@ for (const q of GREP_QUERIES) {
picker.destroy()
// --- Ripgrep service (via Search with file:["."] to force rg path) ---
console.log()
console.log("--- Ripgrep (via Search service) ---")
// warmup
await run(Search.Service.use((svc) => svc.search({ cwd: dir, pattern: "_warmup_rg_", limit: 1, file: ["."] })))
for (const q of GREP_QUERIES) {
const t = performance.now()
const r = await run(Search.Service.use((svc) => svc.search({ cwd: dir, pattern: q, limit: GREP_LIMIT, file: ["."] })))
console.log(
`[ripgrep] grep "${q}": ${(performance.now() - t).toFixed(1)}ms (${r.items.length} total, limit is per-file not total)`,
)
}
// --- Search service: init breakdown ---
console.log()
// 1) runtime + InstanceState + picker create + scan poll
const tRuntime = performance.now()
await run(Search.Service.use((svc) => svc.file({ cwd: dir, query: "_warmup_file_", limit: 1 })))
console.log(`[Search] init file (runtime + picker + scan): ${(performance.now() - tRuntime).toFixed(1)}ms`)
// 2) grep warmup (content index cold-start inside the Search service picker)
const tGrepWarmup = performance.now()
await run(Search.Service.use((svc) => svc.search({ cwd: dir, pattern: "_warmup_grep_", limit: 1 })))
await run(FileSystem.Service.use((svc) => svc.grep({ pattern: "_warmup_grep_", limit: 1 })))
console.log(`[Search] init grep (content index warmup): ${(performance.now() - tGrepWarmup).toFixed(1)}ms`)
console.log()
@ -90,24 +75,20 @@ console.log("--- Search service (warm) ---")
for (const q of FILE_QUERIES) {
const t = performance.now()
const r = await run(Search.Service.use((svc) => svc.file({ cwd: dir, query: q, limit: FILE_LIMIT })))
console.log(`[Search.file] "${q}": ${(performance.now() - t).toFixed(1)}ms (${r.length} results)`)
const r = await run(FileSystem.Service.use((svc) => svc.find({ query: q, limit: FILE_LIMIT })))
console.log(`[Search.find] "${q}": ${(performance.now() - t).toFixed(1)}ms (${r.length} results)`)
}
for (const q of GREP_QUERIES) {
const t = performance.now()
const r = await run(Search.Service.use((svc) => svc.search({ cwd: dir, pattern: q, limit: GREP_LIMIT })))
console.log(
`[Search.search] "${q}": ${(performance.now() - t).toFixed(1)}ms (${r.items.length} matches, engine=${r.engine})`,
)
const r = await run(FileSystem.Service.use((svc) => svc.grep({ pattern: q, limit: GREP_LIMIT })))
console.log(`[Search.grep] "${q}": ${(performance.now() - t).toFixed(1)}ms (${r.length} matches)`)
}
for (const q of GLOB_QUERIES) {
const t = performance.now()
const r = await run(Search.Service.use((svc) => svc.glob({ cwd: dir, pattern: q, limit: GLOB_LIMIT })))
console.log(
`[Search.glob] "${q}": ${(performance.now() - t).toFixed(1)}ms (${r.files.length} files, truncated=${r.truncated})`,
)
const r = await run(FileSystem.Service.use((svc) => svc.glob({ pattern: q, limit: GLOB_LIMIT })))
console.log(`[Search.glob] "${q}": ${(performance.now() - t).toFixed(1)}ms (${r.length} files)`)
}
process.exit(0)

View file

@ -140,6 +140,7 @@ const binaries: Record<string, string> = {}
if (!skipInstall) {
await $`bun install --os="*" --cpu="*" @opentui/core@${pkg.dependencies["@opentui/core"]}`
await $`bun install --os="*" --cpu="*" @parcel/watcher@${pkg.dependencies["@parcel/watcher"]}`
await $`bun install --os="*" --cpu="*" @ff-labs/fff-bun@${pkg.dependencies["@ff-labs/fff-bun"]}`
}
for (const item of targets) {
const name = [
@ -165,7 +166,7 @@ for (const item of targets) {
const workerRelativePath = path.relative(dir, parserWorker).replaceAll("\\", "/")
await Bun.build({
conditions: ["node"],
conditions: ["bun", "node"],
tsconfig: "./tsconfig.json",
plugins: [plugin],
external: ["node-gyp"],
@ -186,6 +187,7 @@ for (const item of targets) {
files: embeddedFileMap ? { "opencode-web-ui.gen.ts": embeddedFileMap } : {},
entrypoints: ["./src/index.ts", parserWorker, workerPath, ...(embeddedFileMap ? ["opencode-web-ui.gen.ts"] : [])],
define: {
FFF_LIBC: JSON.stringify(item.abi === "musl" ? "musl" : "gnu"),
OPENCODE_VERSION: `'${Script.version}'`,
OPENCODE_MODELS_DEV: generated.modelsData,
OTUI_TREE_SITTER_WORKER_PATH: bunfsRoot + workerRelativePath,

View file

@ -2,7 +2,6 @@ import { EOL } from "os"
import { Effect } from "effect"
import { FileSystem } from "@opencode-ai/core/filesystem"
import { LocationServiceMap } from "@opencode-ai/core/location-layer"
import { Search } from "@opencode-ai/core/filesystem/search"
import { AbsolutePath, RelativePath } from "@opencode-ai/core/schema"
import { effectCmd } from "../../effect-cmd"
import { cmd } from "../cmd"
@ -23,7 +22,11 @@ const FileSearchCommand = effectCmd({
description: "Search query",
}),
handler: Effect.fn("Cli.debug.file.search")(function* (args) {
const results = yield* filesystem(FileSystem.Service.use((svc) => svc.find({ query: args.query })))
const results = yield* Effect.orDie(
filesystem(
FileSystem.Service.use((svc) => svc.find({ query: args.query })),
),
)
process.stdout.write(results.map((item) => item.path).join(EOL) + EOL)
}),
})
@ -58,21 +61,6 @@ const FileListCommand = effectCmd({
}),
})
const FileTreeCommand = effectCmd({
command: "tree [dir]",
describe: "show directory tree",
builder: (yargs) =>
yargs.positional("dir", {
type: "string",
description: "Directory to tree",
default: process.cwd(),
}),
handler: Effect.fn("Cli.debug.file.tree")(function* (args) {
const tree = yield* Effect.orDie(Search.Service.use((svc) => svc.tree({ cwd: args.dir, limit: 200 })))
console.log(JSON.stringify(tree, null, 2))
}),
})
export const FileCommand = cmd({
command: "file",
describe: "file system debugging utilities",
@ -81,7 +69,6 @@ export const FileCommand = cmd({
.command(FileReadCommand)
.command(FileListCommand)
.command(FileSearchCommand)
.command(FileTreeCommand)
.demandCommand(),
async handler() {},
})

View file

@ -1,6 +1,6 @@
import { EOL } from "os"
import { Effect, Stream } from "effect"
import { Search } from "@opencode-ai/core/filesystem/search"
import { Effect } from "effect"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { effectCmd } from "../../effect-cmd"
import { cmd } from "../cmd"
import { InstanceRef } from "@/effect/instance-ref"
@ -8,25 +8,10 @@ import { InstanceRef } from "@/effect/instance-ref"
export const RipgrepCommand = cmd({
command: "rg",
describe: "ripgrep debugging utilities",
builder: (yargs) => yargs.command(TreeCommand).command(FilesCommand).command(SearchCommand).demandCommand(),
builder: (yargs) => yargs.command(FilesCommand).command(SearchCommand).demandCommand(),
async handler() {},
})
const TreeCommand = effectCmd({
command: "tree",
describe: "show file tree using ripgrep",
builder: (yargs) =>
yargs.option("limit", {
type: "number",
}),
handler: Effect.fn("Cli.debug.rg.tree")(function* (args) {
const ctx = yield* InstanceRef
if (!ctx) return
const tree = yield* Effect.orDie(Search.Service.use((svc) => svc.tree({ cwd: ctx.directory, limit: args.limit })))
process.stdout.write(tree + EOL)
}),
})
const FilesCommand = effectCmd({
command: "files",
describe: "list files using ripgrep",
@ -47,19 +32,15 @@ const FilesCommand = effectCmd({
handler: Effect.fn("Cli.debug.rg.files")(function* (args) {
const ctx = yield* InstanceRef
if (!ctx) return
const search = yield* Search.Service
const files = yield* search
.files({
const ripgrep = yield* Ripgrep.Service
const files = yield* ripgrep
.glob({
cwd: ctx.directory,
glob: args.glob ? [args.glob] : undefined,
pattern: args.glob ?? "**/*",
limit: args.limit ?? 10_000,
})
.pipe(
Stream.take(args.limit ?? Infinity),
Stream.runCollect,
Effect.map((c) => [...c]),
Effect.orDie,
)
process.stdout.write(files.join(EOL) + EOL)
.pipe(Effect.orDie)
process.stdout.write(files.map((file) => file.path).join(EOL) + EOL)
}),
})
@ -84,16 +65,15 @@ const SearchCommand = effectCmd({
handler: Effect.fn("Cli.debug.rg.search")(function* (args) {
const ctx = yield* InstanceRef
if (!ctx) return
const results = yield* Effect.orDie(
Search.Service.use((svc) =>
svc.search({
cwd: ctx.directory,
pattern: args.pattern,
glob: args.glob as string[] | undefined,
limit: args.limit,
}),
),
)
process.stdout.write(JSON.stringify(results.items, null, 2) + EOL)
const ripgrep = yield* Ripgrep.Service
const results = yield* ripgrep
.grep({
cwd: ctx.directory,
pattern: args.pattern,
include: args.glob?.[0],
limit: args.limit ?? 10_000,
})
.pipe(Effect.orDie)
process.stdout.write(JSON.stringify(results, null, 2) + EOL)
}),
})

View file

@ -8,8 +8,7 @@ import { Auth } from "@/auth"
import { Account } from "@/account/account"
import { Config } from "@/config/config"
import { Git } from "@/git"
import { Ripgrep } from "@opencode-ai/core/filesystem/ripgrep"
import { Search } from "@opencode-ai/core/filesystem/search"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { Storage } from "@/storage/storage"
import { Snapshot } from "@/snapshot"
import { Plugin } from "@/plugin"
@ -61,8 +60,6 @@ export const AppLayer = Layer.mergeAll(
Account.defaultLayer,
Config.defaultLayer,
Git.defaultLayer,
Ripgrep.defaultLayer,
Search.defaultLayer,
Storage.defaultLayer,
Snapshot.defaultLayer,
Plugin.defaultLayer,
@ -102,7 +99,11 @@ export const AppLayer = Layer.mergeAll(
Installation.defaultLayer,
ShareNext.defaultLayer,
SessionShare.defaultLayer,
).pipe(Layer.provideMerge(InstanceLayer.layer), Layer.provideMerge(Observability.layer))
).pipe(
Layer.provideMerge(Ripgrep.defaultLayer),
Layer.provideMerge(InstanceLayer.layer),
Layer.provideMerge(Observability.layer),
)
const rt = ManagedRuntime.make(AppLayer, { memoMap })
type Runtime = Pick<typeof rt, "runSync" | "runPromise" | "runPromiseExit" | "runFork" | "runCallback" | "dispose">

View file

@ -6,9 +6,7 @@ import { Snapshot } from "../snapshot"
import * as Project from "./project"
import * as Vcs from "./vcs"
import { InstanceState } from "@/effect/instance-state"
import { registerDisposer } from "@/effect/instance-registry"
import { ShareNext } from "@/share/share-next"
import { Search } from "@opencode-ai/core/filesystem/search"
import { Effect, Layer } from "effect"
import { Config } from "@/config/config"
import { Service } from "./bootstrap-service"
@ -27,25 +25,15 @@ export const layer = Layer.effect(
const lsp = yield* LSP.Service
const plugin = yield* Plugin.Service
const project = yield* Project.Service
const search = yield* Search.Service
const shareNext = yield* ShareNext.Service
const snapshot = yield* Snapshot.Service
const vcs = yield* Vcs.Service
// once we dispose the service - also release all the internal fff resources
const off = registerDisposer((directory) => Effect.runPromise(search.release(directory)))
yield* Effect.addFinalizer(() => Effect.sync(off))
const run = Effect.gen(function* () {
const ctx = yield* InstanceState.context
yield* Effect.logInfo("bootstrapping", { directory: ctx.directory })
// everything depends on config so eager load it for nice traces
yield* config.get()
// in 99% of use cases user that is opened opencode at certain directory will
// conduct a file search in this direcotry, it could be switched later but
// mostly always we will need a file picker for cwd
// so synchronously start FFF scan for a cwd so it is ready before first toolcall generated
yield* search.warm(ctx.directory).pipe(Effect.ignore)
// Plugin can mutate config so it has to be initialized before anything else.
yield* plugin.init()
// Each service self-manages its own slow work via Effect.forkScoped against
@ -68,7 +56,6 @@ export const defaultLayer: Layer.Layer<Service> = layer.pipe(
LSP.defaultLayer,
Plugin.defaultLayer,
Project.defaultLayer,
Search.defaultLayer,
ShareNext.defaultLayer,
Snapshot.defaultLayer,
Vcs.defaultLayer,
@ -81,7 +68,6 @@ export const node = LayerNode.make(layer, [
LSP.node,
Plugin.node,
Project.node,
Search.node,
ShareNext.node,
Snapshot.node,
Vcs.node,

View file

@ -1,5 +1,4 @@
import { FileSystem } from "@opencode-ai/core/filesystem"
import { Ripgrep } from "@opencode-ai/core/filesystem/ripgrep"
import { NonNegativeInt } from "@opencode-ai/core/schema"
import { LSP } from "@/lsp/lsp"
import { Schema } from "effect"
@ -38,6 +37,20 @@ export const FindSymbolQuery = Schema.Struct({
query: Schema.String,
})
export const LegacyMatch = Schema.Struct({
path: Schema.Struct({ text: Schema.String }),
lines: Schema.Struct({ text: Schema.String }),
line_number: NonNegativeInt,
absolute_offset: NonNegativeInt,
submatches: Schema.Array(
Schema.Struct({
match: Schema.Struct({ text: Schema.String }),
start: NonNegativeInt,
end: NonNegativeInt,
}),
),
})
export const LegacyEntry = Schema.Struct({
name: Schema.String,
path: Schema.String,
@ -94,7 +107,7 @@ export const FileApi = HttpApi.make("file")
.add(
HttpApiEndpoint.get("findText", FilePaths.findText, {
query: FindTextQuery,
success: described(Schema.Array(Ripgrep.SearchMatch), "Matches"),
success: described(Schema.Array(LegacyMatch), "Matches"),
}).annotateMerge(
OpenApi.annotations({
identifier: "find.text",

View file

@ -1,8 +1,7 @@
import * as InstanceState from "@/effect/instance-state"
import { FileSystem } from "@opencode-ai/core/filesystem"
import { LocationServiceMap } from "@opencode-ai/core/location-layer"
import { Ripgrep } from "@opencode-ai/core/filesystem/ripgrep"
import { Search } from "@opencode-ai/core/filesystem/search"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { FSUtil } from "@opencode-ai/core/fs-util"
import { Location } from "@opencode-ai/core/location"
import { AbsolutePath, RelativePath } from "@opencode-ai/core/schema"
@ -15,7 +14,6 @@ import { InstanceHttpApi } from "../api"
export const fileHandlers = HttpApiBuilder.group(InstanceHttpApi, "file", (handlers) =>
Effect.gen(function* () {
const ripgrep = yield* Ripgrep.Service
const search = yield* Search.Service
const locations = yield* LocationServiceMap
const filesystem = Effect.fnUntraced(function* <A, E, R>(effect: Effect.Effect<A, E, R>) {
@ -26,8 +24,18 @@ export const fileHandlers = HttpApiBuilder.group(InstanceHttpApi, "file", (handl
const findText = Effect.fn("FileHttpApi.findText")(function* (ctx: { query: { pattern: string } }) {
return (yield* ripgrep
.search({ cwd: (yield* InstanceState.context).directory, pattern: ctx.query.pattern, limit: 10 })
.pipe(Effect.orDie)).items
.grep({ cwd: (yield* InstanceState.context).directory, pattern: ctx.query.pattern, limit: 10 })
.pipe(Effect.orDie)).map((match) => ({
path: { text: match.entry.path },
lines: { text: match.text },
line_number: match.line,
absolute_offset: match.offset,
submatches: match.submatches.map((submatch) => ({
match: { text: submatch.text },
start: submatch.start,
end: submatch.end,
})),
}))
})
const findFile = Effect.fn("FileHttpApi.findFile")(function* (ctx: {
@ -35,19 +43,18 @@ export const fileHandlers = HttpApiBuilder.group(InstanceHttpApi, "file", (handl
}) {
const directory = (yield* InstanceState.context).directory
const limit = ctx.query.limit ?? 10
const kind = ctx.query.type ?? (ctx.query.dirs === "false" ? "file" : "all")
const type = ctx.query.type ?? (ctx.query.dirs === "false" ? "file" : undefined)
const started = performance.now()
const fff = yield* search.file({ cwd: directory, query: ctx.query.query, limit, kind }).pipe(Effect.orDie)
const found = yield* filesystem(FileSystem.Service.use((fs) => fs.find({ query: ctx.query.query, limit, type })))
yield* Effect.logInfo("find file", {
engine: "fff",
query: ctx.query.query,
kind,
type,
directory,
limit,
results: fff.length,
results: found.length,
duration: Math.round(performance.now() - started),
})
return fff.map((item) => item.path)
return found.map((item) => item.path)
})
const findSymbol = Effect.fn("FileHttpApi.findSymbol")(function* () {
@ -73,10 +80,10 @@ export const fileHandlers = HttpApiBuilder.group(InstanceHttpApi, "file", (handl
return (yield* fs.list({ path: RelativePath.make(ctx.query.path) })).map((item) => ({
name: path.basename(item.path),
path: item.path,
absolute: path.join(directory, item.path),
absolute: path.resolve(location.directory, item.path),
type: item.type,
ignored: ignored.ignores(
path.relative(location.project.directory, path.join(location.directory, item.path)) +
path.relative(location.project.directory, path.resolve(location.directory, item.path)) +
(item.type === "directory" ? "/" : ""),
),
}))
@ -112,4 +119,4 @@ export const fileHandlers = HttpApiBuilder.group(InstanceHttpApi, "file", (handl
.handle("content", content)
.handle("status", status)
}),
).pipe(Layer.provide(LocationServiceMap.layer), Layer.provide(Search.defaultLayer))
).pipe(Layer.provide(LocationServiceMap.layer))

View file

@ -17,7 +17,7 @@ import { BackgroundJob } from "@/background/job"
import { Config } from "@/config/config"
import { Command } from "@/command"
import * as Observability from "@opencode-ai/core/observability"
import { Ripgrep } from "@opencode-ai/core/filesystem/ripgrep"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { Format } from "@/format"
import { RuntimeFlags } from "@/effect/runtime-flags"
import { LSP } from "@/lsp/lsp"
@ -234,7 +234,6 @@ export function createRoutes(
Provider.defaultLayer,
PtyTicket.defaultLayer,
Question.defaultLayer,
Ripgrep.defaultLayer,
RuntimeFlags.defaultLayer,
Session.defaultLayer,
SessionCompaction.defaultLayer,
@ -259,6 +258,7 @@ export function createRoutes(
HttpServer.layerServices,
]),
Layer.provide(Layer.succeed(CorsConfig)(corsOptions)),
Layer.provideMerge(Ripgrep.defaultLayer),
Layer.provide(InstanceLayer.layer),
Layer.provideMerge(Observability.layer),
)

View file

@ -17,13 +17,13 @@ export const assertExternalDirectoryEffect = Effect.fn("Tool.assertExternalDirec
target?: string,
options?: Options,
) {
if (!target) return
if (!target) return false
if (options?.bypass) return
if (options?.bypass) return false
const ins = yield* InstanceState.context
const full = process.platform === "win32" ? FSUtil.normalizePath(target) : target
if (containsPath(full, ins)) return
if (containsPath(full, ins)) return false
const kind = options?.kind ?? "file"
const dir = kind === "directory" ? full : path.dirname(full)
@ -41,6 +41,7 @@ export const assertExternalDirectoryEffect = Effect.fn("Tool.assertExternalDirec
parentDir: dir,
},
})
return true
})
export async function assertExternalDirectory(ctx: Tool.Context, target?: string, options?: Options) {

View file

@ -2,7 +2,7 @@ import path from "path"
import { Effect, Schema } from "effect"
import { InstanceState } from "@/effect/instance-state"
import { FSUtil } from "@opencode-ai/core/fs-util"
import { Search } from "@opencode-ai/core/filesystem/search"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { assertExternalDirectoryEffect } from "./external-directory"
import DESCRIPTION from "./glob.txt"
import * as Tool from "./tool"
@ -18,8 +18,7 @@ export const GlobTool = Tool.define(
"glob",
Effect.gen(function* () {
const fs = yield* FSUtil.Service
const searchSvc = yield* Search.Service
const ripgrep = yield* Ripgrep.Service
return {
description: DESCRIPTION,
parameters: Parameters,
@ -48,18 +47,14 @@ export const GlobTool = Tool.define(
})
const limit = 100
const files = yield* searchSvc.glob({
cwd: search,
pattern: params.pattern,
limit,
signal: ctx.abort,
})
const files = yield* ripgrep.glob({ cwd: search, pattern: params.pattern, limit })
const truncated = files.length === limit
const output = []
if (files.files.length === 0) output.push("No files found")
if (files.files.length > 0) {
output.push(...files.files)
if (files.truncated) {
if (files.length === 0) output.push("No files found")
if (files.length > 0) {
output.push(...files.map((file) => path.resolve(search, file.path)))
if (truncated) {
output.push("")
output.push(
`(Results are truncated: showing first ${limit} results. Consider using a more specific path or pattern.)`,
@ -70,8 +65,8 @@ export const GlobTool = Tool.define(
return {
title: path.relative(ins.worktree, search),
metadata: {
count: files.files.length,
truncated: files.truncated,
count: files.length,
truncated,
},
output: output.join("\n"),
}

View file

@ -2,13 +2,11 @@ import path from "path"
import { Effect, Schema } from "effect"
import { InstanceState } from "@/effect/instance-state"
import { FSUtil } from "@opencode-ai/core/fs-util"
import { Search } from "@opencode-ai/core/filesystem/search"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { assertExternalDirectoryEffect } from "./external-directory"
import DESCRIPTION from "./grep.txt"
import * as Tool from "./tool"
const MAX_LINE_LENGTH = 2000
export const Parameters = Schema.Struct({
pattern: Schema.String.annotate({ description: "The regex pattern to search for in file contents" }),
path: Schema.optional(Schema.String).annotate({
@ -23,8 +21,7 @@ export const GrepTool = Tool.define(
"grep",
Effect.gen(function* () {
const fs = yield* FSUtil.Service
const searchSvc = yield* Search.Service
const ripgrep = yield* Ripgrep.Service
return {
description: DESCRIPTION,
parameters: Parameters,
@ -63,30 +60,27 @@ export const GrepTool = Tool.define(
const search = FSUtil.resolve(requested)
const info = yield* fs.stat(search).pipe(Effect.catch(() => Effect.succeed(undefined)))
const cwd = info?.type === "Directory" ? search : path.dirname(search)
const file = info?.type === "Directory" ? undefined : [path.relative(cwd, search)]
const result = yield* searchSvc.search({
const result = yield* ripgrep.grep({
cwd,
pattern: params.pattern,
glob: params.include ? [params.include] : undefined,
file,
signal: ctx.abort,
include: params.include,
limit: 100,
})
if (result.items.length === 0) return empty
if (result.length === 0) return empty
const rows = result.items.map((item) => ({
path: FSUtil.resolve(path.isAbsolute(item.path.text) ? item.path.text : path.join(cwd, item.path.text)),
line: item.line_number,
text: item.lines.text,
const rows = result.map((item) => ({
path: path.resolve(cwd, item.entry.path),
line: item.line,
text: item.text,
}))
const limit = 100
const truncated = rows.length > limit
const final = truncated ? rows.slice(0, limit) : rows
const truncated = rows.length === limit
const final = rows
if (final.length === 0) return empty
const total = rows.length
const hasMore = truncated || result.hasNextPage
const hasMore = truncated || result.length === limit
const output = [`Found ${total} matches${hasMore ? " (more matches available)" : ""}`]
let current = ""
@ -96,31 +90,12 @@ export const GrepTool = Tool.define(
current = match.path
output.push(`${match.path}:`)
}
const text =
match.text.length > MAX_LINE_LENGTH ? match.text.substring(0, MAX_LINE_LENGTH) + "..." : match.text
output.push(` Line ${match.line}: ${text}`)
output.push(` Line ${match.line}: ${match.text}`)
}
if (truncated) {
output.push("")
output.push(
`(Results truncated: showing ${limit} of ${total} matches (${total - limit} hidden). Consider using a more specific path or pattern.)`,
)
}
if (result.hasNextPage) {
output.push("")
output.push(`(Results truncated. Consider using a more specific path or pattern.)`)
}
if (result.partial) {
output.push("")
output.push("(Some paths were inaccessible and skipped)")
}
if (result.regexFallbackError) {
output.push("")
output.push(`(Regex fallback: ${result.regexFallbackError})`)
output.push("(Results truncated. Consider using a more specific path or pattern.)")
}
return {

View file

@ -8,7 +8,6 @@ import DESCRIPTION from "./read.txt"
import { InstanceState } from "@/effect/instance-state"
import { assertExternalDirectoryEffect } from "./external-directory"
import { Instruction } from "../session/instruction"
import { Search } from "@opencode-ai/core/filesystem/search"
import { isPdfAttachment, sniffAttachmentMime } from "@/util/media"
const DEFAULT_READ_LIMIT = 2000
@ -65,14 +64,13 @@ type Metadata = {
export const ReadTool = Tool.define<
typeof Parameters,
Metadata,
FSUtil.Service | Instruction.Service | LSP.Service | Search.Service | Scope.Scope
FSUtil.Service | Instruction.Service | LSP.Service | Scope.Scope
>(
"read",
Effect.gen(function* () {
const fs = yield* FSUtil.Service
const instruction = yield* Instruction.Service
const lsp = yield* LSP.Service
const search = yield* Search.Service
const scope = yield* Scope.Scope
const miss = Effect.fn("ReadTool.miss")(function* (filepath: string) {
@ -117,7 +115,6 @@ export const ReadTool = Tool.define<
})
const warm = Effect.fn("ReadTool.warm")(function* (filepath: string) {
yield* search.open({ file: filepath }).pipe(Effect.ignore)
// LSP warm-up is optional; do not let a background defect fail an otherwise successful read.
yield* lsp.touchFile(filepath).pipe(Effect.ignoreCause, Effect.forkIn(scope))
})

View file

@ -1,6 +1,6 @@
import { LayerNode } from "@opencode-ai/core/effect/layer-node"
import { httpClient } from "@opencode-ai/core/effect/layer-node-platform"
import { Ripgrep } from "@opencode-ai/core/filesystem/ripgrep"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { PlanExitTool } from "./plan"
import { Session } from "@/session/session"
import { QuestionTool } from "./question"
@ -36,7 +36,6 @@ import { Effect, Layer, Context } from "effect"
import { FetchHttpClient, HttpClient } from "effect/unstable/http"
import { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner"
import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner"
import { Search } from "@opencode-ai/core/filesystem/search"
import { Format } from "../format"
import { InstanceState } from "@/effect/instance-state"
import { EffectBridge } from "@/effect/bridge"
@ -81,30 +80,7 @@ export interface Interface {
export class Service extends Context.Service<Service, Interface>()("@opencode/ToolRegistry") {}
export const layer: Layer.Layer<
Service,
never,
| Config.Service
| Plugin.Service
| Question.Service
| Todo.Service
| Agent.Service
| Skill.Service
| Session.Service
| BackgroundJob.Service
| Provider.Service
| LSP.Service
| Instruction.Service
| FSUtil.Service
| EventV2Bridge.Service
| HttpClient.HttpClient
| ChildProcessSpawner
| Search.Service
| Format.Service
| Truncate.Service
| RuntimeFlags.Service
| Database.Service
> = Layer.effect(
export const layer = Layer.effect(
Service,
Effect.gen(function* () {
const config = yield* Config.Service
@ -358,7 +334,6 @@ export const defaultLayer = Layer.suspend(() =>
Layer.provide(FetchHttpClient.layer),
Layer.provide(Format.defaultLayer),
Layer.provide(CrossSpawnSpawner.defaultLayer),
Layer.provide(Search.defaultLayer),
Layer.provide(Truncate.defaultLayer),
)
.pipe(Layer.provide(Database.defaultLayer), Layer.provide(RuntimeFlags.defaultLayer)),
@ -440,7 +415,7 @@ function isJsonSchemaObject(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value)
}
export const node = LayerNode.make(layer, [
export const node = LayerNode.make(layer.pipe(Layer.provide(Ripgrep.defaultLayer)), [
Config.node,
Plugin.node,
Question.node,
@ -456,8 +431,6 @@ export const node = LayerNode.make(layer, [
EventV2Bridge.node,
httpClient,
CrossSpawnSpawner.node,
Ripgrep.node,
Search.node,
Format.node,
Truncate.node,
RuntimeFlags.node,

View file

@ -1,8 +1,7 @@
import path from "path"
import { pathToFileURL } from "url"
import { Effect, Schema } from "effect"
import * as Stream from "effect/Stream"
import { Search } from "@opencode-ai/core/filesystem/search"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { Skill } from "../skill"
import * as Tool from "./tool"
import DESCRIPTION from "./skill.txt"
@ -15,7 +14,7 @@ export const SkillTool = Tool.define(
"skill",
Effect.gen(function* () {
const skill = yield* Skill.Service
const searchSvc = yield* Search.Service
const ripgrep = yield* Ripgrep.Service
return {
description: DESCRIPTION,
@ -35,14 +34,14 @@ export const SkillTool = Tool.define(
const dir = path.dirname(info.location)
const base = pathToFileURL(dir).href
const limit = 10
const files = yield* searchSvc.files({ cwd: dir, follow: false, hidden: true, signal: ctx.abort }).pipe(
Stream.filter((file) => !file.includes("SKILL.md")),
Stream.map((file) => path.resolve(dir, file)),
Stream.take(limit),
Stream.runCollect,
Effect.map((chunk) => [...chunk].map((file) => `<file>${file}</file>`).join("\n")),
)
const files = yield* ripgrep.find({
cwd: dir,
pattern: "!**/SKILL.md",
hidden: true,
follow: false,
signal: ctx.abort,
limit: 10,
})
return {
title: `Loaded skill: ${info.name}`,
@ -57,7 +56,7 @@ export const SkillTool = Tool.define(
"Note: file list is sampled.",
"",
"<skill_files>",
files,
files.map((file) => `<file>${path.resolve(dir, file.path)}</file>`).join("\n"),
"</skill_files>",
"</skill_content>",
].join("\n"),

View file

@ -33,6 +33,7 @@ import { Project } from "@/project/project"
import { Vcs } from "@/project/vcs"
import { RuntimeFlags } from "@/effect/runtime-flags"
import { EventV2Bridge } from "@/event-v2-bridge"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
const originalEnv = {
OPENCODE_AUTH_CONTENT: process.env.OPENCODE_AUTH_CONTENT,
@ -54,6 +55,7 @@ const workspaceLayer = (experimentalWorkspaces: boolean) =>
Layer.provide(FetchHttpClient.layer),
Layer.provide(FSUtil.defaultLayer),
Layer.provide(RuntimeFlags.layer({ experimentalWorkspaces })),
Layer.provide(Ripgrep.defaultLayer),
Layer.provide(InstanceStore.defaultLayer.pipe(Layer.provide(InstanceBootstrap.defaultLayer))),
)

View file

@ -4,6 +4,7 @@ import { FetchHttpClient } from "effect/unstable/http"
import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner"
import { Database } from "@opencode-ai/core/database/database"
import { FSUtil } from "@opencode-ai/core/fs-util"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { EffectFlock } from "@opencode-ai/core/util/effect-flock"
import path from "path"
import { pathToFileURL } from "url"
@ -55,7 +56,9 @@ const workspaceLayer = Workspace.layer.pipe(
Layer.provide(InstanceStore.defaultLayer.pipe(Layer.provide(noopBootstrapLayer))),
Layer.provide(RuntimeFlags.layer({ experimentalWorkspaces: true })),
)
const it = testEffect(Layer.mergeAll(pluginLayer, workspaceLayer, CrossSpawnSpawner.defaultLayer))
const it = testEffect(
Layer.mergeAll(pluginLayer, workspaceLayer, CrossSpawnSpawner.defaultLayer).pipe(Layer.provide(Ripgrep.defaultLayer)),
)
afterEach(async () => {
await disposeAllInstances()

View file

@ -8,6 +8,7 @@ import { mkdir } from "node:fs/promises"
import path from "node:path"
import { registerAdapter } from "../../src/control-plane/adapters"
import { WorkspaceV2 } from "@opencode-ai/core/workspace"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import type { WorkspaceAdapter } from "../../src/control-plane/types"
import { Workspace } from "../../src/control-plane/workspace"
import { InstanceRef, WorkspaceRef } from "../../src/effect/instance-ref"
@ -53,7 +54,7 @@ const it = testEffect(
InstanceLayer.layer,
Project.defaultLayer,
workspaceLayer,
),
).pipe(Layer.provide(Ripgrep.defaultLayer)),
)
const instanceContextTestLayer = Layer.mergeAll(

View file

@ -16,6 +16,7 @@ import * as Socket from "effect/unstable/socket/Socket"
import { HttpApi, HttpApiBuilder, HttpApiEndpoint, HttpApiGroup, HttpApiSchema } from "effect/unstable/httpapi"
import { mkdir } from "node:fs/promises"
import { registerAdapter } from "../../src/control-plane/adapters"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import type { WorkspaceAdapter } from "../../src/control-plane/types"
import { Workspace } from "../../src/control-plane/workspace"
import { InstanceRef, WorkspaceRef } from "../../src/effect/instance-ref"
@ -58,7 +59,7 @@ const it = testEffect(
InstanceLayer.layer,
Project.defaultLayer,
workspaceLayer,
),
).pipe(Layer.provide(Ripgrep.defaultLayer)),
)
const instanceContextTestLayer = Layer.mergeAll(

View file

@ -9,6 +9,7 @@ import { HttpClient, HttpClientRequest, HttpClientResponse, HttpRouter, HttpServ
import { layerWebSocketConstructorGlobal } from "effect/unstable/socket/Socket"
import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner"
import { Flag } from "@opencode-ai/core/flag/flag"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { registerAdapter } from "../../src/control-plane/adapters"
import type { WorkspaceAdapter } from "../../src/control-plane/types"
import { Workspace } from "../../src/control-plane/workspace"
@ -66,7 +67,7 @@ const it = testEffect(
workspaceLayer,
Database.defaultLayer,
httpApiLayer,
),
).pipe(Layer.provide(Ripgrep.defaultLayer)),
)
function pathFor(path: string, params: Record<string, string>) {

View file

@ -21,6 +21,7 @@ import type { WorkspaceAdapter } from "../../src/control-plane/types"
import { Workspace } from "../../src/control-plane/workspace"
import { WorkspaceTable } from "@opencode-ai/core/control-plane/workspace.sql"
import { Database } from "@opencode-ai/core/database/database"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { Project } from "../../src/project/project"
import { Session } from "../../src/session/session"
import { WorkspacePaths } from "../../src/server/routes/instance/httpapi/groups/workspace"
@ -58,7 +59,7 @@ const it = testEffect(
Project.defaultLayer,
workspaceLayer,
Socket.layerWebSocketConstructorGlobal,
),
).pipe(Layer.provide(Ripgrep.defaultLayer)),
)
type ProxiedRequest = {

View file

@ -11,6 +11,7 @@ import { WorkspacePaths } from "../../src/server/routes/instance/httpapi/groups/
import { EventPaths } from "../../src/server/routes/instance/httpapi/groups/event"
import { Session } from "@/session/session"
import { Database } from "@opencode-ai/core/database/database"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { Server } from "../../src/server/server"
import { resetDatabase } from "../fixture/db"
import { disposeAllInstances, provideInstance, tmpdirScoped } from "../fixture/fixture"
@ -34,7 +35,7 @@ const it = testEffect(
InstanceStore.defaultLayer.pipe(Layer.provide(InstanceBootstrap.defaultLayer)),
Database.defaultLayer,
httpApiLayer,
),
).pipe(Layer.provide(Ripgrep.defaultLayer)),
)
function request(path: string, directory: string, init: RequestInit = {}) {

View file

@ -48,7 +48,7 @@ import { Snapshot } from "../../src/snapshot"
import { ToolRegistry } from "@/tool/registry"
import { Truncate } from "@/tool/truncate"
import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner"
import { Search } from "@opencode-ai/core/filesystem/search"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { Format } from "../../src/format"
import { TestInstance } from "../fixture/fixture"
import { awaitWithTimeout, pollWithTimeout, testEffect } from "../lib/effect"
@ -190,7 +190,7 @@ function makePrompt(input?: { processor?: "blocking" }) {
Layer.provide(FetchHttpClient.layer),
Layer.provide(CrossSpawnSpawner.defaultLayer),
Layer.provide(Git.defaultLayer),
Layer.provide(Search.defaultLayer),
Layer.provide(Ripgrep.defaultLayer),
Layer.provide(Format.defaultLayer),
Layer.provide(RuntimeFlags.layer({ experimentalEventSystem: true })),
Layer.provideMerge(todo),

View file

@ -57,7 +57,7 @@ import { ToolRegistry } from "@/tool/registry"
import { Truncate } from "@/tool/truncate"
import { FSUtil } from "@opencode-ai/core/fs-util"
import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner"
import { Search } from "@opencode-ai/core/filesystem/search"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { Format } from "../../src/format"
import { RuntimeFlags } from "@/effect/runtime-flags"
@ -135,7 +135,7 @@ function makeHttp() {
Layer.provide(FetchHttpClient.layer),
Layer.provide(CrossSpawnSpawner.defaultLayer),
Layer.provide(Git.defaultLayer),
Layer.provide(Search.defaultLayer),
Layer.provide(Ripgrep.defaultLayer),
Layer.provide(Format.defaultLayer),
Layer.provide(RuntimeFlags.layer({ experimentalEventSystem: true })),
Layer.provideMerge(todo),

View file

@ -1,5 +1,6 @@
import { describe, expect, test } from "bun:test"
import { SessionV1 } from "@opencode-ai/core/v1/session"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { Effect, Layer } from "effect"
import { Session } from "@/session/session"
import { SessionPrompt } from "../../src/session/prompt"
@ -8,7 +9,9 @@ import { testEffect } from "../lib/effect"
// Skip tests if no API key is available
const hasApiKey = !!process.env.ANTHROPIC_API_KEY
const it = testEffect(Layer.mergeAll(SessionPrompt.defaultLayer, Session.defaultLayer))
const it = testEffect(
Layer.mergeAll(SessionPrompt.defaultLayer, Session.defaultLayer).pipe(Layer.provide(Ripgrep.defaultLayer)),
)
const live = hasApiKey ? it.instance : it.instance.skip
describe("StructuredOutput Integration", () => {

View file

@ -5,7 +5,7 @@ import { Cause, Effect, Exit, Layer } from "effect"
import { GlobTool } from "../../src/tool/glob"
import { SessionID, MessageID } from "../../src/session/schema"
import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner"
import { Search } from "@opencode-ai/core/filesystem/search"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { FSUtil } from "@opencode-ai/core/fs-util"
import { Global } from "@opencode-ai/core/global"
import { Truncate } from "@/tool/truncate"
@ -23,7 +23,7 @@ const toolLayer = (flags: Partial<RuntimeFlags.Info> = {}) =>
Layer.mergeAll(
CrossSpawnSpawner.defaultLayer,
FSUtil.defaultLayer,
Search.defaultLayer,
Ripgrep.defaultLayer,
Truncate.defaultLayer,
Agent.defaultLayer,
Git.defaultLayer,

View file

@ -11,7 +11,7 @@ import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner"
import { Global } from "@opencode-ai/core/global"
import { Truncate } from "@/tool/truncate"
import { Agent } from "../../src/agent/agent"
import { Search } from "@opencode-ai/core/filesystem/search"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { FSUtil } from "@opencode-ai/core/fs-util"
import { testEffect } from "../lib/effect"
import { Permission } from "../../src/permission"
@ -25,7 +25,7 @@ const toolLayer = (flags: Partial<RuntimeFlags.Info> = {}) =>
Layer.mergeAll(
CrossSpawnSpawner.defaultLayer,
FSUtil.defaultLayer,
Search.defaultLayer,
Ripgrep.defaultLayer,
Truncate.defaultLayer,
Agent.defaultLayer,
Git.defaultLayer,
@ -135,6 +135,25 @@ describe("tool.grep", () => {
}),
)
it.instance("does not report an unknown total when results are truncated", () =>
Effect.gen(function* () {
const test = yield* TestInstance
yield* Effect.promise(() =>
Promise.all(
Array.from({ length: 101 }, (_, index) =>
Bun.write(path.join(test.directory, `match-${index}.txt`), "needle"),
),
),
)
const info = yield* GrepTool
const grep = yield* info.init()
const result = yield* grep.execute({ pattern: "needle", path: test.directory, include: "*.txt" }, ctx)
expect(result.output).toContain("(Results truncated. Consider using a more specific path or pattern.)")
expect(result.output).not.toMatch(/showing \d+ of \d+ matches/)
}),
)
it.instance("supports exact file paths", () =>
Effect.gen(function* () {
const test = yield* TestInstance

View file

@ -8,7 +8,7 @@ import { FSUtil } from "@opencode-ai/core/fs-util"
import { Global } from "@opencode-ai/core/global"
import { Config } from "@/config/config"
import { RuntimeFlags } from "@/effect/runtime-flags"
import { Search } from "@opencode-ai/core/filesystem/search"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { LSP } from "@/lsp/lsp"
import { Permission } from "../../src/permission"
import { SessionID, MessageID } from "../../src/session/schema"
@ -50,7 +50,7 @@ const readLayer = (flags: Partial<RuntimeFlags.Info> = {}) =>
CrossSpawnSpawner.defaultLayer,
Instruction.defaultLayer,
LSP.defaultLayer,
Search.defaultLayer,
Ripgrep.defaultLayer,
Truncate.defaultLayer,
)

View file

@ -26,7 +26,7 @@ import { Instruction } from "@/session/instruction"
import { EventV2Bridge } from "@/event-v2-bridge"
import { FetchHttpClient } from "effect/unstable/http"
import { Format } from "@/format"
import { Search } from "@opencode-ai/core/filesystem/search"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import * as Truncate from "@/tool/truncate"
import { InstanceState } from "@/effect/instance-state"
@ -66,7 +66,7 @@ const registryLayer = (opts: RegistryLayerOptions = {}) =>
Layer.provide(FetchHttpClient.layer),
Layer.provide(Format.defaultLayer),
Layer.provide(Layer.mergeAll(node, Database.defaultLayer)),
Layer.provide(Search.defaultLayer),
Layer.provide(Ripgrep.defaultLayer),
Layer.provide(Truncate.defaultLayer),
)
.pipe(Layer.provide(RuntimeFlags.layer(opts.flags ?? {})))

View file

@ -1,5 +1,6 @@
import { PermissionV1 } from "@opencode-ai/core/v1/permission"
import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { Cause, Effect, Exit, Layer } from "effect"
import { afterEach, describe, expect } from "bun:test"
import path from "path"
@ -28,7 +29,7 @@ afterEach(async () => {
const node = CrossSpawnSpawner.defaultLayer
const it = testEffect(Layer.mergeAll(ToolRegistry.defaultLayer, node))
const it = testEffect(Layer.mergeAll(ToolRegistry.defaultLayer, node).pipe(Layer.provide(Ripgrep.defaultLayer)))
describe("tool.skill", () => {
it.instance("execute returns skill content block with files", () =>

View file

@ -7,6 +7,7 @@ import { BackgroundJob } from "@/background/job"
import { EventV2Bridge } from "@/event-v2-bridge"
import { Config } from "@/config/config"
import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner"
import { Ripgrep } from "@opencode-ai/core/ripgrep"
import { Session } from "@/session/session"
import type { SessionPrompt } from "../../src/session/prompt"
import { MessageID, PartID, SessionID } from "../../src/session/schema"
@ -45,7 +46,7 @@ const layer = (flags: Partial<RuntimeFlags.Info> = {}) =>
ToolRegistry.defaultLayer,
Database.defaultLayer,
RuntimeFlags.layer(flags),
)
).pipe(Layer.provide(Ripgrep.defaultLayer))
const it = testEffect(layer())
const background = testEffect(layer({ experimentalBackgroundSubagents: true }))