feat(opencode): fff search tools (#27802)
Co-authored-by: Shoubhit Dash <shoubhit2005@gmail.com>
This commit is contained in:
parent
4814ab3a3d
commit
7d3d80f840
28 changed files with 1255 additions and 137 deletions
116
packages/opencode/script/bench-search.ts
Normal file
116
packages/opencode/script/bench-search.ts
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
import { Effect } from "effect"
|
||||
import { Fff } from "@opencode-ai/core/filesystem/fff.bun"
|
||||
import { AppRuntime } from "@/effect/app-runtime"
|
||||
import { Search } from "@opencode-ai/core/filesystem/search"
|
||||
import { InstanceStore } from "@/project/instance-store"
|
||||
|
||||
const dir = process.cwd()
|
||||
|
||||
const FILE_QUERIES = ["fff", "package.json", "tools/ experiment"]
|
||||
const GREP_QUERIES = ["FileFinder", "import", "grep", "autocomplete"]
|
||||
const GLOB_QUERIES = ["**/*.test.ts"]
|
||||
|
||||
const FILE_LIMIT = 100
|
||||
const GREP_LIMIT = 50
|
||||
const GLOB_LIMIT = 50
|
||||
|
||||
const run = <A>(effect: Effect.Effect<A, unknown, Search.Service>) =>
|
||||
AppRuntime.runPromise(
|
||||
InstanceStore.Service.use((store) => store.provide({ directory: dir }, effect as never)),
|
||||
) as Promise<A>
|
||||
|
||||
// --- raw Fff picker ---
|
||||
const t0 = performance.now()
|
||||
const made = Fff.create({ basePath: dir, aiMode: true })
|
||||
if (!made.ok) {
|
||||
console.error("Fff.create failed:", made.error)
|
||||
process.exit(1)
|
||||
}
|
||||
const picker = made.value
|
||||
console.log(`picker create: ${(performance.now() - t0).toFixed(1)}ms`)
|
||||
|
||||
const tw = performance.now()
|
||||
await picker.waitForScan(2_500)
|
||||
console.log(`wait for scan: ${(performance.now() - tw).toFixed(1)}ms`)
|
||||
|
||||
// warmup grep to let the content index build
|
||||
const tWarmup = performance.now()
|
||||
picker.grep("_warmup_", { mode: "regex", maxMatchesPerFile: 1, timeBudgetMs: 1_500 })
|
||||
console.log(`grep warmup: ${(performance.now() - tWarmup).toFixed(1)}ms`)
|
||||
|
||||
console.log()
|
||||
console.log("--- raw picker (warm) ---")
|
||||
|
||||
for (const q of FILE_QUERIES) {
|
||||
const t = performance.now()
|
||||
const r = picker.fileSearch(q, { pageSize: Math.max(FILE_LIMIT, 100) })
|
||||
const count = r.ok ? r.value.items.length : "err"
|
||||
console.log(`[picker] fileSearch "${q}": ${(performance.now() - t).toFixed(1)}ms (${count} results)`)
|
||||
}
|
||||
|
||||
for (const q of GREP_QUERIES) {
|
||||
const t = performance.now()
|
||||
const r = picker.grep(q, { mode: "regex", pageSize: GREP_LIMIT, timeBudgetMs: 1_500 })
|
||||
const count = r.ok ? r.value.items.length : "err"
|
||||
console.log(`[picker] grep "${q}": ${(performance.now() - t).toFixed(1)}ms (${count} matches)`)
|
||||
}
|
||||
|
||||
picker.destroy()
|
||||
|
||||
// --- Ripgrep service (via Search with file:["."] to force rg path) ---
|
||||
console.log()
|
||||
console.log("--- Ripgrep (via Search service) ---")
|
||||
|
||||
// warmup
|
||||
await run(Search.Service.use((svc) => svc.search({ cwd: dir, pattern: "_warmup_rg_", limit: 1, file: ["."] })))
|
||||
|
||||
for (const q of GREP_QUERIES) {
|
||||
const t = performance.now()
|
||||
const r = await run(Search.Service.use((svc) => svc.search({ cwd: dir, pattern: q, limit: GREP_LIMIT, file: ["."] })))
|
||||
console.log(
|
||||
`[ripgrep] grep "${q}": ${(performance.now() - t).toFixed(1)}ms (${r.items.length} total, limit is per-file not total)`,
|
||||
)
|
||||
}
|
||||
|
||||
// --- Search service: init breakdown ---
|
||||
console.log()
|
||||
|
||||
// 1) runtime + InstanceState + picker create + scan poll
|
||||
const tRuntime = performance.now()
|
||||
await run(Search.Service.use((svc) => svc.file({ cwd: dir, query: "_warmup_file_", limit: 1 })))
|
||||
console.log(`[Search] init file (runtime + picker + scan): ${(performance.now() - tRuntime).toFixed(1)}ms`)
|
||||
|
||||
// 2) grep warmup (content index cold-start inside the Search service picker)
|
||||
const tGrepWarmup = performance.now()
|
||||
await run(Search.Service.use((svc) => svc.search({ cwd: dir, pattern: "_warmup_grep_", limit: 1 })))
|
||||
console.log(`[Search] init grep (content index warmup): ${(performance.now() - tGrepWarmup).toFixed(1)}ms`)
|
||||
|
||||
console.log()
|
||||
console.log("--- Search service (warm) ---")
|
||||
|
||||
for (const q of FILE_QUERIES) {
|
||||
const t = performance.now()
|
||||
const r = await run(Search.Service.use((svc) => svc.file({ cwd: dir, query: q, limit: FILE_LIMIT })))
|
||||
console.log(
|
||||
`[Search.file] "${q}": ${(performance.now() - t).toFixed(1)}ms (${r?.length ?? "undefined (cache fallback)"} results)`,
|
||||
)
|
||||
}
|
||||
|
||||
for (const q of GREP_QUERIES) {
|
||||
const t = performance.now()
|
||||
const r = await run(Search.Service.use((svc) => svc.search({ cwd: dir, pattern: q, limit: GREP_LIMIT })))
|
||||
console.log(
|
||||
`[Search.search] "${q}": ${(performance.now() - t).toFixed(1)}ms (${r.items.length} matches, engine=${r.engine})`,
|
||||
)
|
||||
}
|
||||
|
||||
for (const q of GLOB_QUERIES) {
|
||||
const t = performance.now()
|
||||
const r = await run(Search.Service.use((svc) => svc.glob({ cwd: dir, pattern: q, limit: GLOB_LIMIT })))
|
||||
console.log(
|
||||
`[Search.glob] "${q}": ${(performance.now() - t).toFixed(1)}ms (${r.files.length} files, truncated=${r.truncated})`,
|
||||
)
|
||||
}
|
||||
|
||||
process.exit(0)
|
||||
|
||||
|
|
@ -2,7 +2,7 @@ import { EOL } from "os"
|
|||
import { Effect } from "effect"
|
||||
import { FileSystem } from "@opencode-ai/core/filesystem"
|
||||
import { LocationServiceMap } from "@opencode-ai/core/location-layer"
|
||||
import { Ripgrep } from "@opencode-ai/core/filesystem/ripgrep"
|
||||
import { Search } from "@opencode-ai/core/filesystem/search"
|
||||
import { AbsolutePath, RelativePath } from "@opencode-ai/core/schema"
|
||||
import { effectCmd } from "../../effect-cmd"
|
||||
import { cmd } from "../cmd"
|
||||
|
|
@ -68,7 +68,7 @@ const FileTreeCommand = effectCmd({
|
|||
default: process.cwd(),
|
||||
}),
|
||||
handler: Effect.fn("Cli.debug.file.tree")(function* (args) {
|
||||
const tree = yield* Effect.orDie(Ripgrep.Service.use((svc) => svc.tree({ cwd: args.dir, limit: 200 })))
|
||||
const tree = yield* Effect.orDie(Search.Service.use((svc) => svc.tree({ cwd: args.dir, limit: 200 })))
|
||||
console.log(JSON.stringify(tree, null, 2))
|
||||
}),
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { EOL } from "os"
|
||||
import { Effect, Stream } from "effect"
|
||||
import { Ripgrep } from "@opencode-ai/core/filesystem/ripgrep"
|
||||
import { Search } from "@opencode-ai/core/filesystem/search"
|
||||
import { effectCmd } from "../../effect-cmd"
|
||||
import { cmd } from "../cmd"
|
||||
import { InstanceRef } from "@/effect/instance-ref"
|
||||
|
|
@ -22,7 +22,7 @@ const TreeCommand = effectCmd({
|
|||
handler: Effect.fn("Cli.debug.rg.tree")(function* (args) {
|
||||
const ctx = yield* InstanceRef
|
||||
if (!ctx) return
|
||||
const tree = yield* Effect.orDie(Ripgrep.Service.use((svc) => svc.tree({ cwd: ctx.directory, limit: args.limit })))
|
||||
const tree = yield* Effect.orDie(Search.Service.use((svc) => svc.tree({ cwd: ctx.directory, limit: args.limit })))
|
||||
process.stdout.write(tree + EOL)
|
||||
}),
|
||||
})
|
||||
|
|
@ -47,8 +47,8 @@ const FilesCommand = effectCmd({
|
|||
handler: Effect.fn("Cli.debug.rg.files")(function* (args) {
|
||||
const ctx = yield* InstanceRef
|
||||
if (!ctx) return
|
||||
const rg = yield* Ripgrep.Service
|
||||
const files = yield* rg
|
||||
const search = yield* Search.Service
|
||||
const files = yield* search
|
||||
.files({
|
||||
cwd: ctx.directory,
|
||||
glob: args.glob ? [args.glob] : undefined,
|
||||
|
|
@ -85,7 +85,7 @@ const SearchCommand = effectCmd({
|
|||
const ctx = yield* InstanceRef
|
||||
if (!ctx) return
|
||||
const results = yield* Effect.orDie(
|
||||
Ripgrep.Service.use((svc) =>
|
||||
Search.Service.use((svc) =>
|
||||
svc.search({
|
||||
cwd: ctx.directory,
|
||||
pattern: args.pattern,
|
||||
|
|
|
|||
|
|
@ -345,21 +345,12 @@ export function Autocomplete(props: {
|
|||
|
||||
const options: AutocompleteOption[] = []
|
||||
|
||||
// Add file options
|
||||
// Add file options. Trust the order returned by fff (frecency, fuzzy
|
||||
// score, filename bonus, etc. are already factored in).
|
||||
if (!result.error && result.data) {
|
||||
const sortedFiles = result.data.sort((a, b) => {
|
||||
const aScore = frecency.getFrecency(a)
|
||||
const bScore = frecency.getFrecency(b)
|
||||
if (aScore !== bScore) return bScore - aScore
|
||||
const aDepth = a.split("/").length
|
||||
const bDepth = b.split("/").length
|
||||
if (aDepth !== bDepth) return aDepth - bDepth
|
||||
return a.localeCompare(b)
|
||||
})
|
||||
|
||||
const width = props.anchor().width - 4
|
||||
options.push(
|
||||
...sortedFiles.map((item): AutocompleteOption => {
|
||||
...result.data.map((item): AutocompleteOption => {
|
||||
const { filename, url, part } = createFilePart(item, lineRange)
|
||||
|
||||
const isDir = item.endsWith("/")
|
||||
|
|
@ -506,45 +497,49 @@ export function Autocomplete(props: {
|
|||
const agentsValue = agents()
|
||||
const referenceAliasesValue = referenceAliases()
|
||||
const commandsValue = commands()
|
||||
|
||||
const mixed: AutocompleteOption[] =
|
||||
store.visible === "@"
|
||||
? referenceMatchValue
|
||||
? referenceAliasesValue.filter((item) => item.display === `@${referenceMatchValue.name}`)
|
||||
: [...referenceAliasesValue, ...agentsValue, ...(filesValue || []), ...mcpResources()]
|
||||
: [...commandsValue]
|
||||
|
||||
const searchValue = search()
|
||||
|
||||
// @<alias>/... — narrow to the matched reference, files come from fff
|
||||
// already ranked so there is no re-ranking here.
|
||||
if (store.visible === "@" && referenceMatchValue) {
|
||||
return referenceAliasesValue.filter((item) => item.display === `@${referenceMatchValue.name}`)
|
||||
}
|
||||
|
||||
// Files come from fff already fuzzy ranked and filtered
|
||||
// it shouldn't be additionally sorted by fuzzysort as it will loose the results
|
||||
const fileOptions: AutocompleteOption[] = store.visible === "@" ? filesValue || [] : []
|
||||
const nonFileOptions: AutocompleteOption[] =
|
||||
store.visible === "@" ? [...referenceAliasesValue, ...agentsValue, ...mcpResources()] : [...commandsValue]
|
||||
|
||||
if (!searchValue) {
|
||||
return mixed
|
||||
return [...nonFileOptions, ...fileOptions]
|
||||
}
|
||||
|
||||
if (files.loading && prev && prev.length > 0) {
|
||||
return prev
|
||||
}
|
||||
|
||||
if (referenceMatchValue) return mixed
|
||||
const fuzziedNonFiles = fuzzysort
|
||||
.go(removeLineRange(searchValue), nonFileOptions, {
|
||||
keys: [
|
||||
(obj) => removeLineRange((obj.value ?? obj.display).trimEnd()),
|
||||
"description",
|
||||
(obj) => obj.aliases?.join(" ") ?? "",
|
||||
],
|
||||
limit: 10,
|
||||
scoreFn: (objResults) => {
|
||||
const displayResult = objResults[0]
|
||||
let score = objResults.score
|
||||
if (displayResult && displayResult.target.startsWith(store.visible + searchValue)) {
|
||||
score *= 2
|
||||
}
|
||||
const frecencyScore = objResults.obj.path ? frecency.getFrecency(objResults.obj.path) : 0
|
||||
return score * (1 + frecencyScore)
|
||||
},
|
||||
})
|
||||
.map((arr) => arr.obj)
|
||||
|
||||
const result = fuzzysort.go(removeLineRange(searchValue), mixed, {
|
||||
keys: [
|
||||
(obj) => removeLineRange((obj.value ?? obj.display).trimEnd()),
|
||||
"description",
|
||||
(obj) => obj.aliases?.join(" ") ?? "",
|
||||
],
|
||||
limit: 10,
|
||||
scoreFn: (objResults) => {
|
||||
const displayResult = objResults[0]
|
||||
let score = objResults.score
|
||||
if (displayResult && displayResult.target.startsWith(store.visible + searchValue)) {
|
||||
score *= 2
|
||||
}
|
||||
const frecencyScore = objResults.obj.path ? frecency.getFrecency(objResults.obj.path) : 0
|
||||
return score * (1 + frecencyScore)
|
||||
},
|
||||
})
|
||||
|
||||
return result.map((arr) => arr.obj)
|
||||
return [...fuzziedNonFiles, ...fileOptions].slice(0, 10)
|
||||
})
|
||||
|
||||
createEffect(() => {
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import { Account } from "@/account/account"
|
|||
import { Config } from "@/config/config"
|
||||
import { Git } from "@/git"
|
||||
import { Ripgrep } from "@opencode-ai/core/filesystem/ripgrep"
|
||||
import { Search } from "@opencode-ai/core/filesystem/search"
|
||||
import { Storage } from "@/storage/storage"
|
||||
import { Snapshot } from "@/snapshot"
|
||||
import { Plugin } from "@/plugin"
|
||||
|
|
@ -62,6 +63,7 @@ export const AppLayer = Layer.mergeAll(
|
|||
Config.defaultLayer,
|
||||
Git.defaultLayer,
|
||||
Ripgrep.defaultLayer,
|
||||
Search.defaultLayer,
|
||||
Storage.defaultLayer,
|
||||
Snapshot.defaultLayer,
|
||||
Plugin.defaultLayer,
|
||||
|
|
|
|||
|
|
@ -5,7 +5,9 @@ import { Snapshot } from "../snapshot"
|
|||
import * as Project from "./project"
|
||||
import * as Vcs from "./vcs"
|
||||
import { InstanceState } from "@/effect/instance-state"
|
||||
import { registerDisposer } from "@/effect/instance-registry"
|
||||
import { ShareNext } from "@/share/share-next"
|
||||
import { Search } from "@opencode-ai/core/filesystem/search"
|
||||
import { Effect, Layer } from "effect"
|
||||
import { Config } from "@/config/config"
|
||||
import { Service } from "./bootstrap-service"
|
||||
|
|
@ -26,15 +28,25 @@ export const layer = Layer.effect(
|
|||
const plugin = yield* Plugin.Service
|
||||
const project = yield* Project.Service
|
||||
const reference = yield* Reference.Service
|
||||
const search = yield* Search.Service
|
||||
const shareNext = yield* ShareNext.Service
|
||||
const snapshot = yield* Snapshot.Service
|
||||
const vcs = yield* Vcs.Service
|
||||
|
||||
// once we dispose the service - also release all the internal fff resources
|
||||
const off = registerDisposer((directory) => Effect.runPromise(search.release(directory)))
|
||||
yield* Effect.addFinalizer(() => Effect.sync(off))
|
||||
|
||||
const run = Effect.gen(function* () {
|
||||
const ctx = yield* InstanceState.context
|
||||
yield* Effect.logInfo("bootstrapping").pipe(Effect.annotateLogs("directory", ctx.directory))
|
||||
// everything depends on config so eager load it for nice traces
|
||||
yield* config.get()
|
||||
// in 99% of use cases user that is opened opencode at certain directory will
|
||||
// conduct a file search in this direcotry, it could be switched later but
|
||||
// mostly always we will need a file picker for cwd
|
||||
// so synchronously start FFF scan for a cwd so it is ready before first toolcall generated
|
||||
yield* search.warm(ctx.directory).pipe(Effect.ignore)
|
||||
// Plugin can mutate config so it has to be initialized before anything else.
|
||||
yield* plugin.init()
|
||||
// Each service self-manages its own slow work via Effect.forkScoped against
|
||||
|
|
@ -58,6 +70,7 @@ export const defaultLayer: Layer.Layer<Service> = layer.pipe(
|
|||
Plugin.defaultLayer,
|
||||
Project.defaultLayer,
|
||||
Reference.defaultLayer,
|
||||
Search.defaultLayer,
|
||||
ShareNext.defaultLayer,
|
||||
Snapshot.defaultLayer,
|
||||
Vcs.defaultLayer,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import * as InstanceState from "@/effect/instance-state"
|
|||
import { FileSystem } from "@opencode-ai/core/filesystem"
|
||||
import { LocationServiceMap } from "@opencode-ai/core/location-layer"
|
||||
import { Ripgrep } from "@opencode-ai/core/filesystem/ripgrep"
|
||||
import { Search } from "@opencode-ai/core/filesystem/search"
|
||||
import { FSUtil } from "@opencode-ai/core/fs-util"
|
||||
import { AbsolutePath, RelativePath } from "@opencode-ai/core/schema"
|
||||
import { Effect, Layer } from "effect"
|
||||
|
|
@ -12,6 +13,7 @@ import { InstanceHttpApi } from "../api"
|
|||
export const fileHandlers = HttpApiBuilder.group(InstanceHttpApi, "file", (handlers) =>
|
||||
Effect.gen(function* () {
|
||||
const ripgrep = yield* Ripgrep.Service
|
||||
const search = yield* Search.Service
|
||||
const locations = yield* LocationServiceMap
|
||||
|
||||
const filesystem = Effect.fnUntraced(function* <A, E, R>(effect: Effect.Effect<A, E, R>) {
|
||||
|
|
@ -29,11 +31,18 @@ export const fileHandlers = HttpApiBuilder.group(InstanceHttpApi, "file", (handl
|
|||
const findFile = Effect.fn("FileHttpApi.findFile")(function* (ctx: {
|
||||
query: { query: string; dirs?: "true" | "false"; type?: "file" | "directory"; limit?: number }
|
||||
}) {
|
||||
const directory = (yield* InstanceState.context).directory
|
||||
const limit = ctx.query.limit ?? 10
|
||||
const kind = ctx.query.type ?? (ctx.query.dirs === "false" ? "file" : "all")
|
||||
// Prefer fff (frecency + fuzzy ranking) and trust its ordering. Fall back
|
||||
// to the ripgrep-backed FileSystem.find when fff is unavailable.
|
||||
const fff = yield* search.file({ cwd: directory, query: ctx.query.query, limit, kind }).pipe(Effect.orDie)
|
||||
if (fff !== undefined) return fff
|
||||
return (yield* filesystem(
|
||||
FileSystem.Service.use((fs) =>
|
||||
fs.find({
|
||||
query: ctx.query.query,
|
||||
limit: ctx.query.limit ?? 10,
|
||||
limit,
|
||||
type: ctx.query.type ?? (ctx.query.dirs === "false" ? "file" : undefined),
|
||||
}),
|
||||
),
|
||||
|
|
@ -91,4 +100,4 @@ export const fileHandlers = HttpApiBuilder.group(InstanceHttpApi, "file", (handl
|
|||
.handle("content", content)
|
||||
.handle("status", status)
|
||||
}),
|
||||
).pipe(Layer.provide(LocationServiceMap.layer))
|
||||
).pipe(Layer.provide(LocationServiceMap.layer), Layer.provide(Search.defaultLayer))
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
import path from "path"
|
||||
import { Effect, Option, Schema } from "effect"
|
||||
import * as Stream from "effect/Stream"
|
||||
import { Effect, Schema } from "effect"
|
||||
import { InstanceState } from "@/effect/instance-state"
|
||||
import { FSUtil } from "@opencode-ai/core/fs-util"
|
||||
import { Ripgrep } from "@opencode-ai/core/filesystem/ripgrep"
|
||||
import { Search } from "@opencode-ai/core/filesystem/search"
|
||||
import { assertExternalDirectoryEffect } from "./external-directory"
|
||||
import DESCRIPTION from "./glob.txt"
|
||||
import * as Tool from "./tool"
|
||||
|
|
@ -19,9 +18,9 @@ export const Parameters = Schema.Struct({
|
|||
export const GlobTool = Tool.define(
|
||||
"glob",
|
||||
Effect.gen(function* () {
|
||||
const rg = yield* Ripgrep.Service
|
||||
const fs = yield* FSUtil.Service
|
||||
const reference = yield* Reference.Service
|
||||
const searchSvc = yield* Search.Service
|
||||
|
||||
return {
|
||||
description: DESCRIPTION,
|
||||
|
|
@ -52,36 +51,18 @@ export const GlobTool = Tool.define(
|
|||
})
|
||||
|
||||
const limit = 100
|
||||
let truncated = false
|
||||
const files = yield* rg.files({ cwd: search, glob: [params.pattern], signal: ctx.abort }).pipe(
|
||||
Stream.mapEffect((file) =>
|
||||
Effect.gen(function* () {
|
||||
const full = path.resolve(search, file)
|
||||
const info = yield* fs.stat(full).pipe(Effect.catch(() => Effect.succeed(undefined)))
|
||||
const mtime =
|
||||
info?.mtime.pipe(
|
||||
Option.map((date) => date.getTime()),
|
||||
Option.getOrElse(() => 0),
|
||||
) ?? 0
|
||||
return { path: full, mtime }
|
||||
}),
|
||||
),
|
||||
Stream.take(limit + 1),
|
||||
Stream.runCollect,
|
||||
Effect.map((chunk) => [...chunk]),
|
||||
)
|
||||
|
||||
if (files.length > limit) {
|
||||
truncated = true
|
||||
files.length = limit
|
||||
}
|
||||
files.sort((a, b) => b.mtime - a.mtime)
|
||||
const files = yield* searchSvc.glob({
|
||||
cwd: search,
|
||||
pattern: params.pattern,
|
||||
limit,
|
||||
signal: ctx.abort,
|
||||
})
|
||||
|
||||
const output = []
|
||||
if (files.length === 0) output.push("No files found")
|
||||
if (files.length > 0) {
|
||||
output.push(...files.map((file) => file.path))
|
||||
if (truncated) {
|
||||
if (files.files.length === 0) output.push("No files found")
|
||||
if (files.files.length > 0) {
|
||||
output.push(...files.files)
|
||||
if (files.truncated) {
|
||||
output.push("")
|
||||
output.push(
|
||||
`(Results are truncated: showing first ${limit} results. Consider using a more specific path or pattern.)`,
|
||||
|
|
@ -92,8 +73,8 @@ export const GlobTool = Tool.define(
|
|||
return {
|
||||
title: path.relative(ins.worktree, search),
|
||||
metadata: {
|
||||
count: files.length,
|
||||
truncated,
|
||||
count: files.files.length,
|
||||
truncated: files.truncated,
|
||||
},
|
||||
output: output.join("\n"),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
- Fast file pattern matching tool that works with any codebase size
|
||||
- Supports glob patterns like "**/*.js" or "src/**/*.ts"
|
||||
- Returns matching file paths sorted by modification time
|
||||
- Returns matching file paths
|
||||
- Use this tool when you need to find files by name patterns
|
||||
- When you are doing an open-ended search that may require multiple rounds of globbing and grepping, use the Task tool instead
|
||||
- You have the capability to call multiple tools in a single response. It is always better to speculatively perform multiple searches as a batch that are potentially useful.
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
import path from "path"
|
||||
import { Schema } from "effect"
|
||||
import { Effect, Option } from "effect"
|
||||
import { Effect, Schema } from "effect"
|
||||
import { InstanceState } from "@/effect/instance-state"
|
||||
import { FSUtil } from "@opencode-ai/core/fs-util"
|
||||
import { Ripgrep } from "@opencode-ai/core/filesystem/ripgrep"
|
||||
import { Search } from "@opencode-ai/core/filesystem/search"
|
||||
import { assertExternalDirectoryEffect } from "./external-directory"
|
||||
import DESCRIPTION from "./grep.txt"
|
||||
import * as Tool from "./tool"
|
||||
|
|
@ -25,7 +24,7 @@ export const GrepTool = Tool.define(
|
|||
"grep",
|
||||
Effect.gen(function* () {
|
||||
const fs = yield* FSUtil.Service
|
||||
const rg = yield* Ripgrep.Service
|
||||
const searchSvc = yield* Search.Service
|
||||
const reference = yield* Reference.Service
|
||||
|
||||
return {
|
||||
|
|
@ -69,7 +68,7 @@ export const GrepTool = Tool.define(
|
|||
const cwd = info?.type === "Directory" ? search : path.dirname(search)
|
||||
const file = info?.type === "Directory" ? undefined : [path.relative(cwd, search)]
|
||||
|
||||
const result = yield* rg.search({
|
||||
const result = yield* searchSvc.search({
|
||||
cwd,
|
||||
pattern: params.pattern,
|
||||
glob: params.include ? [params.include] : undefined,
|
||||
|
|
@ -83,38 +82,15 @@ export const GrepTool = Tool.define(
|
|||
line: item.line_number,
|
||||
text: item.lines.text,
|
||||
}))
|
||||
const times = new Map(
|
||||
(yield* Effect.forEach(
|
||||
[...new Set(rows.map((row) => row.path))],
|
||||
Effect.fnUntraced(function* (file) {
|
||||
const info = yield* fs.stat(file).pipe(Effect.catch(() => Effect.succeed(undefined)))
|
||||
if (!info || info.type === "Directory") return undefined
|
||||
return [
|
||||
file,
|
||||
info.mtime.pipe(
|
||||
Option.map((time) => time.getTime()),
|
||||
Option.getOrElse(() => 0),
|
||||
) ?? 0,
|
||||
] as const
|
||||
}),
|
||||
{ concurrency: 16 },
|
||||
)).filter((entry): entry is readonly [string, number] => Boolean(entry)),
|
||||
)
|
||||
const matches = rows.flatMap((row) => {
|
||||
const mtime = times.get(row.path)
|
||||
if (mtime === undefined) return []
|
||||
return [{ ...row, mtime }]
|
||||
})
|
||||
|
||||
matches.sort((a, b) => b.mtime - a.mtime)
|
||||
|
||||
const limit = 100
|
||||
const truncated = matches.length > limit
|
||||
const final = truncated ? matches.slice(0, limit) : matches
|
||||
const truncated = rows.length > limit
|
||||
const final = truncated ? rows.slice(0, limit) : rows
|
||||
if (final.length === 0) return empty
|
||||
|
||||
const total = matches.length
|
||||
const output = [`Found ${total} matches${truncated ? ` (showing first ${limit})` : ""}`]
|
||||
const total = rows.length
|
||||
const hasMore = truncated || result.hasNextPage
|
||||
const output = [`Found ${total} matches${hasMore ? " (more matches available)" : ""}`]
|
||||
|
||||
let current = ""
|
||||
for (const match of final) {
|
||||
|
|
@ -135,11 +111,23 @@ export const GrepTool = Tool.define(
|
|||
)
|
||||
}
|
||||
|
||||
if (result.hasNextPage) {
|
||||
output.push("")
|
||||
output.push(
|
||||
`(Results truncated. Consider using a more specific path or pattern.)`,
|
||||
)
|
||||
}
|
||||
|
||||
if (result.partial) {
|
||||
output.push("")
|
||||
output.push("(Some paths were inaccessible and skipped)")
|
||||
}
|
||||
|
||||
if (result.regexFallbackError) {
|
||||
output.push("")
|
||||
output.push(`(Regex fallback: ${result.regexFallbackError})`)
|
||||
}
|
||||
|
||||
return {
|
||||
title: params.pattern,
|
||||
metadata: {
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
- Searches file contents using regular expressions
|
||||
- Supports full regex syntax (eg. "log.*Error", "function\s+\w+", etc.)
|
||||
- Filter files by pattern with the include parameter (eg. "*.js", "*.{ts,tsx}")
|
||||
- Returns file paths and line numbers with at least one match sorted by modification time
|
||||
- Returns file paths and line numbers with matching lines
|
||||
- Use this tool when you need to find files containing specific patterns
|
||||
- If you need to identify/count the number of matches within files, use the Bash tool with `rg` (ripgrep) directly. Do NOT use `grep`.
|
||||
- When you are doing an open-ended search that may require multiple rounds of globbing and grepping, use the Task tool instead
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import DESCRIPTION from "./read.txt"
|
|||
import { InstanceState } from "@/effect/instance-state"
|
||||
import { assertExternalDirectoryEffect } from "./external-directory"
|
||||
import { Instruction } from "../session/instruction"
|
||||
import { Search } from "@opencode-ai/core/filesystem/search"
|
||||
import { isPdfAttachment, sniffAttachmentMime } from "@/util/media"
|
||||
import { Reference } from "@/reference/reference"
|
||||
|
||||
|
|
@ -65,7 +66,7 @@ type Metadata = {
|
|||
export const ReadTool = Tool.define<
|
||||
typeof Parameters,
|
||||
Metadata,
|
||||
FSUtil.Service | Instruction.Service | LSP.Service | Reference.Service | Scope.Scope
|
||||
FSUtil.Service | Instruction.Service | LSP.Service | Reference.Service | Search.Service | Scope.Scope
|
||||
>(
|
||||
"read",
|
||||
Effect.gen(function* () {
|
||||
|
|
@ -73,6 +74,7 @@ export const ReadTool = Tool.define<
|
|||
const instruction = yield* Instruction.Service
|
||||
const lsp = yield* LSP.Service
|
||||
const reference = yield* Reference.Service
|
||||
const search = yield* Search.Service
|
||||
const scope = yield* Scope.Scope
|
||||
|
||||
const miss = Effect.fn("ReadTool.miss")(function* (filepath: string) {
|
||||
|
|
@ -117,6 +119,7 @@ export const ReadTool = Tool.define<
|
|||
})
|
||||
|
||||
const warm = Effect.fn("ReadTool.warm")(function* (filepath: string) {
|
||||
yield* search.open({ file: filepath }).pipe(Effect.ignore)
|
||||
// LSP warm-up is optional; do not let a background defect fail an otherwise successful read.
|
||||
yield* lsp.touchFile(filepath).pipe(Effect.ignoreCause, Effect.forkIn(scope))
|
||||
})
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ import { Effect, Layer, Context } from "effect"
|
|||
import { FetchHttpClient, HttpClient } from "effect/unstable/http"
|
||||
import { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner"
|
||||
import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner"
|
||||
import { Ripgrep } from "@opencode-ai/core/filesystem/ripgrep"
|
||||
import { Search } from "@opencode-ai/core/filesystem/search"
|
||||
import { Format } from "../format"
|
||||
import { InstanceState } from "@/effect/instance-state"
|
||||
import { EffectBridge } from "@/effect/bridge"
|
||||
|
|
@ -101,7 +101,7 @@ export const layer: Layer.Layer<
|
|||
| EventV2Bridge.Service
|
||||
| HttpClient.HttpClient
|
||||
| ChildProcessSpawner
|
||||
| Ripgrep.Service
|
||||
| Search.Service
|
||||
| Format.Service
|
||||
| Truncate.Service
|
||||
| RuntimeFlags.Service
|
||||
|
|
@ -386,7 +386,7 @@ export const defaultLayer = Layer.suspend(() =>
|
|||
Layer.provide(FetchHttpClient.layer),
|
||||
Layer.provide(Format.defaultLayer),
|
||||
Layer.provide(CrossSpawnSpawner.defaultLayer),
|
||||
Layer.provide(Ripgrep.defaultLayer),
|
||||
Layer.provide(Search.defaultLayer),
|
||||
Layer.provide(Truncate.defaultLayer),
|
||||
)
|
||||
.pipe(Layer.provide(Database.defaultLayer), Layer.provide(RuntimeFlags.defaultLayer)),
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import path from "path"
|
|||
import { pathToFileURL } from "url"
|
||||
import { Effect, Schema } from "effect"
|
||||
import * as Stream from "effect/Stream"
|
||||
import { Ripgrep } from "@opencode-ai/core/filesystem/ripgrep"
|
||||
import { Search } from "@opencode-ai/core/filesystem/search"
|
||||
import { Skill } from "../skill"
|
||||
import * as Tool from "./tool"
|
||||
import DESCRIPTION from "./skill.txt"
|
||||
|
|
@ -15,7 +15,7 @@ export const SkillTool = Tool.define(
|
|||
"skill",
|
||||
Effect.gen(function* () {
|
||||
const skill = yield* Skill.Service
|
||||
const rg = yield* Ripgrep.Service
|
||||
const searchSvc = yield* Search.Service
|
||||
|
||||
return {
|
||||
description: DESCRIPTION,
|
||||
|
|
@ -36,7 +36,7 @@ export const SkillTool = Tool.define(
|
|||
const dir = path.dirname(info.location)
|
||||
const base = pathToFileURL(dir).href
|
||||
const limit = 10
|
||||
const files = yield* rg.files({ cwd: dir, follow: false, hidden: true, signal: ctx.abort }).pipe(
|
||||
const files = yield* searchSvc.files({ cwd: dir, follow: false, hidden: true, signal: ctx.abort }).pipe(
|
||||
Stream.filter((file) => !file.includes("SKILL.md")),
|
||||
Stream.map((file) => path.resolve(dir, file)),
|
||||
Stream.take(limit),
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ import { ToolRegistry } from "@/tool/registry"
|
|||
import { Truncate } from "@/tool/truncate"
|
||||
import * as Log from "@opencode-ai/core/util/log"
|
||||
import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner"
|
||||
import { Ripgrep } from "@opencode-ai/core/filesystem/ripgrep"
|
||||
import { Search } from "@opencode-ai/core/filesystem/search"
|
||||
import { Format } from "../../src/format"
|
||||
import { Reference } from "../../src/reference/reference"
|
||||
import { RepositoryCache } from "../../src/reference/repository-cache"
|
||||
|
|
@ -196,7 +196,7 @@ function makePrompt(input?: { processor?: "blocking" }) {
|
|||
Layer.provide(RepositoryCache.defaultLayer),
|
||||
Layer.provide(Git.defaultLayer),
|
||||
Layer.provide(Reference.defaultLayer),
|
||||
Layer.provide(Ripgrep.defaultLayer),
|
||||
Layer.provide(Search.defaultLayer),
|
||||
Layer.provide(Format.defaultLayer),
|
||||
Layer.provide(RuntimeFlags.layer({ experimentalEventSystem: true })),
|
||||
Layer.provideMerge(todo),
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ import { ToolRegistry } from "@/tool/registry"
|
|||
import { Truncate } from "@/tool/truncate"
|
||||
import { FSUtil } from "@opencode-ai/core/fs-util"
|
||||
import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner"
|
||||
import { Ripgrep } from "@opencode-ai/core/filesystem/ripgrep"
|
||||
import { Search } from "@opencode-ai/core/filesystem/search"
|
||||
import { Format } from "../../src/format"
|
||||
import { Reference } from "../../src/reference/reference"
|
||||
import { RepositoryCache } from "../../src/reference/repository-cache"
|
||||
|
|
@ -142,7 +142,7 @@ function makeHttp() {
|
|||
Layer.provide(RepositoryCache.defaultLayer),
|
||||
Layer.provide(Git.defaultLayer),
|
||||
Layer.provide(Reference.defaultLayer),
|
||||
Layer.provide(Ripgrep.defaultLayer),
|
||||
Layer.provide(Search.defaultLayer),
|
||||
Layer.provide(Format.defaultLayer),
|
||||
Layer.provide(RuntimeFlags.layer({ experimentalEventSystem: true })),
|
||||
Layer.provideMerge(todo),
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import { Cause, Effect, Exit, Layer } from "effect"
|
|||
import { GlobTool } from "../../src/tool/glob"
|
||||
import { SessionID, MessageID } from "../../src/session/schema"
|
||||
import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner"
|
||||
import { Ripgrep } from "@opencode-ai/core/filesystem/ripgrep"
|
||||
import { Search } from "@opencode-ai/core/filesystem/search"
|
||||
import { FSUtil } from "@opencode-ai/core/fs-util"
|
||||
import { Global } from "@opencode-ai/core/global"
|
||||
import { Truncate } from "@/tool/truncate"
|
||||
|
|
@ -17,6 +17,7 @@ import { RepositoryCache } from "@/reference/repository-cache"
|
|||
import { Config } from "@/config/config"
|
||||
import { RuntimeFlags } from "@/effect/runtime-flags"
|
||||
import { Git } from "@/git"
|
||||
import { Filesystem } from "@/util/filesystem"
|
||||
import { Permission } from "../../src/permission"
|
||||
import type * as Tool from "../../src/tool/tool"
|
||||
|
||||
|
|
@ -31,7 +32,7 @@ const toolLayer = (flags: Partial<RuntimeFlags.Info> = {}) =>
|
|||
Layer.mergeAll(
|
||||
CrossSpawnSpawner.defaultLayer,
|
||||
FSUtil.defaultLayer,
|
||||
Ripgrep.defaultLayer,
|
||||
Search.defaultLayer,
|
||||
Truncate.defaultLayer,
|
||||
Agent.defaultLayer,
|
||||
Git.defaultLayer,
|
||||
|
|
@ -40,6 +41,7 @@ const toolLayer = (flags: Partial<RuntimeFlags.Info> = {}) =>
|
|||
|
||||
const it = testEffect(toolLayer())
|
||||
const references = testEffect(toolLayer({ experimentalReferences: true }))
|
||||
const full = (p: string) => (process.platform === "win32" ? Filesystem.normalizePath(p) : p)
|
||||
|
||||
const ctx = {
|
||||
sessionID: SessionID.make("ses_test"),
|
||||
|
|
@ -172,7 +174,7 @@ describe("tool.glob", () => {
|
|||
)
|
||||
|
||||
expect(result.metadata.count).toBe(1)
|
||||
expect(result.output).toContain(path.join(cache, "src", "index.ts"))
|
||||
expect(full(result.output)).toContain(full(path.join(cache, "src", "index.ts")))
|
||||
expect(items.find((item) => item.permission === "external_directory")).toBeUndefined()
|
||||
}),
|
||||
{
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner"
|
|||
import { Global } from "@opencode-ai/core/global"
|
||||
import { Truncate } from "@/tool/truncate"
|
||||
import { Agent } from "../../src/agent/agent"
|
||||
import { Ripgrep } from "@opencode-ai/core/filesystem/ripgrep"
|
||||
import { Search } from "@opencode-ai/core/filesystem/search"
|
||||
import { FSUtil } from "@opencode-ai/core/fs-util"
|
||||
import { testEffect } from "../lib/effect"
|
||||
import { Reference } from "@/reference/reference"
|
||||
|
|
@ -34,7 +34,7 @@ const toolLayer = (flags: Partial<RuntimeFlags.Info> = {}) =>
|
|||
Layer.mergeAll(
|
||||
CrossSpawnSpawner.defaultLayer,
|
||||
FSUtil.defaultLayer,
|
||||
Ripgrep.defaultLayer,
|
||||
Search.defaultLayer,
|
||||
Truncate.defaultLayer,
|
||||
Agent.defaultLayer,
|
||||
Git.defaultLayer,
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import { FSUtil } from "@opencode-ai/core/fs-util"
|
|||
import { Global } from "@opencode-ai/core/global"
|
||||
import { Config } from "@/config/config"
|
||||
import { RuntimeFlags } from "@/effect/runtime-flags"
|
||||
import { Search } from "@opencode-ai/core/filesystem/search"
|
||||
import { LSP } from "@/lsp/lsp"
|
||||
import { Permission } from "../../src/permission"
|
||||
import { SessionID, MessageID } from "../../src/session/schema"
|
||||
|
|
@ -59,6 +60,7 @@ const readLayer = (flags: Partial<RuntimeFlags.Info> = {}) =>
|
|||
Instruction.defaultLayer,
|
||||
LSP.defaultLayer,
|
||||
referenceLayer(flags),
|
||||
Search.defaultLayer,
|
||||
Truncate.defaultLayer,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ import { Instruction } from "@/session/instruction"
|
|||
import { EventV2Bridge } from "@/event-v2-bridge"
|
||||
import { FetchHttpClient } from "effect/unstable/http"
|
||||
import { Format } from "@/format"
|
||||
import { Ripgrep } from "@opencode-ai/core/filesystem/ripgrep"
|
||||
import { Search } from "@opencode-ai/core/filesystem/search"
|
||||
import * as Truncate from "@/tool/truncate"
|
||||
import { InstanceState } from "@/effect/instance-state"
|
||||
import { Reference } from "@/reference/reference"
|
||||
|
|
@ -69,7 +69,7 @@ const registryLayer = (opts: RegistryLayerOptions = {}) =>
|
|||
Layer.provide(FetchHttpClient.layer),
|
||||
Layer.provide(Format.defaultLayer),
|
||||
Layer.provide(Layer.mergeAll(node, Database.defaultLayer)),
|
||||
Layer.provide(Ripgrep.defaultLayer),
|
||||
Layer.provide(Search.defaultLayer),
|
||||
Layer.provide(Truncate.defaultLayer),
|
||||
)
|
||||
.pipe(Layer.provide(RuntimeFlags.layer(opts.flags ?? {})))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue