refactor(core): unify filesystem search service (#31566)
This commit is contained in:
Dax
GitHub
parent
ce4e658e3f
commit
a0409e64d8
57 changed files with 962 additions and 2852 deletions
|
|
@ -6,8 +6,10 @@ import { Context, Effect, Layer, Option, Schema } from "effect"
|
|||
import { EventV2 } from "./event"
|
||||
import { FSUtil } from "./fs-util"
|
||||
import { Location } from "./location"
|
||||
import { NonNegativeInt, PositiveInt, RelativePath } from "./schema"
|
||||
import { Search } from "./filesystem/search"
|
||||
import { PositiveInt, RelativePath } from "./schema"
|
||||
import { FileSystemSearch } from "./filesystem/search"
|
||||
import { Entry, Match } from "./filesystem/schema"
|
||||
export { Entry, Match, Submatch } from "./filesystem/schema"
|
||||
|
||||
export const ReadInput = Schema.Struct({
|
||||
path: RelativePath,
|
||||
|
|
@ -28,39 +30,23 @@ export const ListInput = Schema.Struct({
|
|||
})
|
||||
export type ListInput = typeof ListInput.Type
|
||||
|
||||
export class Entry extends Schema.Class<Entry>("FileSystem.Entry")({
|
||||
path: RelativePath,
|
||||
uri: Schema.String,
|
||||
type: Schema.Literals(["file", "directory"]),
|
||||
mime: Schema.String,
|
||||
}) {}
|
||||
|
||||
export const FindInput = Schema.Struct({
|
||||
export class FindInput extends Schema.Class<FindInput>("FileSystem.FindInput")({
|
||||
query: Schema.String,
|
||||
type: Schema.Literals(["file", "directory"]).pipe(Schema.optional),
|
||||
limit: PositiveInt.pipe(Schema.optional),
|
||||
})
|
||||
export type FindInput = typeof FindInput.Type
|
||||
}) {}
|
||||
|
||||
export const GrepInput = Schema.Struct({
|
||||
export class GlobInput extends Schema.Class<GlobInput>("FileSystem.GlobInput")({
|
||||
pattern: Schema.String,
|
||||
path: RelativePath.pipe(Schema.optional),
|
||||
limit: PositiveInt.pipe(Schema.optional),
|
||||
}) {}
|
||||
|
||||
export class GrepInput extends Schema.Class<GrepInput>("FileSystem.GrepInput")({
|
||||
pattern: Schema.String,
|
||||
path: RelativePath.pipe(Schema.optional),
|
||||
include: Schema.String.pipe(Schema.optional),
|
||||
limit: PositiveInt.pipe(Schema.optional),
|
||||
})
|
||||
export type GrepInput = typeof GrepInput.Type
|
||||
|
||||
export class GrepMatch extends Schema.Class<GrepMatch>("LocationFileSystem.GrepMatch")({
|
||||
path: RelativePath,
|
||||
lines: Schema.String,
|
||||
line: PositiveInt,
|
||||
offset: NonNegativeInt,
|
||||
submatches: Schema.Array(
|
||||
Schema.Struct({
|
||||
text: Schema.String,
|
||||
start: NonNegativeInt,
|
||||
end: NonNegativeInt,
|
||||
}),
|
||||
),
|
||||
}) {}
|
||||
|
||||
export const Event = {
|
||||
|
|
@ -76,17 +62,18 @@ export interface Interface {
|
|||
readonly read: (input: ReadInput) => Effect.Effect<Content>
|
||||
readonly list: (input?: ListInput) => Effect.Effect<Entry[]>
|
||||
readonly find: (input: FindInput) => Effect.Effect<Entry[]>
|
||||
readonly grep: (input: GrepInput) => Effect.Effect<GrepMatch[]>
|
||||
readonly glob: (input: GlobInput) => Effect.Effect<readonly Entry[]>
|
||||
readonly grep: (input: GrepInput) => Effect.Effect<readonly Match[]>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/v2/FileSystem") {}
|
||||
|
||||
export const layer = Layer.effect(
|
||||
const baseLayer = Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const fs = yield* FSUtil.Service
|
||||
const location = yield* Location.Service
|
||||
const search = yield* Search.Service
|
||||
const search = yield* FileSystemSearch.Service
|
||||
const root = yield* fs.realPath(location.directory).pipe(Effect.orDie)
|
||||
const resolve = Effect.fnUntraced(function* (input?: RelativePath) {
|
||||
const absolute = path.resolve(location.directory, input ?? ".")
|
||||
|
|
@ -96,22 +83,10 @@ export const layer = Layer.effect(
|
|||
if (!FSUtil.contains(root, real)) return yield* Effect.die(new Error("Path escapes the location"))
|
||||
return { absolute, real, directory: location.directory, root }
|
||||
})
|
||||
const entry = Effect.fnUntraced(function* (absolute: string, selected = { directory: location.directory, root }) {
|
||||
const real = yield* fs.realPath(absolute).pipe(Effect.catch(() => Effect.void))
|
||||
if (!real) return
|
||||
if (!FSUtil.contains(selected.root, real)) return
|
||||
const info = yield* fs.stat(real).pipe(Effect.catch(() => Effect.void))
|
||||
const type = info?.type === "Directory" ? "directory" : info?.type === "File" ? "file" : undefined
|
||||
if (!type) return
|
||||
return new Entry({
|
||||
path: RelativePath.make(path.relative(selected.directory, absolute)),
|
||||
uri: pathToFileURL(real).href,
|
||||
type,
|
||||
mime: type === "directory" ? "application/x-directory" : FSUtil.mimeType(real),
|
||||
})
|
||||
})
|
||||
|
||||
return Service.of({
|
||||
find: search.find,
|
||||
glob: search.glob,
|
||||
grep: search.grep,
|
||||
read: Effect.fn("FileSystem.read")(function* (input) {
|
||||
const target = yield* resolve(input.path)
|
||||
const info = yield* fs.stat(target.real).pipe(Effect.orDie)
|
||||
|
|
@ -145,62 +120,28 @@ export const layer = Layer.effect(
|
|||
if (info.type !== "Directory") return yield* Effect.die(new Error("Path is not a directory"))
|
||||
return yield* fs.readDirectoryEntries(target.real).pipe(
|
||||
Effect.orDie,
|
||||
Effect.flatMap((items) =>
|
||||
Effect.forEach(items, (item) => entry(path.join(target.absolute, item.name), target), {
|
||||
concurrency: "unbounded",
|
||||
}),
|
||||
),
|
||||
Effect.map((items) =>
|
||||
items
|
||||
.filter((item): item is Entry => item !== undefined)
|
||||
.flatMap((item) => {
|
||||
if (item.type !== "file" && item.type !== "directory") return []
|
||||
const absolute = path.join(target.absolute, item.name)
|
||||
const relative = path.relative(target.directory, absolute)
|
||||
return [
|
||||
new Entry({
|
||||
path: RelativePath.make(relative + (item.type === "directory" ? path.sep : "")),
|
||||
type: item.type,
|
||||
mime: item.type === "directory" ? "application/x-directory" : FSUtil.mimeType(absolute),
|
||||
}),
|
||||
]
|
||||
})
|
||||
.sort((a, b) => (a.type === b.type ? a.path.localeCompare(b.path) : a.type === "directory" ? -1 : 1)),
|
||||
),
|
||||
)
|
||||
}),
|
||||
find: Effect.fn("FileSystem.find")(function* (input) {
|
||||
const found = yield* search
|
||||
.file({
|
||||
cwd: location.directory,
|
||||
query: input.query,
|
||||
limit: input.limit,
|
||||
kind: input.type ?? "all",
|
||||
})
|
||||
.pipe(Effect.orDie)
|
||||
return found.map(
|
||||
(item) =>
|
||||
new Entry({
|
||||
path: RelativePath.make(item.path),
|
||||
uri: pathToFileURL(path.join(location.directory, item.path)).href,
|
||||
type: item.type,
|
||||
mime: item.type === "directory" ? "application/x-directory" : FSUtil.mimeType(item.path),
|
||||
}),
|
||||
)
|
||||
}),
|
||||
grep: Effect.fn("FileSystem.grep")(function* (input) {
|
||||
return (yield* search
|
||||
.search({
|
||||
cwd: location.directory,
|
||||
pattern: input.pattern,
|
||||
glob: input.include ? [input.include] : undefined,
|
||||
limit: input.limit,
|
||||
})
|
||||
.pipe(Effect.orDie)).items.map(
|
||||
(item) =>
|
||||
new GrepMatch({
|
||||
path: RelativePath.make(item.path.text),
|
||||
lines: item.lines.text,
|
||||
line: item.line_number,
|
||||
offset: item.absolute_offset,
|
||||
submatches: item.submatches.map((submatch) => ({
|
||||
text: submatch.match.text,
|
||||
start: submatch.start,
|
||||
end: submatch.end,
|
||||
})),
|
||||
}),
|
||||
)
|
||||
}),
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
export const layer = baseLayer.pipe(Layer.provide(FileSystemSearch.defaultLayer), Layer.provide(FSUtil.defaultLayer))
|
||||
|
||||
export const locationLayer = layer
|
||||
|
|
|
|||
|
|
@ -1,487 +0,0 @@
|
|||
import path from "path"
|
||||
import { serviceUse } from "../effect/service-use"
|
||||
import { FSUtil } from "../fs-util"
|
||||
import { Cause, Context, Effect, Fiber, Layer, Queue, Schema, Stream } from "effect"
|
||||
import type { PlatformError } from "effect/PlatformError"
|
||||
import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http"
|
||||
import { ChildProcess } from "effect/unstable/process"
|
||||
import { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner"
|
||||
|
||||
import { CrossSpawnSpawner } from "../cross-spawn-spawner"
|
||||
import { Global } from "../global"
|
||||
import { NonNegativeInt } from "../schema"
|
||||
import { which } from "../util/which"
|
||||
import { LayerNode } from "../effect/layer-node"
|
||||
import { httpClient } from "../effect/layer-node-platform"
|
||||
|
||||
const VERSION = "15.1.0"
|
||||
const PLATFORM = {
|
||||
"arm64-darwin": { platform: "aarch64-apple-darwin", extension: "tar.gz" },
|
||||
"arm64-linux": { platform: "aarch64-unknown-linux-gnu", extension: "tar.gz" },
|
||||
"x64-darwin": { platform: "x86_64-apple-darwin", extension: "tar.gz" },
|
||||
"x64-linux": { platform: "x86_64-unknown-linux-musl", extension: "tar.gz" },
|
||||
"arm64-win32": { platform: "aarch64-pc-windows-msvc", extension: "zip" },
|
||||
"ia32-win32": { platform: "i686-pc-windows-msvc", extension: "zip" },
|
||||
"x64-win32": { platform: "x86_64-pc-windows-msvc", extension: "zip" },
|
||||
} as const
|
||||
|
||||
const TimeStats = Schema.Struct({
|
||||
secs: NonNegativeInt,
|
||||
nanos: NonNegativeInt,
|
||||
human: Schema.String,
|
||||
})
|
||||
|
||||
const Stats = Schema.Struct({
|
||||
elapsed: TimeStats,
|
||||
searches: NonNegativeInt,
|
||||
searches_with_match: NonNegativeInt,
|
||||
bytes_searched: NonNegativeInt,
|
||||
bytes_printed: NonNegativeInt,
|
||||
matched_lines: NonNegativeInt,
|
||||
matches: NonNegativeInt,
|
||||
})
|
||||
|
||||
const PathText = Schema.Struct({
|
||||
text: Schema.String,
|
||||
})
|
||||
|
||||
const Begin = Schema.Struct({
|
||||
type: Schema.Literal("begin"),
|
||||
data: Schema.Struct({
|
||||
path: PathText,
|
||||
}),
|
||||
})
|
||||
|
||||
export const SearchMatch = Schema.Struct({
|
||||
path: PathText,
|
||||
lines: Schema.Struct({
|
||||
text: Schema.String,
|
||||
}),
|
||||
line_number: NonNegativeInt,
|
||||
absolute_offset: NonNegativeInt,
|
||||
submatches: Schema.Array(
|
||||
Schema.Struct({
|
||||
match: Schema.Struct({
|
||||
text: Schema.String,
|
||||
}),
|
||||
start: NonNegativeInt,
|
||||
end: NonNegativeInt,
|
||||
}),
|
||||
),
|
||||
})
|
||||
|
||||
export const Match = Schema.Struct({
|
||||
type: Schema.Literal("match"),
|
||||
data: SearchMatch,
|
||||
})
|
||||
|
||||
const End = Schema.Struct({
|
||||
type: Schema.Literal("end"),
|
||||
data: Schema.Struct({
|
||||
path: PathText,
|
||||
binary_offset: Schema.NullOr(NonNegativeInt),
|
||||
stats: Stats,
|
||||
}),
|
||||
})
|
||||
|
||||
const Summary = Schema.Struct({
|
||||
type: Schema.Literal("summary"),
|
||||
data: Schema.Struct({
|
||||
elapsed_total: TimeStats,
|
||||
stats: Stats,
|
||||
}),
|
||||
})
|
||||
|
||||
const Result = Schema.Union([Begin, Match, End, Summary])
|
||||
const decodeResult = Schema.decodeUnknownEffect(Schema.fromJsonString(Result))
|
||||
|
||||
export type Result = Schema.Schema.Type<typeof Result>
|
||||
export type Match = Schema.Schema.Type<typeof Match>
|
||||
export type Item = Match["data"]
|
||||
export type Begin = Schema.Schema.Type<typeof Begin>
|
||||
export type End = Schema.Schema.Type<typeof End>
|
||||
export type Summary = Schema.Schema.Type<typeof Summary>
|
||||
export type Row = Match["data"]
|
||||
|
||||
export interface SearchResult {
|
||||
items: Item[]
|
||||
partial: boolean
|
||||
}
|
||||
|
||||
export interface FilesInput {
|
||||
cwd: string
|
||||
glob?: string[]
|
||||
hidden?: boolean
|
||||
follow?: boolean
|
||||
maxDepth?: number
|
||||
signal?: AbortSignal
|
||||
}
|
||||
|
||||
export interface SearchInput {
|
||||
cwd: string
|
||||
pattern: string
|
||||
glob?: string[]
|
||||
limit?: number
|
||||
follow?: boolean
|
||||
file?: string[]
|
||||
signal?: AbortSignal
|
||||
}
|
||||
|
||||
export interface TreeInput {
|
||||
cwd: string
|
||||
limit?: number
|
||||
signal?: AbortSignal
|
||||
}
|
||||
|
||||
export interface Interface {
|
||||
readonly filepath: Effect.Effect<string, Error>
|
||||
readonly files: (input: FilesInput) => Stream.Stream<string, PlatformError | Error>
|
||||
readonly tree: (input: TreeInput) => Effect.Effect<string, PlatformError | Error>
|
||||
readonly search: (input: SearchInput) => Effect.Effect<SearchResult, PlatformError | Error>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/Ripgrep") {}
|
||||
|
||||
export const use = serviceUse(Service)
|
||||
|
||||
function env() {
|
||||
const env = Object.fromEntries(
|
||||
Object.entries(process.env).filter((entry): entry is [string, string] => entry[1] !== undefined),
|
||||
)
|
||||
delete env.RIPGREP_CONFIG_PATH
|
||||
return env
|
||||
}
|
||||
|
||||
function aborted(signal?: AbortSignal) {
|
||||
const err = signal?.reason
|
||||
if (err instanceof Error) return err
|
||||
const out = new Error("Aborted")
|
||||
out.name = "AbortError"
|
||||
return out
|
||||
}
|
||||
|
||||
function waitForAbort(signal?: AbortSignal) {
|
||||
if (!signal) return Effect.never
|
||||
if (signal.aborted) return Effect.fail(aborted(signal))
|
||||
return Effect.callback<never, Error>((resume) => {
|
||||
const onabort = () => resume(Effect.fail(aborted(signal)))
|
||||
signal.addEventListener("abort", onabort, { once: true })
|
||||
return Effect.sync(() => signal.removeEventListener("abort", onabort))
|
||||
})
|
||||
}
|
||||
|
||||
function error(stderr: string, code: number) {
|
||||
const err = new Error(stderr.trim() || `ripgrep failed with code ${code}`)
|
||||
err.name = "RipgrepError"
|
||||
return err
|
||||
}
|
||||
|
||||
function clean(file: string) {
|
||||
return path.normalize(file.replace(/^\.[\\/]/, ""))
|
||||
}
|
||||
|
||||
function row(data: Row): Row {
|
||||
return {
|
||||
...data,
|
||||
path: {
|
||||
...data.path,
|
||||
text: clean(data.path.text),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function parse(line: string) {
|
||||
return decodeResult(line).pipe(Effect.mapError((cause) => new Error("invalid ripgrep output", { cause })))
|
||||
}
|
||||
|
||||
function fail(queue: Queue.Queue<string, PlatformError | Error | Cause.Done>, err: PlatformError | Error) {
|
||||
Queue.failCauseUnsafe(queue, Cause.fail(err))
|
||||
}
|
||||
|
||||
function filesArgs(input: FilesInput) {
|
||||
const args = ["--no-config", "--files", "--glob=!.git/*"]
|
||||
if (input.follow) args.push("--follow")
|
||||
if (input.hidden !== false) args.push("--hidden")
|
||||
if (input.hidden === false) args.push("--glob=!.*")
|
||||
if (input.maxDepth !== undefined) args.push(`--max-depth=${input.maxDepth}`)
|
||||
if (input.glob) {
|
||||
for (const glob of input.glob) args.push(`--glob=${glob}`)
|
||||
}
|
||||
args.push(".")
|
||||
return args
|
||||
}
|
||||
|
||||
function searchArgs(input: SearchInput) {
|
||||
const args = ["--no-config", "--json", "--hidden", "--glob=!.git/*", "--no-messages"]
|
||||
if (input.follow) args.push("--follow")
|
||||
if (input.glob) {
|
||||
for (const glob of input.glob) args.push(`--glob=${glob}`)
|
||||
}
|
||||
if (input.limit) args.push(`--max-count=${input.limit}`)
|
||||
args.push("--", input.pattern, ...(input.file ?? ["."]))
|
||||
return args
|
||||
}
|
||||
|
||||
function raceAbort<A, E, R>(effect: Effect.Effect<A, E, R>, signal?: AbortSignal) {
|
||||
return signal ? effect.pipe(Effect.raceFirst(waitForAbort(signal))) : effect
|
||||
}
|
||||
|
||||
export const layer: Layer.Layer<Service, never, FSUtil.Service | ChildProcessSpawner | HttpClient.HttpClient> =
|
||||
Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const fs = yield* FSUtil.Service
|
||||
const http = HttpClient.filterStatusOk(yield* HttpClient.HttpClient)
|
||||
const spawner = yield* ChildProcessSpawner
|
||||
|
||||
const run = Effect.fnUntraced(function* (command: string, args: string[], opts?: { cwd?: string }) {
|
||||
const handle = yield* spawner.spawn(
|
||||
ChildProcess.make(command, args, { cwd: opts?.cwd, extendEnv: true, stdin: "ignore" }),
|
||||
)
|
||||
const [stdout, stderr, code] = yield* Effect.all(
|
||||
[
|
||||
Stream.mkString(Stream.decodeText(handle.stdout)),
|
||||
Stream.mkString(Stream.decodeText(handle.stderr)),
|
||||
handle.exitCode,
|
||||
],
|
||||
{ concurrency: "unbounded" },
|
||||
)
|
||||
return { stdout, stderr, code }
|
||||
}, Effect.scoped)
|
||||
|
||||
const extract = Effect.fnUntraced(function* (
|
||||
archive: string,
|
||||
config: (typeof PLATFORM)[keyof typeof PLATFORM],
|
||||
target: string,
|
||||
) {
|
||||
const dir = yield* fs.makeTempDirectoryScoped({ directory: Global.Path.bin, prefix: "ripgrep-" })
|
||||
|
||||
if (config.extension === "zip") {
|
||||
const shell = (yield* Effect.sync(() => which("powershell.exe") ?? which("pwsh.exe"))) ?? "powershell.exe"
|
||||
const result = yield* run(shell, [
|
||||
"-NoProfile",
|
||||
"-NonInteractive",
|
||||
"-Command",
|
||||
`$global:ProgressPreference = 'SilentlyContinue'; Expand-Archive -LiteralPath '${archive.replaceAll("'", "''")}' -DestinationPath '${dir.replaceAll("'", "''")}' -Force`,
|
||||
])
|
||||
if (result.code !== 0) {
|
||||
return yield* Effect.fail(error(result.stderr || result.stdout, result.code))
|
||||
}
|
||||
}
|
||||
|
||||
if (config.extension === "tar.gz") {
|
||||
const result = yield* run("tar", ["-xzf", archive, "-C", dir])
|
||||
if (result.code !== 0) {
|
||||
return yield* Effect.fail(error(result.stderr || result.stdout, result.code))
|
||||
}
|
||||
}
|
||||
|
||||
const extracted = path.join(
|
||||
dir,
|
||||
`ripgrep-${VERSION}-${config.platform}`,
|
||||
process.platform === "win32" ? "rg.exe" : "rg",
|
||||
)
|
||||
if (!(yield* fs.isFile(extracted))) {
|
||||
return yield* Effect.fail(new Error(`ripgrep archive did not contain executable: ${extracted}`))
|
||||
}
|
||||
|
||||
yield* fs.copyFile(extracted, target)
|
||||
if (process.platform === "win32") return
|
||||
yield* fs.chmod(target, 0o755)
|
||||
}, Effect.scoped)
|
||||
|
||||
const filepath = yield* Effect.cached(
|
||||
Effect.gen(function* () {
|
||||
const system = yield* Effect.sync(() => which(process.platform === "win32" ? "rg.exe" : "rg"))
|
||||
if (system && (yield* fs.isFile(system).pipe(Effect.orDie))) return system
|
||||
|
||||
const target = path.join(Global.Path.bin, `rg${process.platform === "win32" ? ".exe" : ""}`)
|
||||
if (yield* fs.isFile(target).pipe(Effect.orDie)) return target
|
||||
|
||||
const platformKey = `${process.arch}-${process.platform}` as keyof typeof PLATFORM
|
||||
const config = PLATFORM[platformKey]
|
||||
if (!config) {
|
||||
return yield* Effect.fail(new Error(`unsupported platform for ripgrep: ${platformKey}`))
|
||||
}
|
||||
|
||||
const filename = `ripgrep-${VERSION}-${config.platform}.${config.extension}`
|
||||
const url = `https://github.com/BurntSushi/ripgrep/releases/download/${VERSION}/${filename}`
|
||||
const archive = path.join(Global.Path.bin, filename)
|
||||
|
||||
yield* Effect.logInfo("downloading ripgrep", { url })
|
||||
yield* fs.ensureDir(Global.Path.bin).pipe(Effect.orDie)
|
||||
|
||||
const bytes = yield* HttpClientRequest.get(url).pipe(
|
||||
http.execute,
|
||||
Effect.flatMap((response) => response.arrayBuffer),
|
||||
Effect.mapError((cause) => (cause instanceof Error ? cause : new Error(String(cause)))),
|
||||
)
|
||||
if (bytes.byteLength === 0) {
|
||||
return yield* Effect.fail(new Error(`failed to download ripgrep from ${url}`))
|
||||
}
|
||||
|
||||
yield* fs.writeWithDirs(archive, new Uint8Array(bytes))
|
||||
yield* extract(archive, config, target)
|
||||
yield* fs.remove(archive, { force: true }).pipe(Effect.ignore)
|
||||
return target
|
||||
}),
|
||||
)
|
||||
|
||||
const check = Effect.fnUntraced(function* (cwd: string) {
|
||||
if (yield* fs.isDir(cwd).pipe(Effect.orDie)) return
|
||||
return yield* Effect.fail(
|
||||
Object.assign(new Error(`No such file or directory: '${cwd}'`), {
|
||||
code: "ENOENT",
|
||||
errno: -2,
|
||||
path: cwd,
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
const command = Effect.fnUntraced(function* (cwd: string, args: string[]) {
|
||||
const binary = yield* filepath
|
||||
return ChildProcess.make(binary, args, {
|
||||
cwd,
|
||||
env: env(),
|
||||
extendEnv: true,
|
||||
stdin: "ignore",
|
||||
})
|
||||
})
|
||||
|
||||
const files: Interface["files"] = (input) =>
|
||||
Stream.callback<string, PlatformError | Error>((queue) =>
|
||||
Effect.gen(function* () {
|
||||
yield* Effect.forkScoped(
|
||||
Effect.gen(function* () {
|
||||
yield* check(input.cwd)
|
||||
const handle = yield* spawner.spawn(yield* command(input.cwd, filesArgs(input)))
|
||||
const stderr = yield* Stream.mkString(Stream.decodeText(handle.stderr)).pipe(Effect.forkScoped)
|
||||
const stdout = yield* Stream.decodeText(handle.stdout).pipe(
|
||||
Stream.splitLines,
|
||||
Stream.filter((line) => line.length > 0),
|
||||
Stream.runForEach((line) => Effect.sync(() => Queue.offerUnsafe(queue, clean(line)))),
|
||||
Effect.forkScoped,
|
||||
)
|
||||
const code = yield* raceAbort(handle.exitCode, input.signal)
|
||||
yield* Fiber.join(stdout)
|
||||
if (code === 0 || code === 1) {
|
||||
Queue.endUnsafe(queue)
|
||||
return
|
||||
}
|
||||
fail(queue, error(yield* Fiber.join(stderr), code))
|
||||
}).pipe(
|
||||
Effect.catch((err) =>
|
||||
Effect.sync(() => {
|
||||
fail(queue, err)
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
const search: Interface["search"] = Effect.fn("Ripgrep.search")(function* (input: SearchInput) {
|
||||
yield* check(input.cwd)
|
||||
|
||||
const program = Effect.scoped(
|
||||
Effect.gen(function* () {
|
||||
const handle = yield* spawner.spawn(yield* command(input.cwd, searchArgs(input)))
|
||||
|
||||
const [items, stderr, code] = yield* Effect.all(
|
||||
[
|
||||
Stream.decodeText(handle.stdout).pipe(
|
||||
Stream.splitLines,
|
||||
Stream.filter((line) => line.length > 0),
|
||||
Stream.mapEffect(parse),
|
||||
Stream.filter((item): item is Match => item.type === "match"),
|
||||
Stream.map((item) => row(item.data)),
|
||||
Stream.runCollect,
|
||||
Effect.map((chunk) => [...chunk]),
|
||||
),
|
||||
Stream.mkString(Stream.decodeText(handle.stderr)),
|
||||
handle.exitCode,
|
||||
],
|
||||
{ concurrency: "unbounded" },
|
||||
)
|
||||
|
||||
if (code !== 0 && code !== 1 && code !== 2) {
|
||||
return yield* Effect.fail(error(stderr, code))
|
||||
}
|
||||
|
||||
return {
|
||||
items: code === 1 ? [] : items,
|
||||
partial: code === 2,
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
return yield* raceAbort(program, input.signal)
|
||||
})
|
||||
|
||||
const tree: Interface["tree"] = Effect.fn("Ripgrep.tree")(function* (input: TreeInput) {
|
||||
yield* Effect.logInfo("tree", input)
|
||||
const list = Array.from(yield* files({ cwd: input.cwd, signal: input.signal }).pipe(Stream.runCollect))
|
||||
|
||||
interface Node {
|
||||
name: string
|
||||
children: Map<string, Node>
|
||||
}
|
||||
|
||||
function child(node: Node, name: string) {
|
||||
const item = node.children.get(name)
|
||||
if (item) return item
|
||||
const next = { name, children: new Map() }
|
||||
node.children.set(name, next)
|
||||
return next
|
||||
}
|
||||
|
||||
function count(node: Node): number {
|
||||
return Array.from(node.children.values()).reduce((sum, child) => sum + 1 + count(child), 0)
|
||||
}
|
||||
|
||||
const root: Node = { name: "", children: new Map() }
|
||||
for (const file of list) {
|
||||
if (file.includes(".opencode")) continue
|
||||
const parts = file.split(path.sep)
|
||||
if (parts.length < 2) continue
|
||||
let node = root
|
||||
for (const part of parts.slice(0, -1)) {
|
||||
node = child(node, part)
|
||||
}
|
||||
}
|
||||
|
||||
const total = count(root)
|
||||
const limit = input.limit ?? total
|
||||
const lines: string[] = []
|
||||
const queue: Array<{ node: Node; path: string }> = Array.from(root.children.values())
|
||||
.sort((a, b) => a.name.localeCompare(b.name))
|
||||
.map((node) => ({ node, path: node.name }))
|
||||
|
||||
let used = 0
|
||||
for (let i = 0; i < queue.length && used < limit; i++) {
|
||||
const item = queue[i]
|
||||
lines.push(item.path)
|
||||
used++
|
||||
queue.push(
|
||||
...Array.from(item.node.children.values())
|
||||
.sort((a, b) => a.name.localeCompare(b.name))
|
||||
.map((node) => ({ node, path: `${item.path}/${node.name}` })),
|
||||
)
|
||||
}
|
||||
|
||||
if (total > used) lines.push(`[${total - used} truncated]`)
|
||||
return lines.join("\n")
|
||||
})
|
||||
|
||||
return Service.of({ filepath, files, tree, search })
|
||||
}),
|
||||
)
|
||||
|
||||
export const defaultLayer = layer.pipe(
|
||||
Layer.provide(FetchHttpClient.layer),
|
||||
Layer.provide(FSUtil.defaultLayer),
|
||||
Layer.provide(CrossSpawnSpawner.defaultLayer),
|
||||
)
|
||||
export const node = LayerNode.make(layer, [FSUtil.node, CrossSpawnSpawner.node, httpClient])
|
||||
|
||||
export * as Ripgrep from "./ripgrep"
|
||||
23
packages/core/src/filesystem/schema.ts
Normal file
23
packages/core/src/filesystem/schema.ts
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
import { Schema } from "effect"
|
||||
import { NonNegativeInt, PositiveInt, RelativePath } from "../schema"
|
||||
|
||||
export class Entry extends Schema.Class<Entry>("FileSystem.Entry")({
|
||||
path: RelativePath,
|
||||
type: Schema.Literals(["file", "directory"]),
|
||||
mime: Schema.String,
|
||||
}) {}
|
||||
|
||||
export const Submatch = Schema.Struct({
|
||||
text: Schema.String,
|
||||
start: NonNegativeInt,
|
||||
end: NonNegativeInt,
|
||||
})
|
||||
export type Submatch = typeof Submatch.Type
|
||||
|
||||
export class Match extends Schema.Class<Match>("FileSystem.Match")({
|
||||
entry: Entry,
|
||||
line: PositiveInt,
|
||||
offset: NonNegativeInt,
|
||||
text: Schema.String,
|
||||
submatches: Schema.Array(Submatch),
|
||||
}) {}
|
||||
|
|
@ -1,567 +1,230 @@
|
|||
export * as FileSystemSearch from "./search"
|
||||
|
||||
import path from "path"
|
||||
import { Context, Deferred, Effect, Layer, Option, Stream } from "effect"
|
||||
import type { PlatformError } from "effect/PlatformError"
|
||||
import { FSUtil } from "../fs-util"
|
||||
import { Glob } from "../util/glob"
|
||||
import { Global } from "../global"
|
||||
import { serviceUse } from "../effect/service-use"
|
||||
import { makeRuntime } from "../effect/runtime"
|
||||
import { Context, Effect, Fiber, Layer, Scope } from "effect"
|
||||
import { Fff } from "#fff"
|
||||
import { Ripgrep } from "./ripgrep"
|
||||
import { LayerNode } from "../effect/layer-node"
|
||||
|
||||
const root = path.join(Global.Path.cache, "fff")
|
||||
|
||||
export type Item = Ripgrep.Item
|
||||
export type SearchError = PlatformError | globalThis.Error
|
||||
|
||||
export interface Result {
|
||||
readonly items: Item[]
|
||||
readonly partial: boolean
|
||||
readonly hasNextPage: boolean
|
||||
readonly engine: "fff" | "ripgrep"
|
||||
readonly regexFallbackError?: string
|
||||
}
|
||||
|
||||
export interface FileInput {
|
||||
readonly cwd: string
|
||||
readonly query: string
|
||||
readonly limit?: number
|
||||
readonly current?: string
|
||||
readonly kind?: "file" | "directory" | "all"
|
||||
}
|
||||
|
||||
export interface FileResult {
|
||||
readonly path: string
|
||||
readonly type: "file" | "directory"
|
||||
}
|
||||
|
||||
export interface GlobInput {
|
||||
readonly cwd: string
|
||||
readonly pattern: string
|
||||
readonly limit?: number
|
||||
readonly signal?: AbortSignal
|
||||
}
|
||||
|
||||
interface Query {
|
||||
readonly dir: string
|
||||
readonly text: string
|
||||
readonly files: string[]
|
||||
}
|
||||
|
||||
// A created picker plus its cached scan-readiness gate. The picker is created
|
||||
// (and its native background scan kicked off) eagerly; `ready` is only awaited
|
||||
// when the picker is actually used.
|
||||
interface Picker {
|
||||
readonly pick: Fff.Picker
|
||||
readonly ready: Effect.Effect<void, Error>
|
||||
}
|
||||
|
||||
interface State {
|
||||
readonly pick: Map<string, Picker>
|
||||
readonly wait: Map<string, Deferred.Deferred<Picker, Error>>
|
||||
readonly recent: Query[]
|
||||
}
|
||||
import fuzzysort from "fuzzysort"
|
||||
import { FileSystem } from "../filesystem"
|
||||
import { FSUtil } from "../fs-util"
|
||||
import { Location } from "../location"
|
||||
import { Ripgrep } from "../ripgrep"
|
||||
import { RelativePath } from "../schema"
|
||||
|
||||
export interface Interface {
|
||||
readonly files: Ripgrep.Interface["files"]
|
||||
readonly tree: Ripgrep.Interface["tree"]
|
||||
readonly search: (input: Ripgrep.SearchInput) => Effect.Effect<Result, SearchError>
|
||||
readonly file: (input: FileInput) => Effect.Effect<readonly FileResult[], SearchError>
|
||||
readonly glob: (input: GlobInput) => Effect.Effect<{ files: string[]; truncated: boolean }, SearchError>
|
||||
readonly open: (input: { cwd?: string; file: string }) => Effect.Effect<void, SearchError>
|
||||
readonly warm: (cwd: string) => Effect.Effect<void>
|
||||
// Destroy the picker for a directory and drop its cached state. Called when a
|
||||
// directory's instance is disposed so fff's native watcher thread is torn
|
||||
// down instead of leaking until process exit.
|
||||
readonly release: (cwd: string) => Effect.Effect<void>
|
||||
readonly find: (input: FileSystem.FindInput) => Effect.Effect<FileSystem.Entry[]>
|
||||
readonly glob: (input: FileSystem.GlobInput) => Effect.Effect<readonly FileSystem.Entry[]>
|
||||
readonly grep: (input: FileSystem.GrepInput) => Effect.Effect<readonly FileSystem.Match[]>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/Search") {}
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/v2/FileSystem/Search") {}
|
||||
|
||||
export const use = serviceUse(Service)
|
||||
|
||||
function key(dir: string) {
|
||||
return Buffer.from(dir).toString("base64url")
|
||||
}
|
||||
|
||||
function fffSync<A>(action: string, run: () => A) {
|
||||
return Effect.try({
|
||||
try: run,
|
||||
catch: (cause) => new Error(`fff ${action} failed`, { cause }),
|
||||
})
|
||||
}
|
||||
|
||||
function normalize(text: string) {
|
||||
return text.replaceAll("\\", "/")
|
||||
}
|
||||
|
||||
// fff supports glob narrowing for any search out of the box
|
||||
function fffGlobbedQuery(query: string, glob?: string | string[]) {
|
||||
if (query && glob) {
|
||||
const resolvedGlob = Array.isArray(glob) ? glob.join(" ") : glob
|
||||
return `${resolvedGlob} ${query}`
|
||||
}
|
||||
|
||||
return query ?? glob
|
||||
}
|
||||
|
||||
function remember(state: State, dir: string, text: string, files: string[]) {
|
||||
if (!files.length) return
|
||||
const next = Array.from(new Set(files.map(FSUtil.resolve))).slice(0, 64)
|
||||
if (!next.length) return
|
||||
const idx = state.recent.findIndex((item) => item.dir === dir && item.text === text)
|
||||
if (idx >= 0) state.recent.splice(idx, 1)
|
||||
state.recent.unshift({ dir, text, files: next })
|
||||
if (state.recent.length > 32) state.recent.length = 32
|
||||
}
|
||||
|
||||
function item(hit: Fff.Hit): Item {
|
||||
const line = Buffer.from(hit.lineContent)
|
||||
return {
|
||||
path: { text: normalize(hit.relativePath) },
|
||||
lines: { text: hit.lineContent },
|
||||
line_number: hit.lineNumber,
|
||||
absolute_offset: hit.byteOffset,
|
||||
submatches: hit.matchRanges
|
||||
.map(([start, end]) => {
|
||||
const text = line.subarray(start, end).toString("utf8")
|
||||
if (!text) return undefined
|
||||
return {
|
||||
match: { text },
|
||||
start,
|
||||
end,
|
||||
}
|
||||
})
|
||||
.filter((row): row is Item["submatches"][number] => Boolean(row)),
|
||||
}
|
||||
}
|
||||
|
||||
function collectPaths<T>(
|
||||
items: T[],
|
||||
scores: Array<{ total: number }>,
|
||||
toResult: (item: T) => FileResult,
|
||||
): FileResult[] {
|
||||
const rows = items.flatMap((item, index): Array<FileResult & { score: number }> => {
|
||||
const result = toResult(item)
|
||||
if (!result.path) return []
|
||||
return [{ ...result, score: scores[index]?.total ?? 0 }]
|
||||
})
|
||||
rows.sort(
|
||||
(a, b) => b.score - a.score || a.path.length - b.path.length || (a.path < b.path ? -1 : a.path > b.path ? 1 : 0),
|
||||
)
|
||||
|
||||
const seen = new Set<string>()
|
||||
return rows.flatMap((item) => {
|
||||
if (seen.has(item.path)) return []
|
||||
seen.add(item.path)
|
||||
return [{ path: item.path, type: item.type }]
|
||||
})
|
||||
}
|
||||
|
||||
function searchFff(
|
||||
pick: Fff.Picker,
|
||||
kind: "file" | "directory" | "all",
|
||||
query: string,
|
||||
opts: { currentFile?: string; pageIndex?: number; pageSize?: number },
|
||||
): Fff.Result<FileResult[]> {
|
||||
if (kind === "directory") {
|
||||
const out = pick.directorySearch(query, opts)
|
||||
if (!out.ok) return out
|
||||
return {
|
||||
ok: true,
|
||||
value: collectPaths(out.value.items, out.value.scores, (entry) => ({
|
||||
path: normalize(entry.relativePath),
|
||||
type: "directory",
|
||||
})),
|
||||
}
|
||||
}
|
||||
if (kind === "all") {
|
||||
const out = pick.mixedSearch(query, opts)
|
||||
if (!out.ok) return out
|
||||
return {
|
||||
ok: true,
|
||||
value: collectPaths(out.value.items, out.value.scores, (entry) => ({
|
||||
path: normalize(entry.item.relativePath),
|
||||
type: entry.type,
|
||||
})),
|
||||
}
|
||||
}
|
||||
const out = pick.fileSearch(query, opts)
|
||||
if (!out.ok) return out
|
||||
return {
|
||||
ok: true,
|
||||
value: collectPaths(out.value.items, out.value.scores, (entry) => ({
|
||||
path: normalize(entry.relativePath),
|
||||
type: "file",
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
export const layer: Layer.Layer<Service, never, FSUtil.Service | Ripgrep.Service> = Layer.effect(
|
||||
export const ripgrepLayer = Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const fs = yield* FSUtil.Service
|
||||
const rg = yield* Ripgrep.Service
|
||||
|
||||
const state: State = {
|
||||
pick: new Map<string, Picker>(),
|
||||
wait: new Map<string, Deferred.Deferred<Picker, Error>>(),
|
||||
recent: [] as Query[],
|
||||
const location = yield* Location.Service
|
||||
const ripgrep = yield* Ripgrep.Service
|
||||
const scope = yield* Scope.Scope
|
||||
const state = {
|
||||
files: [] as string[],
|
||||
directories: [] as string[],
|
||||
scan: undefined as Fiber.Fiber<void, never> | undefined,
|
||||
}
|
||||
|
||||
yield* fs.ensureDir(root).pipe(Effect.ignore)
|
||||
yield* Effect.addFinalizer(() =>
|
||||
Effect.forEach(
|
||||
state.pick.values(),
|
||||
(entry) => fffSync("destroy picker", () => entry.pick.destroy()).pipe(Effect.ignore),
|
||||
{ discard: true },
|
||||
),
|
||||
)
|
||||
|
||||
const rip = Effect.fn("Search.rip")(function* (input: Ripgrep.SearchInput) {
|
||||
const out = yield* rg.search(input)
|
||||
return {
|
||||
items: out.items,
|
||||
partial: out.partial,
|
||||
hasNextPage: false,
|
||||
engine: "ripgrep" as const,
|
||||
}
|
||||
})
|
||||
|
||||
// Lazy, shared scan-wait for a picker. Preserves the original behavior: if
|
||||
// the scan does not finish within the budget the picker is destroyed and
|
||||
// dropped from the cache so callers fall back to ripgrep (and the next
|
||||
// request recreates a fresh picker).
|
||||
const scanReady = (dir: string, pick: Fff.Picker) =>
|
||||
Effect.gen(function* () {
|
||||
const scanned = yield* Effect.tryPromise({
|
||||
try: () => pick.waitForScan(5_000),
|
||||
catch: (cause) => new Error("fff waitForScan failed", { cause }),
|
||||
})
|
||||
if (!scanned.ok || !scanned.value) {
|
||||
yield* fffSync("destroy picker", () => pick.destroy()).pipe(Effect.ignore)
|
||||
state.pick.delete(dir)
|
||||
yield* Effect.logWarning("fff scan not ready", { dir })
|
||||
return yield* Effect.fail(new Error(scanned.ok ? "fff scan timed out" : scanned.error))
|
||||
}
|
||||
|
||||
const git = yield* fffSync("refresh git status", () => pick.refreshGitStatus())
|
||||
if (!git.ok) {
|
||||
yield* Effect.logWarning("fff git refresh failed", { dir, error: git.error })
|
||||
}
|
||||
})
|
||||
|
||||
// Create (or return) the picker for a directory. Creation is synchronous
|
||||
// and does not await the scan; the native background scan starts as soon as
|
||||
// the picker exists. The `wait` gate dedupes concurrent creation.
|
||||
const acquire = Effect.fn("Search.acquire")(function* (cwd: string) {
|
||||
const dir = FSUtil.resolve(cwd)
|
||||
const existing = state.pick.get(dir)
|
||||
if (existing) return existing
|
||||
|
||||
const pending = state.wait.get(dir)
|
||||
if (pending) return yield* Deferred.await(pending)
|
||||
|
||||
const available = yield* fffSync("check availability", () => Fff.available()).pipe(
|
||||
Effect.catch((error) => Effect.logWarning("fff availability check failed", { error }).pipe(Effect.as(false))),
|
||||
)
|
||||
if (!available) return undefined
|
||||
|
||||
const gate = yield* Deferred.make<Picker, Error>()
|
||||
state.wait.set(dir, gate)
|
||||
return yield* Effect.gen(function* () {
|
||||
const id = key(dir)
|
||||
const isFirstPicker = state.pick.size === 0
|
||||
const made = yield* fffSync("create picker", () =>
|
||||
Fff.create({
|
||||
basePath: dir,
|
||||
frecencyDbPath: path.join(root, `${id}.frecency.mdb`),
|
||||
historyDbPath: path.join(root, `${id}.history.mdb`),
|
||||
aiMode: true,
|
||||
// only the first toolcall picker can accumulate resources to index
|
||||
// home directory, if the user specifically opened opencode at the
|
||||
// $HOME level or asked it to search there on purpose, otherwise fallback
|
||||
enableHomeDirScanning: isFirstPicker,
|
||||
// on unix system it is 99.9% that you do not need to search for the
|
||||
// content at the / so make fff fail creation and fallback to rg
|
||||
enableFsRootScanning: isFirstPicker && process.platform === "win32",
|
||||
}),
|
||||
)
|
||||
if (!made.ok) {
|
||||
yield* Effect.logWarning("fff init failed", { dir, error: made.error })
|
||||
const err = new Error(made.error)
|
||||
yield* Deferred.fail(gate, err)
|
||||
return yield* Effect.fail(err)
|
||||
}
|
||||
|
||||
const pick = made.value
|
||||
const entry: Picker = { pick, ready: yield* Effect.cached(scanReady(dir, pick)) }
|
||||
state.pick.set(dir, entry)
|
||||
yield* Deferred.succeed(gate, entry)
|
||||
return entry
|
||||
}).pipe(
|
||||
Effect.ensuring(
|
||||
Effect.gen(function* () {
|
||||
if (state.wait.get(dir) === gate) state.wait.delete(dir)
|
||||
yield* Deferred.fail(gate, new Error("fff init interrupted")).pipe(Effect.ignore)
|
||||
}),
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
// Resolve a usable, scanned picker for a directory, or undefined when fff is
|
||||
// unavailable or the scan did not become ready.
|
||||
const picker = Effect.fn("Search.picker")(function* (cwd: string) {
|
||||
const entry = yield* acquire(cwd).pipe(Effect.catch(() => Effect.succeed<Picker | undefined>(undefined)))
|
||||
if (!entry) return undefined
|
||||
const ready = yield* entry.ready.pipe(
|
||||
Effect.as(true),
|
||||
Effect.catch(() => Effect.succeed(false)),
|
||||
)
|
||||
if (!ready) return undefined
|
||||
return entry.pick
|
||||
})
|
||||
|
||||
const files: Interface["files"] = (input) => rg.files(input)
|
||||
const tree: Interface["tree"] = (input) => rg.tree(input)
|
||||
|
||||
// in 99% of use cases user that is opened opencode at certain directory will
|
||||
// conduct a file search in this direcotry, it could be switched later but
|
||||
// mostly always we will need a file picker for cwd
|
||||
// so synchronously start FFF scan for a cwd so it is ready before first toolcall generated
|
||||
const warm: Interface["warm"] = Effect.fn("Search.warm")(function* (cwd) {
|
||||
yield* acquire(cwd).pipe(Effect.ignore)
|
||||
})
|
||||
|
||||
// Tear down the picker for a directory. fff pickers own a native background
|
||||
// watcher thread that otherwise lives until the runtime scope closes (i.e.
|
||||
// process exit), so disposing the instance that warmed it must destroy it
|
||||
// here or the thread leaks against a directory that may already be gone.
|
||||
const release: Interface["release"] = Effect.fn("Search.release")(function* (cwd) {
|
||||
const dir = FSUtil.resolve(cwd)
|
||||
|
||||
const pending = state.wait.get(dir)
|
||||
if (pending) {
|
||||
state.wait.delete(dir)
|
||||
yield* Deferred.fail(pending, new Error("fff picker released")).pipe(Effect.ignore)
|
||||
}
|
||||
|
||||
const entry = state.pick.get(dir)
|
||||
if (entry) {
|
||||
state.pick.delete(dir)
|
||||
yield* fffSync("destroy picker", () => entry.pick.destroy()).pipe(Effect.ignore)
|
||||
}
|
||||
|
||||
const remaining = state.recent.filter((item) => item.dir !== dir)
|
||||
state.recent.splice(0, state.recent.length, ...remaining)
|
||||
})
|
||||
|
||||
const file: Interface["file"] = Effect.fn("Search.file")(function* (input) {
|
||||
const query = input.query.trim()
|
||||
const kind = input.kind ?? "file"
|
||||
|
||||
const entry = yield* acquire(input.cwd)
|
||||
if (!entry) return yield* Effect.fail(new Error("fff is unavailable"))
|
||||
yield* entry.ready
|
||||
const dir = FSUtil.resolve(input.cwd)
|
||||
const limit = input.limit ?? 100
|
||||
const fffResult = yield* fffSync(`${kind} search`, () =>
|
||||
searchFff(entry.pick, kind, query, {
|
||||
pageIndex: 0,
|
||||
currentFile: input.current, // supports both relative and absolute (relative preferred)
|
||||
pageSize: limit,
|
||||
}),
|
||||
).pipe(
|
||||
Effect.catch((error) =>
|
||||
Effect.logWarning(`fff ${kind} search failed`, { dir, query, error }).pipe(
|
||||
Effect.andThen(Effect.fail(error)),
|
||||
),
|
||||
),
|
||||
)
|
||||
if (!fffResult.ok) {
|
||||
yield* Effect.logWarning(`fff ${kind} search failed`, { dir, query, error: fffResult.error })
|
||||
return yield* Effect.fail(new Error(fffResult.error))
|
||||
}
|
||||
|
||||
const rows = fffResult.value
|
||||
remember(
|
||||
state,
|
||||
dir,
|
||||
query,
|
||||
rows.map((row) => path.join(dir, row.path)),
|
||||
)
|
||||
return rows.slice(0, limit)
|
||||
})
|
||||
|
||||
const search: Interface["search"] = Effect.fn("Search.search")(function* (input) {
|
||||
input.signal?.throwIfAborted()
|
||||
if (input.file?.length) return yield* rip(input)
|
||||
|
||||
const pick = yield* picker(input.cwd)
|
||||
if (!pick) return yield* rip(input)
|
||||
|
||||
const dir = FSUtil.resolve(input.cwd)
|
||||
const limit = input.limit ?? 100
|
||||
|
||||
const fffGrep = yield* fffSync("grep", () =>
|
||||
pick.grep(fffGlobbedQuery(input.pattern, input.glob), {
|
||||
mode: "regex",
|
||||
pageSize: limit,
|
||||
timeBudgetMs: 1_500,
|
||||
}),
|
||||
).pipe(
|
||||
Effect.catch((error) =>
|
||||
Effect.logWarning("fff grep failed", { dir, pattern: input.pattern, error }).pipe(
|
||||
Effect.as<Fff.Result<Fff.Grep> | undefined>(undefined),
|
||||
),
|
||||
),
|
||||
)
|
||||
if (!fffGrep) return yield* rip(input)
|
||||
if (!fffGrep.ok) {
|
||||
yield* Effect.logWarning("fff grep failed", { dir, pattern: input.pattern, error: fffGrep.error })
|
||||
return yield* rip(input)
|
||||
}
|
||||
|
||||
const rows: Item[] = fffGrep.value.items.map(item)
|
||||
const regexFallbackError = fffGrep.value.regexFallbackError
|
||||
|
||||
remember(state, dir, input.pattern, Array.from(new Set(rows.map((row) => path.join(dir, row.path.text)))))
|
||||
|
||||
return {
|
||||
items: rows,
|
||||
partial: false,
|
||||
hasNextPage: !!fffGrep.value.nextCursor,
|
||||
engine: "fff" as const,
|
||||
regexFallbackError,
|
||||
}
|
||||
})
|
||||
|
||||
const glob: Interface["glob"] = Effect.fn("Search.glob")(function* (input) {
|
||||
input.signal?.throwIfAborted()
|
||||
|
||||
const dir = FSUtil.resolve(input.cwd)
|
||||
const limit = input.limit ?? 100
|
||||
const pick = yield* picker(dir)
|
||||
|
||||
if (pick) {
|
||||
const fffGlob = yield* fffSync("glob file search", () =>
|
||||
pick.glob(normalize(input.pattern), {
|
||||
pageIndex: 0,
|
||||
pageSize: limit,
|
||||
}),
|
||||
).pipe(
|
||||
Effect.catch((error) =>
|
||||
Effect.logWarning("fff glob failed", { dir, pattern: input.pattern, error }).pipe(
|
||||
Effect.as<Fff.Result<Fff.Search> | undefined>(undefined),
|
||||
state.scan = yield* ripgrep.find({ cwd: location.directory, pattern: "*", limit: 100_000 }).pipe(
|
||||
Effect.tap((result) =>
|
||||
Effect.sync(() => {
|
||||
state.files = result.map((item) => item.path)
|
||||
state.directories = Array.from(
|
||||
new Set(
|
||||
state.files.flatMap((file) => {
|
||||
const parts = file.split("/")
|
||||
return parts.slice(0, -1).map((_, index) => parts.slice(0, index + 1).join("/") + path.sep)
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
if (fffGlob?.ok) {
|
||||
const rows: string[] = Array.from(new Set(fffGlob.value.items.map((item) => normalize(item.relativePath))))
|
||||
|
||||
remember(
|
||||
state,
|
||||
dir,
|
||||
input.pattern,
|
||||
rows.map((row) => path.join(dir, row)),
|
||||
)
|
||||
|
||||
return {
|
||||
files: rows.slice(0, limit).map((row) => path.join(dir, row)),
|
||||
truncated: fffGlob.value.totalMatched > rows.length,
|
||||
}
|
||||
} else if (fffGlob) {
|
||||
yield* Effect.logWarning("fff glob failed", { dir, pattern: input.pattern, error: fffGlob.error })
|
||||
// fall through to the fallback
|
||||
}
|
||||
}
|
||||
|
||||
const rows = yield* rg.files({ cwd: dir, glob: [input.pattern], signal: input.signal }).pipe(
|
||||
Stream.take(limit + 1),
|
||||
Stream.runCollect,
|
||||
Effect.map((chunk) => [...chunk]),
|
||||
)
|
||||
const truncated = rows.length > limit
|
||||
if (truncated) rows.length = limit
|
||||
|
||||
const output = yield* Effect.forEach(
|
||||
rows,
|
||||
Effect.fnUntraced(function* (file) {
|
||||
const full = path.join(dir, file)
|
||||
const info = yield* fs.stat(full).pipe(Effect.catch(() => Effect.succeed(undefined)))
|
||||
const time =
|
||||
info?.mtime.pipe(
|
||||
Option.map((item) => item.getTime()),
|
||||
Option.getOrElse(() => 0),
|
||||
) ?? 0
|
||||
return { file: full, time }
|
||||
}),
|
||||
{ concurrency: 16 },
|
||||
)
|
||||
output.sort((a, b) => b.time - a.time)
|
||||
return {
|
||||
files: output.map((item) => item.file),
|
||||
truncated,
|
||||
}
|
||||
),
|
||||
Effect.orDie,
|
||||
Effect.asVoid,
|
||||
Effect.forkIn(scope),
|
||||
)
|
||||
return Service.of({
|
||||
glob: (input) =>
|
||||
Effect.gen(function* () {
|
||||
const target = path.resolve(location.directory, input.path ?? ".")
|
||||
const info = yield* fs.stat(target).pipe(Effect.orDie)
|
||||
const cwd = info.type === "File" ? path.dirname(target) : target
|
||||
return yield* ripgrep
|
||||
.glob({
|
||||
cwd,
|
||||
pattern: input.pattern,
|
||||
limit: input.limit ?? Number.MAX_SAFE_INTEGER,
|
||||
})
|
||||
.pipe(
|
||||
Effect.map((result) =>
|
||||
result.map(
|
||||
(entry) =>
|
||||
new FileSystem.Entry({
|
||||
...entry,
|
||||
path: RelativePath.make(path.relative(location.directory, path.resolve(cwd, entry.path))),
|
||||
}),
|
||||
),
|
||||
),
|
||||
Effect.orDie,
|
||||
)
|
||||
}),
|
||||
grep: (input) =>
|
||||
Effect.gen(function* () {
|
||||
const target = path.resolve(location.directory, input.path ?? ".")
|
||||
const info = yield* fs.stat(target).pipe(Effect.orDie)
|
||||
const cwd = info.type === "File" ? path.dirname(target) : target
|
||||
return yield* ripgrep
|
||||
.grep({
|
||||
cwd,
|
||||
pattern: input.pattern,
|
||||
file: info.type === "File" ? path.basename(target) : undefined,
|
||||
include: input.include,
|
||||
limit: input.limit ?? Number.MAX_SAFE_INTEGER,
|
||||
})
|
||||
.pipe(
|
||||
Effect.map((result) =>
|
||||
result.map(
|
||||
(match) =>
|
||||
new FileSystem.Match({
|
||||
...match,
|
||||
entry: new FileSystem.Entry({
|
||||
...match.entry,
|
||||
path: RelativePath.make(path.relative(location.directory, path.resolve(cwd, match.entry.path))),
|
||||
}),
|
||||
}),
|
||||
),
|
||||
),
|
||||
Effect.orDie,
|
||||
)
|
||||
}),
|
||||
find: (input) =>
|
||||
Effect.gen(function* () {
|
||||
if (input.query) yield* Fiber.join(state.scan!)
|
||||
const items =
|
||||
input.type === "file"
|
||||
? state.files
|
||||
: input.type === "directory"
|
||||
? state.directories
|
||||
: [...state.files, ...state.directories]
|
||||
return fuzzysort.go(input.query, items, { limit: input.limit ?? 50 }).map((item) => {
|
||||
const relative = item.target
|
||||
const type = relative.endsWith(path.sep) ? ("directory" as const) : ("file" as const)
|
||||
const clean = type === "directory" ? relative.slice(0, -path.sep.length) : relative
|
||||
const absolute = path.resolve(location.directory, clean)
|
||||
return new FileSystem.Entry({
|
||||
path: RelativePath.make(relative),
|
||||
type,
|
||||
mime: type === "directory" ? "application/x-directory" : FSUtil.mimeType(absolute),
|
||||
})
|
||||
})
|
||||
}),
|
||||
})
|
||||
|
||||
const open: Interface["open"] = Effect.fn("Search.open")(function* (input) {
|
||||
const file = input.cwd
|
||||
? FSUtil.resolve(path.isAbsolute(input.file) ? input.file : path.join(input.cwd, input.file))
|
||||
: FSUtil.resolve(input.file)
|
||||
const idx = state.recent.findIndex((item) => item.files.includes(file))
|
||||
if (idx < 0) return
|
||||
|
||||
const row = state.recent[idx]
|
||||
state.recent.splice(idx, 1)
|
||||
const entry = state.pick.get(row.dir)
|
||||
if (!entry) return
|
||||
|
||||
const out = yield* fffSync("track query", () => entry.pick.trackQuery(row.text, file)).pipe(
|
||||
Effect.catch((error) =>
|
||||
Effect.logWarning("fff track query failed", { dir: row.dir, query: row.text, file, error }).pipe(
|
||||
Effect.as<Fff.Result<boolean> | undefined>(undefined),
|
||||
),
|
||||
),
|
||||
)
|
||||
if (!out) return
|
||||
if (!out.ok) {
|
||||
yield* Effect.logWarning("fff track query failed", { dir: row.dir, query: row.text, file, error: out.error })
|
||||
}
|
||||
})
|
||||
|
||||
return Service.of({ files, tree, search, file, glob, open, warm, release })
|
||||
}),
|
||||
)
|
||||
|
||||
export const defaultLayer: Layer.Layer<Service> = layer.pipe(
|
||||
Layer.provide(Ripgrep.defaultLayer),
|
||||
Layer.provide(FSUtil.defaultLayer),
|
||||
export const fffLayer = Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const location = yield* Location.Service
|
||||
const result = yield* Effect.try({
|
||||
try: () =>
|
||||
Fff.create({
|
||||
basePath: location.directory,
|
||||
aiMode: true,
|
||||
enableFsRootScanning: true,
|
||||
enableHomeDirScanning: true,
|
||||
}),
|
||||
catch: (cause) => cause,
|
||||
}).pipe(Effect.orDie)
|
||||
if (!result.ok) return yield* Effect.die(result.error)
|
||||
yield* Effect.addFinalizer(() => Effect.sync(() => result.value.destroy()).pipe(Effect.ignore))
|
||||
const scanned = yield* Effect.tryPromise({
|
||||
try: () => result.value.waitForScan(5_000),
|
||||
catch: (cause) => cause,
|
||||
}).pipe(Effect.orDie)
|
||||
if (!scanned.ok || !scanned.value) return yield* Effect.die(scanned.ok ? "fff scan timed out" : scanned.error)
|
||||
return Service.of({
|
||||
glob: (input) =>
|
||||
Effect.sync(() => {
|
||||
const prefix = input.path?.replaceAll("\\", "/").replace(/\/$/, "")
|
||||
const found = result.value.glob(prefix ? `${prefix}/${input.pattern}` : input.pattern, {
|
||||
pageIndex: 0,
|
||||
pageSize: input.limit,
|
||||
})
|
||||
if (!found.ok) throw found.error
|
||||
return found.value.items.map((item) => {
|
||||
const absolute = path.resolve(location.directory, item.relativePath)
|
||||
return new FileSystem.Entry({
|
||||
path: RelativePath.make(item.relativePath.replaceAll("\\", "/")),
|
||||
type: "file",
|
||||
mime: FSUtil.mimeType(absolute),
|
||||
})
|
||||
})
|
||||
}),
|
||||
grep: (input) =>
|
||||
Effect.sync(() => {
|
||||
const prefix = input.path?.replaceAll("\\", "/").replace(/\/$/, "")
|
||||
const found = result.value.grep(
|
||||
[prefix ? `${prefix}/**` : undefined, input.include, input.pattern]
|
||||
.filter((value) => value !== undefined)
|
||||
.join(" "),
|
||||
{ mode: "regex", pageSize: input.limit, timeBudgetMs: 1_500 },
|
||||
)
|
||||
if (!found.ok) throw found.error
|
||||
return found.value.items.map((match) => {
|
||||
const bytes = Buffer.from(match.lineContent)
|
||||
return new FileSystem.Match({
|
||||
entry: new FileSystem.Entry({
|
||||
path: RelativePath.make(match.relativePath.replaceAll("\\", "/")),
|
||||
type: "file",
|
||||
mime: FSUtil.mimeType(match.relativePath),
|
||||
}),
|
||||
line: match.lineNumber,
|
||||
offset: match.byteOffset,
|
||||
text: match.lineContent.length > 2_000 ? match.lineContent.slice(0, 2_000) + "..." : match.lineContent,
|
||||
submatches: match.matchRanges.map(([start, end]) => ({
|
||||
text: bytes.subarray(start, end).toString("utf8"),
|
||||
start,
|
||||
end,
|
||||
})),
|
||||
})
|
||||
})
|
||||
}),
|
||||
find: (input) =>
|
||||
Effect.sync(() => {
|
||||
const options = { pageIndex: 0, pageSize: input.limit ?? 50 }
|
||||
const items = (() => {
|
||||
if (input.type === "file") {
|
||||
const found = result.value.fileSearch(input.query.trim(), options)
|
||||
if (!found.ok) throw found.error
|
||||
return found.value.items.map((item) => ({ path: item.relativePath, type: "file" as const }))
|
||||
}
|
||||
if (input.type === "directory") {
|
||||
const found = result.value.directorySearch(input.query.trim(), options)
|
||||
if (!found.ok) throw found.error
|
||||
return found.value.items.map((item) => ({ path: item.relativePath, type: "directory" as const }))
|
||||
}
|
||||
const found = result.value.mixedSearch(input.query.trim(), options)
|
||||
if (!found.ok) throw found.error
|
||||
return found.value.items.map((item) => ({ path: item.item.relativePath, type: item.type }))
|
||||
})()
|
||||
return items.map((item) => {
|
||||
const relative = item.path.replaceAll("\\", "/").replace(/\/$/, "")
|
||||
const absolute = path.resolve(location.directory, relative)
|
||||
return new FileSystem.Entry({
|
||||
path: RelativePath.make(relative + (item.type === "directory" ? path.sep : "")),
|
||||
type: item.type,
|
||||
mime: item.type === "directory" ? "application/x-directory" : FSUtil.mimeType(absolute),
|
||||
})
|
||||
})
|
||||
}),
|
||||
})
|
||||
}),
|
||||
)
|
||||
export const node = LayerNode.make(layer, [FSUtil.node, Ripgrep.node])
|
||||
|
||||
const { runPromise } = makeRuntime(Service, defaultLayer)
|
||||
|
||||
export function tree(input: Ripgrep.TreeInput) {
|
||||
return runPromise((svc) => svc.tree(input))
|
||||
}
|
||||
|
||||
export function search(input: Ripgrep.SearchInput) {
|
||||
return runPromise((svc) => svc.search(input))
|
||||
}
|
||||
|
||||
export function file(input: FileInput) {
|
||||
return runPromise((svc) => svc.file(input))
|
||||
}
|
||||
|
||||
export function glob(input: GlobInput) {
|
||||
return runPromise((svc) => svc.glob(input))
|
||||
}
|
||||
|
||||
export function open(input: { cwd?: string; file: string }) {
|
||||
return runPromise((svc) => svc.open(input))
|
||||
}
|
||||
|
||||
export * as Search from "./search"
|
||||
export const defaultLayer = Layer.unwrap(Effect.sync(() => (Fff.available() ? fffLayer : ripgrepLayer)))
|
||||
|
|
|
|||
|
|
@ -18,10 +18,9 @@ import { Database } from "./database/database"
|
|||
import { PermissionV2 } from "./permission"
|
||||
import { PermissionSaved } from "./permission/saved"
|
||||
import { FileSystem } from "./filesystem"
|
||||
import { Ripgrep } from "./ripgrep"
|
||||
import { Watcher } from "./filesystem/watcher"
|
||||
import { Search } from "./filesystem/search"
|
||||
import { LocationMutation } from "./location-mutation"
|
||||
import { LocationSearch } from "./location-search"
|
||||
import { FileMutation } from "./file-mutation"
|
||||
import { Reference } from "./reference"
|
||||
import { RepositoryCache } from "./repository-cache"
|
||||
|
|
@ -34,7 +33,6 @@ import { ToolRegistry } from "./tool/registry"
|
|||
import { ApplicationTools } from "./tool/application-tools"
|
||||
import { ToolOutputStore } from "./tool-output-store"
|
||||
import { AppProcess } from "./process"
|
||||
import { Ripgrep } from "./ripgrep"
|
||||
import { SessionStore } from "./session/store"
|
||||
import { SessionTodo } from "./session/todo"
|
||||
import { QuestionV2 } from "./question"
|
||||
|
|
@ -75,14 +73,12 @@ export class LocationServiceMap extends LayerMap.Service<LocationServiceMap>()("
|
|||
const services = Layer.mergeAll(base, resources, permissionsAndTools)
|
||||
const image = Image.layer.pipe(Layer.provide(services))
|
||||
const mutation = FileMutation.locationLayer.pipe(Layer.provide(services))
|
||||
const searches = LocationSearch.layer.pipe(Layer.provide(Ripgrep.layer), Layer.provide(services))
|
||||
const skillGuidance = SkillGuidance.locationLayer.pipe(Layer.provide(services))
|
||||
const todos = SessionTodo.layer.pipe(Layer.provide(services))
|
||||
const questions = QuestionV2.locationLayer.pipe(Layer.provide(services))
|
||||
const builtInTools = BuiltInTools.locationLayer.pipe(
|
||||
Layer.provide(services),
|
||||
Layer.provide(mutation),
|
||||
Layer.provide(searches),
|
||||
Layer.provide(resources),
|
||||
Layer.provide(todos),
|
||||
Layer.provide(questions),
|
||||
|
|
@ -94,18 +90,9 @@ export class LocationServiceMap extends LayerMap.Service<LocationServiceMap>()("
|
|||
Layer.provide(model),
|
||||
Layer.provide(skillGuidance),
|
||||
)
|
||||
return Layer.mergeAll(
|
||||
services,
|
||||
image,
|
||||
mutation,
|
||||
searches,
|
||||
resources,
|
||||
todos,
|
||||
questions,
|
||||
model,
|
||||
runner,
|
||||
builtInTools,
|
||||
).pipe(Layer.fresh)
|
||||
return Layer.mergeAll(services, image, mutation, resources, todos, questions, model, runner, builtInTools).pipe(
|
||||
Layer.fresh,
|
||||
)
|
||||
},
|
||||
idleTimeToLive: "60 minutes",
|
||||
dependencies: [
|
||||
|
|
@ -117,6 +104,7 @@ export class LocationServiceMap extends LayerMap.Service<LocationServiceMap>()("
|
|||
FSUtil.defaultLayer,
|
||||
AppProcess.defaultLayer,
|
||||
Global.defaultLayer,
|
||||
Ripgrep.defaultLayer,
|
||||
Database.defaultLayer,
|
||||
SessionStore.layer.pipe(Layer.provide(Database.defaultLayer)),
|
||||
PermissionSaved.defaultLayer,
|
||||
|
|
@ -125,6 +113,5 @@ export class LocationServiceMap extends LayerMap.Service<LocationServiceMap>()("
|
|||
FetchHttpClient.layer,
|
||||
ToolOutputStore.defaultCleanupLayer,
|
||||
ApplicationTools.layer,
|
||||
Search.defaultLayer,
|
||||
],
|
||||
}) {}
|
||||
|
|
|
|||
|
|
@ -1,219 +0,0 @@
|
|||
export * as LocationSearch from "./location-search"
|
||||
|
||||
import path from "path"
|
||||
import { Context, Effect, Layer, Option, Schema } from "effect"
|
||||
import { FSUtil } from "./fs-util"
|
||||
import { Global } from "./global"
|
||||
import { Location } from "./location"
|
||||
import { Ripgrep } from "./ripgrep"
|
||||
import { NonNegativeInt, PositiveInt, RelativePath } from "./schema"
|
||||
import { ToolOutputStore } from "./tool-output-store"
|
||||
|
||||
/**
|
||||
* Location-scoped raw search substrate. Search authority is selected only by
|
||||
* FileSystem, preserving Location-relative paths. Model formatting, leaf-tool permissions, and HTTP transport stay
|
||||
* outside this service so future GlobTool, GrepTool, and HTTP consumers can
|
||||
* share the same bounded filesystem behavior.
|
||||
*
|
||||
* TODO: Expose this substrate through HTTP fs.search/fs.grep endpoints.
|
||||
* TODO: Reuse this substrate for instruction and skill discovery where suitable.
|
||||
*/
|
||||
|
||||
export const DEFAULT_RESULT_LIMIT = 100
|
||||
export const MAX_RESULT_LIMIT = 100
|
||||
export const MAX_LINE_PREVIEW_LENGTH = 2_000
|
||||
|
||||
export const ResultLimit = PositiveInt.check(Schema.isLessThanOrEqualTo(MAX_RESULT_LIMIT))
|
||||
|
||||
export const FilesInput = Schema.Struct({
|
||||
pattern: Schema.String,
|
||||
path: Schema.String.pipe(Schema.optional),
|
||||
limit: ResultLimit.pipe(Schema.optional),
|
||||
})
|
||||
export type FilesInput = typeof FilesInput.Type & { readonly signal?: AbortSignal }
|
||||
|
||||
export const GrepInput = Schema.Struct({
|
||||
pattern: Schema.String,
|
||||
include: Schema.String.pipe(Schema.optional),
|
||||
path: Schema.String.pipe(Schema.optional),
|
||||
limit: ResultLimit.pipe(Schema.optional),
|
||||
})
|
||||
export type GrepInput = typeof GrepInput.Type & { readonly signal?: AbortSignal }
|
||||
|
||||
export class File extends Schema.Class<File>("LocationSearch.File")({
|
||||
path: RelativePath,
|
||||
canonical: Schema.String,
|
||||
resource: Schema.String,
|
||||
mtime: Schema.Number,
|
||||
}) {}
|
||||
|
||||
export class Submatch extends Schema.Class<Submatch>("LocationSearch.Submatch")({
|
||||
text: Schema.String,
|
||||
start: NonNegativeInt,
|
||||
end: NonNegativeInt,
|
||||
}) {}
|
||||
|
||||
export class Match extends Schema.Class<Match>("LocationSearch.Match")({
|
||||
path: RelativePath,
|
||||
canonical: Schema.String,
|
||||
resource: Schema.String,
|
||||
lines: Schema.String,
|
||||
linePreviewTruncated: Schema.Boolean,
|
||||
line: PositiveInt,
|
||||
offset: NonNegativeInt,
|
||||
submatches: Schema.Array(Submatch),
|
||||
mtime: Schema.Number,
|
||||
}) {}
|
||||
|
||||
export class FilesResult extends Schema.Class<FilesResult>("LocationSearch.FilesResult")({
|
||||
items: Schema.Array(File),
|
||||
truncated: Schema.Boolean,
|
||||
partial: Schema.Boolean,
|
||||
}) {}
|
||||
|
||||
export class GrepResult extends Schema.Class<GrepResult>("LocationSearch.GrepResult")({
|
||||
items: Schema.Array(Match),
|
||||
truncated: Schema.Boolean,
|
||||
partial: Schema.Boolean,
|
||||
}) {}
|
||||
|
||||
export interface Interface {
|
||||
readonly files: (input: FilesInput) => Effect.Effect<FilesResult, Ripgrep.Error>
|
||||
readonly grep: (input: GrepInput) => Effect.Effect<GrepResult, Ripgrep.Error | Ripgrep.InvalidPatternError>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/v2/LocationSearch") {}
|
||||
|
||||
const slash = (value: string) => value.replaceAll("\\", "/")
|
||||
const cap = (limit?: number) => Math.min(limit ?? DEFAULT_RESULT_LIMIT, MAX_RESULT_LIMIT)
|
||||
|
||||
export const layer = Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const fs = yield* FSUtil.Service
|
||||
const location = yield* Location.Service
|
||||
const global = yield* Effect.serviceOption(Global.Service)
|
||||
const ripgrep = yield* Ripgrep.Service
|
||||
|
||||
const resolve = Effect.fnUntraced(function* (input?: string) {
|
||||
const directory = input && path.isAbsolute(input) ? path.dirname(input) : location.directory
|
||||
const absolute = path.resolve(location.directory, input ?? ".")
|
||||
if (!path.isAbsolute(input ?? "") && !FSUtil.contains(location.directory, absolute))
|
||||
return yield* Effect.die(new globalThis.Error("Path escapes the location"))
|
||||
if (path.isAbsolute(input ?? "")) {
|
||||
const managed = path.join(
|
||||
Option.match(global, { onNone: () => Global.Path.data, onSome: (value) => value.data }),
|
||||
ToolOutputStore.MANAGED_DIRECTORY,
|
||||
)
|
||||
if (directory !== managed || !path.basename(absolute).startsWith("tool_"))
|
||||
return yield* Effect.die(new globalThis.Error("Absolute path is not managed tool output"))
|
||||
}
|
||||
const real = yield* fs.realPath(absolute).pipe(Effect.orDie)
|
||||
const root = yield* fs.realPath(directory).pipe(Effect.orDie)
|
||||
if (!FSUtil.contains(root, real)) return yield* Effect.die(new globalThis.Error("Path escapes the search root"))
|
||||
const info = yield* fs.stat(real).pipe(Effect.orDie)
|
||||
const type =
|
||||
info.type === "File" ? ("file" as const) : info.type === "Directory" ? ("directory" as const) : undefined
|
||||
if (!type) return yield* Effect.die(new globalThis.Error("Search root is not a file or directory"))
|
||||
return { real, root, resource: slash(path.relative(root, real)) || ".", type }
|
||||
})
|
||||
|
||||
const candidate = Effect.fnUntraced(function* (
|
||||
root: { readonly real: string; readonly root: string; readonly type: "file" | "directory" },
|
||||
cwd: string,
|
||||
value: string,
|
||||
) {
|
||||
const absolute = path.resolve(cwd, value)
|
||||
const lexicallyContained =
|
||||
root.type === "directory" ? FSUtil.contains(root.real, absolute) : absolute === root.real
|
||||
if (!lexicallyContained) return
|
||||
const canonical = yield* fs.realPath(absolute).pipe(Effect.catch(() => Effect.void))
|
||||
if (!canonical || !FSUtil.contains(root.root, canonical)) return
|
||||
const info = yield* fs.stat(canonical).pipe(Effect.catch(() => Effect.void))
|
||||
if (!info || info.type !== "File") return
|
||||
const relative = slash(path.relative(root.root, canonical))
|
||||
return {
|
||||
path: RelativePath.make(relative),
|
||||
canonical,
|
||||
resource: relative,
|
||||
mtime: info.mtime.pipe(
|
||||
Option.map((date) => date.getTime()),
|
||||
Option.getOrElse(() => 0),
|
||||
),
|
||||
}
|
||||
})
|
||||
|
||||
return Service.of({
|
||||
files: Effect.fn("LocationSearch.files")(function* (input) {
|
||||
const root = yield* resolve(input.path)
|
||||
if (root.type !== "directory")
|
||||
return yield* Effect.die(new globalThis.Error("Files search path must be a directory"))
|
||||
const result = yield* ripgrep.files({
|
||||
cwd: root.real,
|
||||
pattern: input.pattern,
|
||||
limit: cap(input.limit),
|
||||
signal: input.signal,
|
||||
})
|
||||
const mapped = yield* Effect.forEach(result.items, (item) => candidate(root, root.real, item), {
|
||||
concurrency: 16,
|
||||
})
|
||||
const items = mapped.filter((item): item is File => item !== undefined).map((item) => new File(item))
|
||||
// TODO: Decide result ordering policy: V1 mtime sorting versus stable path ordering.
|
||||
// TODO: Report inaccessible paths discovered after bounded ripgrep termination when practical.
|
||||
return new FilesResult({
|
||||
items,
|
||||
truncated: result.truncated,
|
||||
partial: result.partial || items.length !== result.items.length,
|
||||
})
|
||||
}),
|
||||
grep: Effect.fn("LocationSearch.grep")(function* (input) {
|
||||
const root = yield* resolve(input.path)
|
||||
const cwd = root.type === "directory" ? root.real : path.dirname(root.real)
|
||||
const result = yield* ripgrep.grep({
|
||||
cwd,
|
||||
pattern: input.pattern,
|
||||
include: input.include,
|
||||
file: root.type === "file" ? path.basename(root.real) : undefined,
|
||||
limit: cap(input.limit),
|
||||
signal: input.signal,
|
||||
})
|
||||
const candidates = new Map<string, ReturnType<typeof candidate>>()
|
||||
for (const item of result.items) {
|
||||
if (!candidates.has(item.path.text)) {
|
||||
candidates.set(item.path.text, yield* Effect.cached(candidate(root, cwd, item.path.text)))
|
||||
}
|
||||
}
|
||||
const mapped = yield* Effect.forEach(
|
||||
result.items,
|
||||
(item) =>
|
||||
candidates.get(item.path.text)!.pipe(
|
||||
Effect.map(
|
||||
(file) =>
|
||||
file &&
|
||||
new Match({
|
||||
...file,
|
||||
lines: item.lines.text.slice(0, MAX_LINE_PREVIEW_LENGTH),
|
||||
linePreviewTruncated: item.lines.text.length > MAX_LINE_PREVIEW_LENGTH,
|
||||
line: item.line_number,
|
||||
offset: item.absolute_offset,
|
||||
submatches: item.submatches.map(
|
||||
(submatch) =>
|
||||
new Submatch({ text: submatch.match.text, start: submatch.start, end: submatch.end }),
|
||||
),
|
||||
}),
|
||||
),
|
||||
),
|
||||
{ concurrency: 16 },
|
||||
)
|
||||
const items = mapped.filter((item): item is Match => item !== undefined)
|
||||
// TODO: Decide result ordering policy: V1 mtime sorting versus stable path ordering.
|
||||
// TODO: Report inaccessible paths discovered after bounded ripgrep termination when practical.
|
||||
return new GrepResult({
|
||||
items,
|
||||
truncated: result.truncated,
|
||||
partial: result.partial || items.length !== result.items.length,
|
||||
})
|
||||
}),
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
|
@ -32,7 +32,8 @@ class SessionModelValidation extends Context.Service<
|
|||
}
|
||||
>()("@opencode/public/OpenCode/SessionModelValidation") {}
|
||||
|
||||
const LocationServicesLayer = LocationServiceMap.layer
|
||||
const ApplicationToolsLayer = ApplicationTools.layer
|
||||
const LocationServicesLayer = LocationServiceMap.layer.pipe(Layer.provide(ApplicationToolsLayer))
|
||||
const SessionModelValidationLayer = Layer.effect(
|
||||
SessionModelValidation,
|
||||
Effect.gen(function* () {
|
||||
|
|
@ -78,8 +79,6 @@ const SessionsLayer = Layer.merge(
|
|||
),
|
||||
SessionModelValidationLayer,
|
||||
).pipe(Layer.provide(LocationServicesLayer))
|
||||
const ApplicationToolsLayer = ApplicationTools.layer
|
||||
|
||||
// TODO: Accept explicit storage so tests and embeddings can select disposable or application-owned persistence.
|
||||
export const layer = Layer.effect(
|
||||
Service,
|
||||
|
|
|
|||
|
|
@ -2,20 +2,23 @@ export * as Ripgrep from "./ripgrep"
|
|||
|
||||
import { Context, Effect, Fiber, Layer, Schema, Stream } from "effect"
|
||||
import { ChildProcess } from "effect/unstable/process"
|
||||
import { Ripgrep as FileSystemRipgrep } from "./filesystem/ripgrep"
|
||||
import path from "path"
|
||||
import { Entry, Match } from "./filesystem/schema"
|
||||
import { FSUtil } from "./fs-util"
|
||||
import { AppProcess, collectStream, waitForAbort } from "./process"
|
||||
import { NonNegativeInt, PositiveInt } from "./schema"
|
||||
import { NonNegativeInt, PositiveInt, RelativePath } from "./schema"
|
||||
import { RipgrepBinary } from "./ripgrep/binary"
|
||||
|
||||
/**
|
||||
* Small core-owned ripgrep execution adapter. It deliberately exposes raw
|
||||
* process-oriented rows, not model text or permission behavior. LocationSearch
|
||||
* supplies read authority and bounded substrate results; future leaf tools own
|
||||
* process-oriented rows, not model text or permission behavior. Search maps
|
||||
* these rows into filesystem results; leaf tools own
|
||||
* presentation and permission prompts.
|
||||
*/
|
||||
|
||||
const ERROR_BYTES = 8 * 1024
|
||||
export const MAX_RECORD_BYTES = 64 * 1024
|
||||
export const MAX_SUBMATCHES = 100
|
||||
const MAX_RECORD_BYTES = 64 * 1024
|
||||
const MAX_SUBMATCHES = 100
|
||||
|
||||
const RawMatch = Schema.Struct({
|
||||
type: Schema.Literal("match"),
|
||||
|
|
@ -34,7 +37,7 @@ const RawMatch = Schema.Struct({
|
|||
}),
|
||||
})
|
||||
|
||||
export type Match = (typeof RawMatch.Type)["data"]
|
||||
type RawMatchData = (typeof RawMatch.Type)["data"]
|
||||
|
||||
export class Error extends Schema.TaggedErrorClass<Error>()("Ripgrep.Error", {
|
||||
message: Schema.String,
|
||||
|
|
@ -46,16 +49,21 @@ export class InvalidPatternError extends Schema.TaggedErrorClass<InvalidPatternE
|
|||
message: Schema.String,
|
||||
}) {}
|
||||
|
||||
export interface Result<A> {
|
||||
readonly items: A[]
|
||||
readonly truncated: boolean
|
||||
readonly partial: boolean
|
||||
}
|
||||
|
||||
export interface FilesInput {
|
||||
export interface FindInput {
|
||||
readonly cwd: string
|
||||
readonly pattern: string
|
||||
readonly limit: number
|
||||
readonly hidden?: boolean
|
||||
readonly follow?: boolean
|
||||
readonly signal?: AbortSignal
|
||||
}
|
||||
|
||||
export interface GlobInput {
|
||||
readonly cwd: string
|
||||
readonly pattern: string
|
||||
readonly limit: number
|
||||
readonly hidden?: boolean
|
||||
readonly follow?: boolean
|
||||
readonly signal?: AbortSignal
|
||||
}
|
||||
|
||||
|
|
@ -69,8 +77,9 @@ export interface GrepInput {
|
|||
}
|
||||
|
||||
export interface Interface {
|
||||
readonly files: (input: FilesInput) => Effect.Effect<Result<string>, Error>
|
||||
readonly grep: (input: GrepInput) => Effect.Effect<Result<Match>, Error | InvalidPatternError>
|
||||
readonly find: (input: FindInput) => Effect.Effect<readonly Entry[], Error>
|
||||
readonly glob: (input: GlobInput) => Effect.Effect<readonly Entry[], Error>
|
||||
readonly grep: (input: GrepInput) => Effect.Effect<readonly Match[], Error | InvalidPatternError>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/v2/Ripgrep") {}
|
||||
|
|
@ -84,7 +93,7 @@ export const layer = Layer.effect(
|
|||
Service,
|
||||
Effect.gen(function* () {
|
||||
const process = yield* AppProcess.Service
|
||||
const binary = yield* FileSystemRipgrep.Service
|
||||
const binary = yield* RipgrepBinary.Service
|
||||
|
||||
const run = <A>(input: {
|
||||
readonly cwd: string
|
||||
|
|
@ -137,31 +146,84 @@ export const layer = Layer.effect(
|
|||
}
|
||||
|
||||
return Service.of({
|
||||
files: (input) =>
|
||||
glob: (input) =>
|
||||
run<string>({
|
||||
...input,
|
||||
cwd: input.cwd,
|
||||
limit: input.limit,
|
||||
signal: input.signal,
|
||||
args: [
|
||||
"--no-config",
|
||||
"--files",
|
||||
"--glob=!.git/*", // TODO: Review .git exclusion policy before leaf tool exposure.
|
||||
"--glob=!**/.git/**",
|
||||
...(input.hidden ? ["--hidden"] : []),
|
||||
...(input.follow ? ["--follow"] : []),
|
||||
`--glob=${input.pattern}`,
|
||||
"--glob=!.*",
|
||||
"--glob=!**/.*",
|
||||
".",
|
||||
],
|
||||
parse: (line) => Effect.succeed(line.replace(/^\.\//, "")),
|
||||
}).pipe(Effect.catchTag("Ripgrep.InvalidPatternError", (cause) => Effect.fail(failure(cause.message, cause)))),
|
||||
parse: (line) =>
|
||||
Effect.succeed(
|
||||
line
|
||||
.replace(/^(?:\.[\\/])+/u, "")
|
||||
.replace(/^[\\/]+/u, "")
|
||||
.replaceAll("\\", "/"),
|
||||
),
|
||||
}).pipe(
|
||||
Effect.map((result) =>
|
||||
result.items.map((relative) => {
|
||||
const absolute = path.resolve(input.cwd, relative)
|
||||
return new Entry({
|
||||
path: RelativePath.make(relative),
|
||||
type: "file",
|
||||
mime: FSUtil.mimeType(absolute),
|
||||
})
|
||||
}),
|
||||
),
|
||||
Effect.catchTag("Ripgrep.InvalidPatternError", (cause) => Effect.fail(failure(cause.message, cause))),
|
||||
),
|
||||
find: (input) =>
|
||||
run<string>({
|
||||
cwd: input.cwd,
|
||||
limit: input.limit,
|
||||
signal: input.signal,
|
||||
args: [
|
||||
"--no-config",
|
||||
"--files",
|
||||
"--glob=!**/.git/**",
|
||||
...(input.hidden ? ["--hidden"] : []),
|
||||
...(input.follow ? ["--follow"] : []),
|
||||
`--glob=${input.pattern}`,
|
||||
".",
|
||||
],
|
||||
parse: (line) =>
|
||||
Effect.succeed(
|
||||
line
|
||||
.replace(/^(?:\.[\\/])+/u, "")
|
||||
.replace(/^[\\/]+/u, "")
|
||||
.replaceAll("\\", "/"),
|
||||
),
|
||||
}).pipe(
|
||||
Effect.map((result) =>
|
||||
result.items.map((relative) => {
|
||||
const absolute = path.resolve(input.cwd, relative)
|
||||
return new Entry({
|
||||
path: RelativePath.make(relative),
|
||||
type: "file",
|
||||
mime: FSUtil.mimeType(absolute),
|
||||
})
|
||||
}),
|
||||
),
|
||||
Effect.catchTag("Ripgrep.InvalidPatternError", (cause) => Effect.fail(failure(cause.message, cause))),
|
||||
),
|
||||
grep: (input) =>
|
||||
run<Match>({
|
||||
run<RawMatchData>({
|
||||
...input,
|
||||
args: [
|
||||
"--no-config",
|
||||
"--json",
|
||||
"--glob=!.git/*", // TODO: Review .git exclusion policy before leaf tool exposure.
|
||||
"--hidden",
|
||||
"--glob=!**/.git/**",
|
||||
"--no-messages",
|
||||
...(input.include ? [`--glob=${input.include}`] : []),
|
||||
"--glob=!.*",
|
||||
"--glob=!**/.*",
|
||||
"--",
|
||||
input.pattern,
|
||||
input.file ?? ".",
|
||||
|
|
@ -180,13 +242,43 @@ export const layer = Layer.effect(
|
|||
return Schema.decodeUnknownEffect(RawMatch)(json).pipe(
|
||||
Effect.map((match) => ({
|
||||
...match.data,
|
||||
path: { text: match.data.path.text.replace(/^\.[\\/]/, "") },
|
||||
submatches: match.data.submatches.slice(0, MAX_SUBMATCHES),
|
||||
})),
|
||||
Effect.mapError((cause) => failure("Invalid ripgrep match output", cause)),
|
||||
)
|
||||
}),
|
||||
),
|
||||
}),
|
||||
}).pipe(
|
||||
Effect.map((result) =>
|
||||
result.items.map((match) => {
|
||||
const relative = match.path.text
|
||||
.replace(/^(?:\.[\\/])+/u, "")
|
||||
.replace(/^[\\/]+/u, "")
|
||||
.replaceAll("\\", "/")
|
||||
const absolute = path.resolve(input.cwd, relative)
|
||||
return new Match({
|
||||
entry: new Entry({
|
||||
path: RelativePath.make(relative),
|
||||
type: "file",
|
||||
mime: FSUtil.mimeType(absolute),
|
||||
}),
|
||||
line: match.line_number,
|
||||
offset: match.absolute_offset,
|
||||
text: match.lines.text.length > 2_000 ? match.lines.text.slice(0, 2_000) + "..." : match.lines.text,
|
||||
submatches: match.submatches.map((submatch) => ({
|
||||
text: submatch.match.text,
|
||||
start: submatch.start,
|
||||
end: submatch.end,
|
||||
})),
|
||||
})
|
||||
}),
|
||||
),
|
||||
),
|
||||
})
|
||||
}),
|
||||
).pipe(Layer.provide(FileSystemRipgrep.defaultLayer))
|
||||
)
|
||||
|
||||
export const defaultLayer = layer.pipe(
|
||||
Layer.provide(Layer.merge(RipgrepBinary.defaultLayer, AppProcess.defaultLayer)),
|
||||
)
|
||||
|
|
|
|||
124
packages/core/src/ripgrep/binary.ts
Normal file
124
packages/core/src/ripgrep/binary.ts
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
import path from "path"
|
||||
import { Context, Effect, Layer, Stream } from "effect"
|
||||
import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http"
|
||||
import { ChildProcess } from "effect/unstable/process"
|
||||
import { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner"
|
||||
import { CrossSpawnSpawner } from "../cross-spawn-spawner"
|
||||
import { FSUtil } from "../fs-util"
|
||||
import { Global } from "../global"
|
||||
import { which } from "../util/which"
|
||||
|
||||
export namespace RipgrepBinary {
|
||||
const VERSION = "15.1.0"
|
||||
const PLATFORM = {
|
||||
"arm64-darwin": { platform: "aarch64-apple-darwin", extension: "tar.gz" },
|
||||
"arm64-linux": { platform: "aarch64-unknown-linux-gnu", extension: "tar.gz" },
|
||||
"x64-darwin": { platform: "x86_64-apple-darwin", extension: "tar.gz" },
|
||||
"x64-linux": { platform: "x86_64-unknown-linux-musl", extension: "tar.gz" },
|
||||
"arm64-win32": { platform: "aarch64-pc-windows-msvc", extension: "zip" },
|
||||
"ia32-win32": { platform: "i686-pc-windows-msvc", extension: "zip" },
|
||||
"x64-win32": { platform: "x86_64-pc-windows-msvc", extension: "zip" },
|
||||
} as const
|
||||
|
||||
interface Interface {
|
||||
readonly filepath: Effect.Effect<string, Error>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/RipgrepBinary") {}
|
||||
|
||||
export const layer = Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const fs = yield* FSUtil.Service
|
||||
const http = HttpClient.filterStatusOk(yield* HttpClient.HttpClient)
|
||||
const spawner = yield* ChildProcessSpawner
|
||||
|
||||
const run = Effect.fnUntraced(function* (command: string, args: string[]) {
|
||||
const handle = yield* spawner.spawn(ChildProcess.make(command, args, { extendEnv: true, stdin: "ignore" }))
|
||||
const [stdout, stderr, code] = yield* Effect.all(
|
||||
[
|
||||
Stream.mkString(Stream.decodeText(handle.stdout)),
|
||||
Stream.mkString(Stream.decodeText(handle.stderr)),
|
||||
handle.exitCode,
|
||||
],
|
||||
{ concurrency: "unbounded" },
|
||||
)
|
||||
return { stdout, stderr, code }
|
||||
}, Effect.scoped)
|
||||
|
||||
const extract = Effect.fnUntraced(function* (
|
||||
archive: string,
|
||||
config: (typeof PLATFORM)[keyof typeof PLATFORM],
|
||||
target: string,
|
||||
) {
|
||||
const dir = yield* fs.makeTempDirectoryScoped({ directory: Global.Path.bin, prefix: "ripgrep-" })
|
||||
|
||||
if (config.extension === "zip") {
|
||||
const shell = (yield* Effect.sync(() => which("powershell.exe") ?? which("pwsh.exe"))) ?? "powershell.exe"
|
||||
const result = yield* run(shell, [
|
||||
"-NoProfile",
|
||||
"-NonInteractive",
|
||||
"-Command",
|
||||
`$global:ProgressPreference = 'SilentlyContinue'; Expand-Archive -LiteralPath '${archive.replaceAll("'", "''")}' -DestinationPath '${dir.replaceAll("'", "''")}' -Force`,
|
||||
])
|
||||
if (result.code !== 0) throw new Error(result.stderr.trim() || result.stdout.trim() || `ripgrep extraction failed with code ${result.code}`)
|
||||
}
|
||||
|
||||
if (config.extension === "tar.gz") {
|
||||
const result = yield* run("tar", ["-xzf", archive, "-C", dir])
|
||||
if (result.code !== 0) throw new Error(result.stderr.trim() || result.stdout.trim() || `ripgrep extraction failed with code ${result.code}`)
|
||||
}
|
||||
|
||||
const extracted = path.join(
|
||||
dir,
|
||||
`ripgrep-${VERSION}-${config.platform}`,
|
||||
process.platform === "win32" ? "rg.exe" : "rg",
|
||||
)
|
||||
if (!(yield* fs.isFile(extracted))) throw new Error(`ripgrep archive did not contain executable: ${extracted}`)
|
||||
|
||||
yield* fs.copyFile(extracted, target)
|
||||
if (process.platform !== "win32") yield* fs.chmod(target, 0o755)
|
||||
}, Effect.scoped)
|
||||
|
||||
return Service.of({
|
||||
filepath: yield* Effect.cached(
|
||||
Effect.gen(function* () {
|
||||
const system = yield* Effect.sync(() => which(process.platform === "win32" ? "rg.exe" : "rg"))
|
||||
if (system && (yield* fs.isFile(system).pipe(Effect.orDie))) return system
|
||||
|
||||
const target = path.join(Global.Path.bin, `rg${process.platform === "win32" ? ".exe" : ""}`)
|
||||
if (yield* fs.isFile(target).pipe(Effect.orDie)) return target
|
||||
|
||||
const platformKey = `${process.arch}-${process.platform}` as keyof typeof PLATFORM
|
||||
const config = PLATFORM[platformKey]
|
||||
if (!config) throw new Error(`unsupported platform for ripgrep: ${platformKey}`)
|
||||
|
||||
const filename = `ripgrep-${VERSION}-${config.platform}.${config.extension}`
|
||||
const url = `https://github.com/BurntSushi/ripgrep/releases/download/${VERSION}/${filename}`
|
||||
const archive = path.join(Global.Path.bin, filename)
|
||||
|
||||
yield* Effect.logInfo("downloading ripgrep", { url })
|
||||
yield* fs.ensureDir(Global.Path.bin).pipe(Effect.orDie)
|
||||
const bytes = yield* HttpClientRequest.get(url).pipe(
|
||||
http.execute,
|
||||
Effect.flatMap((response) => response.arrayBuffer),
|
||||
Effect.mapError((cause) => (cause instanceof Error ? cause : new Error(String(cause)))),
|
||||
)
|
||||
if (bytes.byteLength === 0) throw new Error(`failed to download ripgrep from ${url}`)
|
||||
|
||||
yield* fs.writeWithDirs(archive, new Uint8Array(bytes))
|
||||
yield* extract(archive, config, target)
|
||||
yield* fs.remove(archive, { force: true }).pipe(Effect.ignore)
|
||||
return target
|
||||
}),
|
||||
),
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
export const defaultLayer = layer.pipe(
|
||||
Layer.provide(FetchHttpClient.layer),
|
||||
Layer.provide(FSUtil.defaultLayer),
|
||||
Layer.provide(CrossSpawnSpawner.defaultLayer),
|
||||
)
|
||||
}
|
||||
|
|
@ -2,7 +2,11 @@ export * as GlobTool from "./glob"
|
|||
|
||||
import { ToolFailure } from "@opencode-ai/llm"
|
||||
import { Effect, Layer, Schema } from "effect"
|
||||
import { LocationSearch } from "../location-search"
|
||||
import path from "path"
|
||||
import { FileSystem } from "../filesystem"
|
||||
import { Location } from "../location"
|
||||
import { Ripgrep } from "../ripgrep"
|
||||
import { RelativePath } from "../schema"
|
||||
import { PermissionV2 } from "../permission"
|
||||
import { Tool } from "./tool"
|
||||
import { Tools } from "./tools"
|
||||
|
|
@ -10,38 +14,30 @@ import { Tools } from "./tools"
|
|||
export const name = "glob"
|
||||
|
||||
export const Input = Schema.Struct({
|
||||
pattern: LocationSearch.FilesInput.fields.pattern.annotate({ description: "Glob pattern to match files against" }),
|
||||
path: LocationSearch.FilesInput.fields.path.annotate({
|
||||
pattern: FileSystem.GlobInput.fields.pattern.annotate({ description: "Glob pattern to match files against" }),
|
||||
path: RelativePath.pipe(Schema.optional).annotate({
|
||||
description: "Relative directory to search. Defaults to the active Location.",
|
||||
}),
|
||||
limit: LocationSearch.FilesInput.fields.limit.annotate({
|
||||
description: `Maximum results to return (default: ${LocationSearch.DEFAULT_RESULT_LIMIT})`,
|
||||
limit: FileSystem.GlobInput.fields.limit.annotate({
|
||||
description: "Maximum results to return",
|
||||
}),
|
||||
})
|
||||
|
||||
type ModelOutput = typeof LocationSearch.FilesResult.Encoded
|
||||
export const Output = Schema.Array(FileSystem.Entry)
|
||||
type ModelOutput = typeof Output.Encoded
|
||||
|
||||
/** Format raw Location search results into the concise line-oriented output models expect. */
|
||||
/** Format raw search results into the concise line-oriented output models expect. */
|
||||
export const toModelOutput = (output: ModelOutput) => {
|
||||
const lines = output.items.length === 0 ? ["No files found"] : output.items.map((item) => item.resource)
|
||||
if (output.truncated) {
|
||||
lines.push(
|
||||
"",
|
||||
`(Results are truncated: showing first ${output.items.length} results. Consider using a more specific path or pattern.)`,
|
||||
)
|
||||
}
|
||||
if (output.partial) lines.push("", "(Results may be incomplete because some discovered files could not be read.)")
|
||||
const lines = output.length === 0 ? ["No files found"] : output.map((item) => item.path)
|
||||
return lines.join("\n")
|
||||
}
|
||||
|
||||
/**
|
||||
* Location-scoped glob leaf. FileSystem supplies canonical permission metadata;
|
||||
* LocationSearch resolves the current root and owns containment and traversal.
|
||||
*/
|
||||
/** Glob leaf that defaults its filesystem root to the active Location. */
|
||||
export const layer = Layer.effectDiscard(
|
||||
Effect.gen(function* () {
|
||||
const tools = yield* Tools.Service
|
||||
const search = yield* LocationSearch.Service
|
||||
const ripgrep = yield* Ripgrep.Service
|
||||
const location = yield* Location.Service
|
||||
const permission = yield* PermissionV2.Service
|
||||
|
||||
yield* tools
|
||||
|
|
@ -50,8 +46,13 @@ export const layer = Layer.effectDiscard(
|
|||
description:
|
||||
"Find files by glob pattern within the active Location. Returns concise relative file resources. Use a relative path to narrow the search and limit to bound the result count.",
|
||||
input: Input,
|
||||
output: LocationSearch.FilesResult,
|
||||
toModelOutput: ({ output }) => [{ type: "text", text: toModelOutput(output) }],
|
||||
output: Output,
|
||||
toModelOutput: ({ output }) => [
|
||||
{
|
||||
type: "text",
|
||||
text: toModelOutput(output.map((entry) => ({ ...entry, path: path.resolve(location.directory, entry.path) }))),
|
||||
},
|
||||
],
|
||||
execute: (input, context) =>
|
||||
Effect.gen(function* () {
|
||||
yield* permission.assert({
|
||||
|
|
@ -67,7 +68,22 @@ export const layer = Layer.effectDiscard(
|
|||
agent: context.agent,
|
||||
source: { type: "tool", messageID: context.assistantMessageID, callID: context.toolCallID },
|
||||
})
|
||||
return yield* search.files(input)
|
||||
const cwd = path.resolve(location.directory, input.path ?? ".")
|
||||
return yield* ripgrep.glob({
|
||||
cwd,
|
||||
pattern: input.pattern,
|
||||
limit: input.limit ?? Number.MAX_SAFE_INTEGER,
|
||||
}).pipe(
|
||||
Effect.map((result) =>
|
||||
result.map(
|
||||
(entry) =>
|
||||
new FileSystem.Entry({
|
||||
...entry,
|
||||
path: RelativePath.make(path.relative(location.directory, path.resolve(cwd, entry.path))),
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
}).pipe(
|
||||
Effect.mapError(() => new ToolFailure({ message: `Unable to find files matching ${input.pattern}` })),
|
||||
),
|
||||
|
|
|
|||
|
|
@ -2,61 +2,58 @@ export * as GrepTool from "./grep"
|
|||
|
||||
import { ToolFailure } from "@opencode-ai/llm"
|
||||
import { Effect, Layer, Schema } from "effect"
|
||||
import { LocationSearch } from "../location-search"
|
||||
import { Ripgrep } from "../ripgrep"
|
||||
import path from "path"
|
||||
import { FileSystem } from "../filesystem"
|
||||
import { FSUtil } from "../fs-util"
|
||||
import { Location } from "../location"
|
||||
import { PermissionV2 } from "../permission"
|
||||
import { Ripgrep } from "../ripgrep"
|
||||
import { RelativePath } from "../schema"
|
||||
import { Tool } from "./tool"
|
||||
import { Tools } from "./tools"
|
||||
|
||||
export const name = "grep"
|
||||
|
||||
export const Input = Schema.Struct({
|
||||
pattern: LocationSearch.GrepInput.fields.pattern.annotate({
|
||||
pattern: FileSystem.GrepInput.fields.pattern.annotate({
|
||||
description: "Regex pattern to search for in file contents",
|
||||
}),
|
||||
path: LocationSearch.GrepInput.fields.path.annotate({
|
||||
description: "Relative file or directory to search. Defaults to the active Location.",
|
||||
path: RelativePath.pipe(Schema.optional).annotate({
|
||||
description: "Relative directory to search. Defaults to the active Location.",
|
||||
}),
|
||||
include: LocationSearch.GrepInput.fields.include.annotate({
|
||||
include: FileSystem.GrepInput.fields.include.annotate({
|
||||
description: 'File glob to include in the search (for example, "*.js" or "*.{ts,tsx}")',
|
||||
}),
|
||||
limit: LocationSearch.GrepInput.fields.limit.annotate({
|
||||
description: `Maximum matches to return (default: ${LocationSearch.DEFAULT_RESULT_LIMIT})`,
|
||||
limit: FileSystem.GrepInput.fields.limit.annotate({
|
||||
description: "Maximum matches to return",
|
||||
}),
|
||||
})
|
||||
|
||||
type Output = typeof LocationSearch.GrepResult.Encoded
|
||||
export const Output = Schema.Array(FileSystem.Match)
|
||||
type ModelOutput = typeof Output.Encoded
|
||||
|
||||
/** Format raw Location search matches into the familiar concise model output. */
|
||||
export const toModelOutput = (output: Output) => {
|
||||
const lines = output.items.length === 0 ? ["No files found"] : [`Found ${output.items.length} matches`]
|
||||
/** Format raw search matches into the familiar concise model output. */
|
||||
export const toModelOutput = (output: ModelOutput) => {
|
||||
const lines = output.length === 0 ? ["No files found"] : [`Found ${output.length} matches`]
|
||||
let current = ""
|
||||
for (const match of output.items) {
|
||||
if (current !== match.resource) {
|
||||
for (const match of output) {
|
||||
if (current !== match.entry.path) {
|
||||
if (current) lines.push("")
|
||||
current = match.resource
|
||||
lines.push(`${match.resource}:`)
|
||||
current = match.entry.path
|
||||
lines.push(`${match.entry.path}:`)
|
||||
}
|
||||
lines.push(` Line ${match.line}: ${match.lines}${match.linePreviewTruncated ? "..." : ""}`)
|
||||
lines.push(` Line ${match.line}: ${match.text}`)
|
||||
}
|
||||
if (output.truncated) {
|
||||
lines.push(
|
||||
"",
|
||||
`(Results are truncated: showing first ${output.items.length} matches. Consider using a more specific path or pattern.)`,
|
||||
)
|
||||
}
|
||||
if (output.partial) lines.push("", "(Some paths were inaccessible and skipped)")
|
||||
return lines.join("\n")
|
||||
}
|
||||
|
||||
/**
|
||||
* Location-scoped grep leaf. FileSystem supplies canonical permission metadata;
|
||||
* LocationSearch resolves the current root and owns containment and ripgrep execution.
|
||||
*/
|
||||
/** Grep leaf that defaults its filesystem root to the active Location. */
|
||||
export const layer = Layer.effectDiscard(
|
||||
Effect.gen(function* () {
|
||||
const tools = yield* Tools.Service
|
||||
const search = yield* LocationSearch.Service
|
||||
const fs = yield* FSUtil.Service
|
||||
const ripgrep = yield* Ripgrep.Service
|
||||
const location = yield* Location.Service
|
||||
const permission = yield* PermissionV2.Service
|
||||
|
||||
yield* tools
|
||||
|
|
@ -65,8 +62,18 @@ export const layer = Layer.effectDiscard(
|
|||
description:
|
||||
"Search file contents by regular expression within the active Location or an absolute managed tool-output file. Use a path to narrow the search, include to filter files by glob, and limit to bound the match count. Returns concise file resources, line numbers, and bounded line previews.",
|
||||
input: Input,
|
||||
output: LocationSearch.GrepResult,
|
||||
toModelOutput: ({ output }) => [{ type: "text", text: toModelOutput(output) }],
|
||||
output: Output,
|
||||
toModelOutput: ({ output }) => [
|
||||
{
|
||||
type: "text",
|
||||
text: toModelOutput(
|
||||
output.map((match) => ({
|
||||
...match,
|
||||
entry: { ...match.entry, path: path.resolve(location.directory, match.entry.path) },
|
||||
})),
|
||||
),
|
||||
},
|
||||
],
|
||||
execute: (input, context) =>
|
||||
Effect.gen(function* () {
|
||||
yield* permission.assert({
|
||||
|
|
@ -74,7 +81,7 @@ export const layer = Layer.effectDiscard(
|
|||
resources: [input.pattern],
|
||||
save: ["*"],
|
||||
metadata: {
|
||||
root: input.path ?? ".",
|
||||
root: ".",
|
||||
path: input.path,
|
||||
include: input.include,
|
||||
limit: input.limit,
|
||||
|
|
@ -83,15 +90,35 @@ export const layer = Layer.effectDiscard(
|
|||
agent: context.agent,
|
||||
source: { type: "tool", messageID: context.assistantMessageID, callID: context.toolCallID },
|
||||
})
|
||||
return yield* search.grep(input)
|
||||
const target = path.resolve(location.directory, input.path ?? ".")
|
||||
const info = yield* fs.stat(target).pipe(Effect.catch(() => Effect.succeed(undefined)))
|
||||
return yield* ripgrep.grep({
|
||||
cwd: info?.type === "Directory" ? target : path.dirname(target),
|
||||
pattern: input.pattern,
|
||||
file: info?.type === "File" ? path.basename(target) : undefined,
|
||||
include: input.include,
|
||||
limit: input.limit ?? Number.MAX_SAFE_INTEGER,
|
||||
}).pipe(
|
||||
Effect.map((result) =>
|
||||
result.map(
|
||||
(match) =>
|
||||
new FileSystem.Match({
|
||||
...match,
|
||||
entry: new FileSystem.Entry({
|
||||
...match.entry,
|
||||
path: RelativePath.make(
|
||||
path.relative(
|
||||
location.directory,
|
||||
path.resolve(info?.type === "Directory" ? target : path.dirname(target), match.entry.path),
|
||||
),
|
||||
),
|
||||
}),
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
}).pipe(
|
||||
Effect.mapError((error) => {
|
||||
const message =
|
||||
error instanceof Ripgrep.InvalidPatternError
|
||||
? `Invalid grep pattern ${JSON.stringify(input.pattern)}: ${error.message}`
|
||||
: `Unable to grep for ${input.pattern}`
|
||||
return new ToolFailure({ message })
|
||||
}),
|
||||
Effect.mapError(() => new ToolFailure({ message: `Unable to grep for ${input.pattern}` })),
|
||||
),
|
||||
}),
|
||||
})
|
||||
|
|
|
|||
|
|
@ -276,8 +276,7 @@ export const list = Effect.fn("ReadTool.list")(function* (fs: FSUtil.Interface,
|
|||
const type = info?.type === "Directory" ? "directory" : info?.type === "File" ? "file" : undefined
|
||||
if (!type) return
|
||||
return new FileSystem.Entry({
|
||||
path: RelativePath.make(item.name),
|
||||
uri: pathToFileURL(target).href,
|
||||
path: RelativePath.make(item.name + (type === "directory" ? path.sep : "")),
|
||||
type,
|
||||
mime: type === "directory" ? "application/x-directory" : FSUtil.mimeType(target),
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue