feat(core): add skill registry and file agent loading (#30617)

packages/core/src/skill/discovery.ts

@@ -0,0 +1,99 @@
+export * as SkillDiscovery from "./discovery"
+
+import path from "path"
+import { Context, Effect, Layer, Schedule, Schema } from "effect"
+import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
+import { FSUtil } from "../fs-util"
+import { Global } from "../global"
+import { AbsolutePath } from "../schema"
+import * as Log from "../util/log"
+
+const skillConcurrency = 4
+const fileConcurrency = 8
+
+class IndexSkill extends Schema.Class<IndexSkill>("SkillDiscovery.IndexSkill")({
+  name: Schema.String,
+  files: Schema.Array(Schema.String),
+}) {}
+
+class Index extends Schema.Class<Index>("SkillDiscovery.Index")({
+  skills: Schema.Array(IndexSkill),
+}) {}
+
+export interface Interface {
+  readonly pull: (url: string) => Effect.Effect<AbsolutePath[]>
+}
+
+export class Service extends Context.Service<Service, Interface>()("@opencode/v2/SkillDiscovery") {}
+
+export const layer = Layer.effect(
+  Service,
+  Effect.gen(function* () {
+    const fs = yield* FSUtil.Service
+    const global = yield* Global.Service
+    const log = Log.create({ service: "skill-discovery" })
+    const http = (yield* HttpClient.HttpClient).pipe(
+      HttpClient.retryTransient({
+        retryOn: "errors-and-responses",
+        times: 2,
+        schedule: Schedule.exponential(200).pipe(Schedule.jittered),
+      }),
+      HttpClient.filterStatusOk,
+    )
+
+    const download = Effect.fn("SkillDiscovery.download")(function* (url: string, destination: string) {
+      if (yield* fs.exists(destination).pipe(Effect.orDie)) return
+      yield* HttpClientRequest.get(url).pipe(
+        http.execute,
+        Effect.flatMap((response) => response.arrayBuffer),
+        Effect.flatMap((body) => fs.writeWithDirs(destination, new Uint8Array(body))),
+        Effect.catch((error) => Effect.sync(() => log.error("failed to download skill file", { url, error }))),
+      )
+    })
+
+    return Service.of({
+      pull: Effect.fn("SkillDiscovery.pull")(function* (url) {
+        const base = url.endsWith("/") ? url : `${url}/`
+        const index = new URL("index.json", base).href
+        const data = yield* HttpClientRequest.get(index).pipe(
+          HttpClientRequest.acceptJson,
+          http.execute,
+          Effect.flatMap(HttpClientResponse.schemaBodyJson(Index)),
+          Effect.catch((error) => {
+            log.error("failed to fetch skill index", { url: index, error })
+            return Effect.succeed(undefined)
+          }),
+        )
+        if (!data) return []
+
+        return yield* Effect.forEach(
+          data.skills.filter((skill) => {
+            if (skill.files.includes("SKILL.md") || skill.files.includes(`${skill.name}.md`)) return true
+            log.warn("skill entry missing Markdown definition", { url: index, skill: skill.name })
+            return false
+          }),
+          (skill) =>
+            Effect.gen(function* () {
+              const root = path.join(global.cache, "skills", Bun.hash(base).toString(16), skill.name)
+              yield* Effect.forEach(
+                skill.files,
+                (file) => download(new URL(file, `${base}${skill.name}/`).href, path.join(root, file)),
+                { concurrency: fileConcurrency, discard: true },
+              )
+              return (yield* fs.exists(path.join(root, "SKILL.md")).pipe(Effect.orDie)) ||
+                (yield* fs.exists(path.join(root, `${skill.name}.md`)).pipe(Effect.orDie))
+                ? [AbsolutePath.make(root)]
+                : []
+            }),
+          { concurrency: skillConcurrency },
+        ).pipe(Effect.map((directories) => directories.flat()))
+      }),
+    })
+  }),
+)
+
+export const defaultLayer = layer.pipe(
+  Layer.provide(FetchHttpClient.layer),
+  Layer.provide(FSUtil.defaultLayer),
+  Layer.provide(Global.defaultLayer),
+)