refactor(core): simplify Firecrawl adapter
This commit is contained in:
parent
c8f86e2f34
commit
f38e558910
2 changed files with 38 additions and 52 deletions
|
|
@ -1,6 +1,7 @@
|
||||||
export * as SearchFirecrawl from "./firecrawl"
|
export * as SearchFirecrawl from "./firecrawl"
|
||||||
|
|
||||||
import { define } from "@opencode-ai/plugin/v2/effect"
|
import { define } from "@opencode-ai/plugin/v2/effect"
|
||||||
|
import type { Search } from "@opencode-ai/schema/search"
|
||||||
import { Duration, Effect, Schema, Scope } from "effect"
|
import { Duration, Effect, Schema, Scope } from "effect"
|
||||||
import { HttpClient, HttpClientRequest } from "effect/unstable/http"
|
import { HttpClient, HttpClientRequest } from "effect/unstable/http"
|
||||||
import { collectBoundedResponseBody } from "../../tool/http-body"
|
import { collectBoundedResponseBody } from "../../tool/http-body"
|
||||||
|
|
@ -8,36 +9,37 @@ import { SearchMcp } from "./mcp"
|
||||||
|
|
||||||
export const endpoint = "https://api.firecrawl.dev/v2/search"
|
export const endpoint = "https://api.firecrawl.dev/v2/search"
|
||||||
|
|
||||||
const Request = Schema.Struct({
|
const FirecrawlRequest = Schema.Struct({
|
||||||
query: Schema.String,
|
query: Schema.String,
|
||||||
limit: Schema.optional(Schema.Number),
|
limit: Schema.optional(Schema.Number),
|
||||||
})
|
})
|
||||||
|
|
||||||
const WebResult = Schema.Struct({
|
const ResultBase = {
|
||||||
url: Schema.String,
|
url: Schema.String,
|
||||||
title: Schema.optional(Schema.String),
|
title: Schema.optional(Schema.String),
|
||||||
description: Schema.optional(Schema.String),
|
|
||||||
markdown: Schema.optional(Schema.NullOr(Schema.String)),
|
|
||||||
position: Schema.optional(Schema.Number),
|
position: Schema.optional(Schema.Number),
|
||||||
|
}
|
||||||
|
const PageResult = {
|
||||||
|
...ResultBase,
|
||||||
|
markdown: Schema.optional(Schema.NullOr(Schema.String)),
|
||||||
|
}
|
||||||
|
const WebResult = Schema.Struct({
|
||||||
|
...PageResult,
|
||||||
|
description: Schema.optional(Schema.String),
|
||||||
category: Schema.optional(Schema.String),
|
category: Schema.optional(Schema.String),
|
||||||
})
|
})
|
||||||
const NewsResult = Schema.Struct({
|
const NewsResult = Schema.Struct({
|
||||||
url: Schema.String,
|
...PageResult,
|
||||||
title: Schema.optional(Schema.String),
|
|
||||||
snippet: Schema.optional(Schema.String),
|
snippet: Schema.optional(Schema.String),
|
||||||
date: Schema.optional(Schema.String),
|
date: Schema.optional(Schema.String),
|
||||||
markdown: Schema.optional(Schema.NullOr(Schema.String)),
|
|
||||||
position: Schema.optional(Schema.Number),
|
|
||||||
})
|
})
|
||||||
const ImageResult = Schema.Struct({
|
const ImageResult = Schema.Struct({
|
||||||
url: Schema.String,
|
...ResultBase,
|
||||||
title: Schema.optional(Schema.String),
|
|
||||||
imageUrl: Schema.String,
|
imageUrl: Schema.String,
|
||||||
imageWidth: Schema.optional(Schema.Number),
|
imageWidth: Schema.optional(Schema.Number),
|
||||||
imageHeight: Schema.optional(Schema.Number),
|
imageHeight: Schema.optional(Schema.Number),
|
||||||
position: Schema.optional(Schema.Number),
|
|
||||||
})
|
})
|
||||||
const Response = Schema.Struct({
|
const FirecrawlResponse = Schema.Struct({
|
||||||
success: Schema.Literal(true),
|
success: Schema.Literal(true),
|
||||||
data: Schema.Struct({
|
data: Schema.Struct({
|
||||||
web: Schema.optional(Schema.Array(WebResult)),
|
web: Schema.optional(Schema.Array(WebResult)),
|
||||||
|
|
@ -49,9 +51,9 @@ const Response = Schema.Struct({
|
||||||
creditsUsed: Schema.optional(Schema.Number),
|
creditsUsed: Schema.optional(Schema.Number),
|
||||||
})
|
})
|
||||||
const decodeJson = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Json))
|
const decodeJson = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Json))
|
||||||
const decodeResponse = Schema.decodeUnknownEffect(Response)
|
const decodeResponse = Schema.decodeUnknownEffect(FirecrawlResponse)
|
||||||
|
|
||||||
const format = (response: typeof Response.Type) =>
|
const formatResults = (response: typeof FirecrawlResponse.Type) =>
|
||||||
[
|
[
|
||||||
...(response.data.web ?? []).map((result) =>
|
...(response.data.web ?? []).map((result) =>
|
||||||
[`## ${result.title ?? result.url}`, `URL: ${result.url}`, result.description, result.markdown || undefined]
|
[`## ${result.title ?? result.url}`, `URL: ${result.url}`, result.description, result.markdown || undefined]
|
||||||
|
|
@ -76,16 +78,14 @@ const format = (response: typeof Response.Type) =>
|
||||||
|
|
||||||
const search = (
|
const search = (
|
||||||
http: HttpClient.HttpClient,
|
http: HttpClient.HttpClient,
|
||||||
input: { readonly query: string; readonly numResults?: number; readonly contextMaxCharacters?: number },
|
input: Pick<Search.Input, "query" | "numResults" | "contextMaxCharacters">,
|
||||||
apiKey?: string,
|
apiKey?: string,
|
||||||
) =>
|
) =>
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
const request = yield* HttpClientRequest.post(endpoint).pipe(
|
const request = yield* HttpClientRequest.post(endpoint).pipe(
|
||||||
HttpClientRequest.acceptJson,
|
HttpClientRequest.acceptJson,
|
||||||
HttpClientRequest.setHeaders({
|
HttpClientRequest.setHeaders(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
|
||||||
...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
|
HttpClientRequest.schemaBodyJson(FirecrawlRequest)({ query: input.query, limit: input.numResults }),
|
||||||
}),
|
|
||||||
HttpClientRequest.schemaBodyJson(Request)({ query: input.query, limit: input.numResults }),
|
|
||||||
)
|
)
|
||||||
const response = yield* HttpClient.filterStatusOk(http).execute(request)
|
const response = yield* HttpClient.filterStatusOk(http).execute(request)
|
||||||
const body = yield* collectBoundedResponseBody(
|
const body = yield* collectBoundedResponseBody(
|
||||||
|
|
@ -95,7 +95,7 @@ const search = (
|
||||||
)
|
)
|
||||||
const metadata = yield* decodeJson(body.toString("utf8"))
|
const metadata = yield* decodeJson(body.toString("utf8"))
|
||||||
const result = yield* decodeResponse(metadata)
|
const result = yield* decodeResponse(metadata)
|
||||||
const text = format(result)
|
const text = formatResults(result)
|
||||||
return {
|
return {
|
||||||
text: input.contextMaxCharacters ? text.slice(0, input.contextMaxCharacters) : text,
|
text: input.contextMaxCharacters ? text.slice(0, input.contextMaxCharacters) : text,
|
||||||
metadata,
|
metadata,
|
||||||
|
|
|
||||||
|
|
@ -6,24 +6,24 @@ import { SearchFirecrawl } from "@opencode-ai/core/plugin/search/firecrawl"
|
||||||
import { host, integrationHost } from "./host"
|
import { host, integrationHost } from "./host"
|
||||||
import { requests, resetSearchFixture, searchIntegrationTest } from "./search-fixture"
|
import { requests, resetSearchFixture, searchIntegrationTest } from "./search-fixture"
|
||||||
|
|
||||||
beforeEach(() => {
|
const metadata = {
|
||||||
resetSearchFixture(
|
success: true,
|
||||||
JSON.stringify({
|
data: {
|
||||||
success: true,
|
web: [
|
||||||
data: {
|
{
|
||||||
web: [
|
url: "https://effect.website/",
|
||||||
{
|
title: "Effect",
|
||||||
url: "https://effect.website/",
|
description: "Build production TypeScript applications.",
|
||||||
title: "Effect",
|
position: 1,
|
||||||
description: "Build production TypeScript applications.",
|
|
||||||
position: 1,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
},
|
||||||
id: "search_1",
|
],
|
||||||
creditsUsed: 2,
|
},
|
||||||
}),
|
id: "search_1",
|
||||||
)
|
creditsUsed: 2,
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
resetSearchFixture(JSON.stringify(metadata))
|
||||||
})
|
})
|
||||||
|
|
||||||
const it = searchIntegrationTest
|
const it = searchIntegrationTest
|
||||||
|
|
@ -39,21 +39,7 @@ describe("Firecrawl search integration", () => {
|
||||||
const output = yield* provider.execute({ query: "effect", numResults: 3 }, {})
|
const output = yield* provider.execute({ query: "effect", numResults: 3 }, {})
|
||||||
expect(output).toEqual({
|
expect(output).toEqual({
|
||||||
text: "## Effect\n\nURL: https://effect.website/\n\nBuild production TypeScript applications.",
|
text: "## Effect\n\nURL: https://effect.website/\n\nBuild production TypeScript applications.",
|
||||||
metadata: {
|
metadata,
|
||||||
success: true,
|
|
||||||
data: {
|
|
||||||
web: [
|
|
||||||
{
|
|
||||||
url: "https://effect.website/",
|
|
||||||
title: "Effect",
|
|
||||||
description: "Build production TypeScript applications.",
|
|
||||||
position: 1,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
id: "search_1",
|
|
||||||
creditsUsed: 2,
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
expect(requests).toEqual([
|
expect(requests).toEqual([
|
||||||
{
|
{
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue