ceos/opencode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactor LLM route-first provider API (#28523)

Browse source
This commit is contained in:
Kit Langton 2026-05-20 20:15:52 -04:00 • committed by GitHub
commit 41f6daf96a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
87 changed files with 2436 additions and 1506 deletions
Download patch file Download diff file Expand all files Collapse all files

99
packages/llm/test/adapter.test.ts
View file

@ -1,12 +1,12 @@
import { describe, expect } from "bun:test"
import { Effect, Schema, Stream } from "effect"
import { LLM } from "../src"
import { Route, Endpoint, LLMClient, Protocol, type RouteModelInput, type FramingDef } from "../src/route"
import { ModelRef } from "../src/schema"
import { Route, Endpoint, LLMClient, Protocol, type FramingDef } from "../src/route"
import { Model } from "../src/schema"
import { testEffect } from "./lib/effect"
import { dynamicResponse } from "./lib/http"
const updateModel = (model: ModelRef, patch: Partial<ModelRef.Input>) => ModelRef.update(model, patch)
const updateModel = (model: Model, patch: Partial<Model.Input>) => Model.update(model, patch)
const Json = Schema.fromJsonString(Schema.Unknown)
const encodeJson = Schema.encodeSync(Json)
@ -38,17 +38,6 @@ const fakeFraming: FramingDef<FakeEvent> = {
).pipe(Stream.flatMap(Stream.fromIterable)),
}
const request = LLM.request({
id: "req_1",
model: LLM.model({
id: "fake-model",
provider: "fake-provider",
route: "fake",
baseURL: "https://fake.local",
}),
prompt: "hello",
})
const raiseEvent = (event: FakeEvent): import("../src/schema").LLMEvent =>
event.type === "finish"
? { type: "finish", reason: event.reason }
@ -84,6 +73,7 @@ const fake = Route.make({
endpoint: Endpoint.path("/chat"),
framing: fakeFraming,
})
const configuredFake = fake.with({ endpoint: { baseURL: "https://fake.local" } })
const gemini = Route.make({
id: "gemini-fake",
@ -91,6 +81,17 @@ const gemini = Route.make({
endpoint: Endpoint.path("/chat"),
framing: fakeFraming,
})
const configuredGemini = gemini.with({ endpoint: { baseURL: "https://fake.local" } })
const request = LLM.request({
id: "req_1",
model: Model.make({
id: "fake-model",
provider: "fake-provider",
route: configuredFake,
}),
prompt: "hello",
})
const echoLayer = dynamicResponse(({ text, respond }) =>
Effect.succeed(
@ -117,61 +118,47 @@ describe("llm route", () => {
}),
)
it.effect("selects routes by request route", () =>
it.effect("selects routes by model route value", () =>
Effect.gen(function* () {
const llm = yield* LLMClient.Service
const prepared = yield* llm.prepare(
LLM.updateRequest(request, { model: updateModel(request.model, { route: "gemini-fake" }) }),
LLM.updateRequest(request, { model: updateModel(request.model, { route: configuredGemini }) }),
)
expect(prepared.route).toBe("gemini-fake")
}),
)
it.effect("maps model input before building refs", () =>
it.effect("builds models from configured routes", () =>
Effect.gen(function* () {
const mapped = Route.model<RouteModelInput & { readonly region?: string }>(
fake,
{ provider: "fake-provider", baseURL: "https://fake.local" },
{
mapInput: (input) => {
const { region, ...rest } = input
return { ...rest, native: { region } }
const configured = fake.with({ provider: "fake-provider", endpoint: { baseURL: "https://fake.local" } })
expect(configured.model({ id: "fake-model" })).toMatchObject({
provider: "fake-provider",
})
}),
)
it.effect("does not register duplicate route ids globally", () =>
Effect.gen(function* () {
const duplicate = Route.make({
id: "fake",
protocol: Protocol.make({
...fakeProtocol,
body: {
...fakeProtocol.body,
from: () => Effect.succeed({ body: "late-default" }),
},
},
}),
endpoint: Endpoint.path("/chat", { baseURL: "https://fake.local" }),
framing: fakeFraming,
})
const prepared = yield* (yield* LLMClient.Service).prepare(
LLM.updateRequest(request, { model: updateModel(request.model, { route: duplicate }) }),
)
expect(mapped({ id: "fake-model", region: "us-east-1" }).native).toEqual({ region: "us-east-1" })
}),
)
it.effect("rejects duplicate route ids", () =>
Effect.gen(function* () {
expect(() =>
Route.make({
id: "fake",
protocol: Protocol.make({
...fakeProtocol,
body: {
...fakeProtocol.body,
from: () => Effect.succeed({ body: "late-default" }),
},
}),
endpoint: Endpoint.path("/chat"),
framing: fakeFraming,
}),
).toThrow('Duplicate LLM route id "fake"')
}),
)
it.effect("rejects missing route", () =>
Effect.gen(function* () {
const llm = yield* LLMClient.Service
const error = yield* llm
.prepare(LLM.updateRequest(request, { model: updateModel(request.model, { route: "missing" }) }))
.pipe(Effect.flip)
expect(error.message).toContain("No LLM route")
expect(prepared.body).toEqual({ body: "late-default" })
}),
)
})

124
packages/llm/test/auth-options.types.ts
View file

@ -2,8 +2,17 @@ import { Config } from "effect"
import type { Auth } from "../src/route/auth"
import type { ModelFactory } from "../src/route/auth-options"
import { Auth as RuntimeAuth } from "../src/route/auth"
import * as OpenAIChat from "../src/protocols/openai-chat"
import * as AmazonBedrock from "../src/providers/amazon-bedrock"
import * as Anthropic from "../src/providers/anthropic"
import * as Azure from "../src/providers/azure"
import * as Cloudflare from "../src/providers/cloudflare"
import * as GitHubCopilot from "../src/providers/github-copilot"
import * as Google from "../src/providers/google"
import * as OpenAI from "../src/providers/openai"
import * as OpenAICompatible from "../src/providers/openai-compatible"
import * as OpenRouter from "../src/providers/openrouter"
import * as XAI from "../src/providers/xai"
type BaseOptions = {
readonly baseURL?: string
@ -19,6 +28,20 @@ declare const optionalAuthModel: ModelFactory<BaseOptions, "optional", Model>
declare const requiredAuthModel: ModelFactory<BaseOptions, "required", Model>
const configApiKey = Config.redacted("OPENAI_API_KEY")
OpenAIChat.route.model({ id: "gpt-4.1-mini" })
// @ts-expect-error route model selection does not configure endpoints.
OpenAIChat.route.model({ id: "gpt-4.1-mini", baseURL: "https://gateway.example.com/v1" })
// @ts-expect-error route model selection does not configure query params.
OpenAIChat.route.model({ id: "gpt-4.1-mini", queryParams: { debug: "1" } })
// @ts-expect-error route model selection does not configure auth.
OpenAIChat.route.model({ id: "gpt-4.1-mini", auth })
// @ts-expect-error route model selection does not configure api keys.
OpenAIChat.route.model({ id: "gpt-4.1-mini", apiKey: "sk-test" })
optionalAuthModel("gpt-4.1-mini")
optionalAuthModel("gpt-4.1-mini", {})
optionalAuthModel("gpt-4.1-mini", { apiKey: "sk-test" })
@ -45,56 +68,101 @@ requiredAuthModel("custom-model", {})
requiredAuthModel("custom-model", { apiKey: "key", auth })
OpenAI.responses("gpt-4.1-mini")
OpenAI.responses("gpt-4.1-mini", {})
OpenAI.responses("gpt-4.1-mini", { apiKey: "sk-test" })
OpenAI.responses("gpt-4.1-mini", { apiKey: configApiKey })
OpenAI.responses("gpt-4.1-mini", { auth: RuntimeAuth.bearer("oauth-token") })
OpenAI.responses("gpt-4.1-mini", {
OpenAI.configure({}).responses("gpt-4.1-mini")
OpenAI.configure({ apiKey: "sk-test" }).responses("gpt-4.1-mini")
OpenAI.configure({ apiKey: configApiKey }).responses("gpt-4.1-mini")
OpenAI.configure({ auth: RuntimeAuth.bearer("oauth-token") }).responses("gpt-4.1-mini")
OpenAI.configure({
auth: RuntimeAuth.headers({ authorization: "Bearer gateway" }),
baseURL: "https://gateway.example.com/v1",
})
OpenAI.responses("gpt-4.1-mini", {
}).responses("gpt-4.1-mini")
OpenAI.configure({
generation: { maxTokens: 100 },
providerOptions: { openai: { store: false } },
})
}).responses("gpt-4.1-mini")
// @ts-expect-error OpenAI model selectors only accept model ids.
OpenAI.configure({ apiKey: "sk-test" }).responses("gpt-4.1-mini", {})
// @ts-expect-error apiKey only accepts string, Redacted<string>, or Config<string | Redacted<string>>.
OpenAI.responses("gpt-4.1-mini", { apiKey: 123 })
OpenAI.configure({ apiKey: 123 })
// @ts-expect-error provider helpers reject unknown top-level options.
OpenAI.responses("gpt-4.1-mini", { bogus: true })
OpenAI.configure({ bogus: true })
// @ts-expect-error common generation options remain typed.
OpenAI.responses("gpt-4.1-mini", { generation: { maxTokens: "many" } })
OpenAI.configure({ generation: { maxTokens: "many" } })
// @ts-expect-error provider-native options remain typed.
OpenAI.responses("gpt-4.1-mini", { providerOptions: { openai: { store: "false" } } })
OpenAI.configure({ providerOptions: { openai: { store: "false" } } })
// @ts-expect-error auth is an override, so OpenAI rejects apiKey with auth.
OpenAI.responses("gpt-4.1-mini", { apiKey: "sk-test", auth: RuntimeAuth.bearer("oauth-token") })
OpenAI.configure({ apiKey: "sk-test", auth: RuntimeAuth.bearer("oauth-token") })
OpenAI.chat("gpt-4.1-mini")
OpenAI.chat("gpt-4.1-mini", { apiKey: "sk-test" })
OpenAI.chat("gpt-4.1-mini", { apiKey: configApiKey })
OpenAI.chat("gpt-4.1-mini", { auth: RuntimeAuth.bearer("oauth-token") })
OpenAI.configure({ apiKey: "sk-test" }).chat("gpt-4.1-mini")
OpenAI.configure({ apiKey: configApiKey }).chat("gpt-4.1-mini")
OpenAI.configure({ auth: RuntimeAuth.bearer("oauth-token") }).chat("gpt-4.1-mini")
// @ts-expect-error OpenAI chat selectors only accept model ids.
OpenAI.configure({ apiKey: "sk-test" }).chat("gpt-4.1-mini", {})
// @ts-expect-error auth is an override, so OpenAI Chat rejects apiKey with auth.
OpenAI.chat("gpt-4.1-mini", { apiKey: "sk-test", auth: RuntimeAuth.bearer("oauth-token") })
OpenAI.configure({ apiKey: "sk-test", auth: RuntimeAuth.bearer("oauth-token") })
// @ts-expect-error Azure requires at least one of `resourceName` or `baseURL`.
Azure.responses("deployment")
Azure.responses("deployment", { apiKey: "azure-key", resourceName: "resource" })
Azure.responses("deployment", { apiKey: configApiKey, resourceName: "resource" })
Azure.responses("deployment", { auth: RuntimeAuth.header("api-key", "azure-key"), resourceName: "resource" })
Azure.configure()
Azure.configure({ apiKey: "azure-key", resourceName: "resource" }).responses("deployment")
Azure.configure({ apiKey: configApiKey, resourceName: "resource" }).responses("deployment")
Azure.configure({ auth: RuntimeAuth.header("api-key", "azure-key"), resourceName: "resource" }).responses("deployment")
// @ts-expect-error Azure model selectors only accept deployment ids.
Azure.configure({ apiKey: "azure-key", resourceName: "resource" }).responses("deployment", {})
// @ts-expect-error auth is an override, so Azure rejects apiKey with auth.
Azure.responses("deployment", { apiKey: "azure-key", auth: RuntimeAuth.header("api-key", "override") })
Azure.configure({ resourceName: "resource", apiKey: "azure-key", auth: RuntimeAuth.header("api-key", "override") })
// @ts-expect-error Azure requires at least one of `resourceName` or `baseURL`.
Azure.chat("deployment")
Azure.chat("deployment", { apiKey: "azure-key", resourceName: "resource" })
Azure.chat("deployment", { apiKey: configApiKey, resourceName: "resource" })
Azure.chat("deployment", { auth: RuntimeAuth.header("api-key", "azure-key"), resourceName: "resource" })
Azure.configure({ apiKey: "azure-key", resourceName: "resource" }).chat("deployment")
Azure.configure({ apiKey: configApiKey, resourceName: "resource" }).chat("deployment")
Azure.configure({ auth: RuntimeAuth.header("api-key", "azure-key"), resourceName: "resource" }).chat("deployment")
// @ts-expect-error Azure chat model selectors only accept deployment ids.
Azure.configure({ apiKey: "azure-key", resourceName: "resource" }).chat("deployment", {})
// @ts-expect-error auth is an override, so Azure Chat rejects apiKey with auth.
Azure.chat("deployment", { apiKey: "azure-key", auth: RuntimeAuth.header("api-key", "override") })
Azure.configure({ resourceName: "resource", apiKey: "azure-key", auth: RuntimeAuth.header("api-key", "override") })
Anthropic.configure({ apiKey: "anthropic-key" }).model("claude-haiku")
// @ts-expect-error Anthropic model selectors only accept model ids.
Anthropic.configure({ apiKey: "anthropic-key" }).model("claude-haiku", {})
Google.configure({ apiKey: "google-key" }).model("gemini-2.5-flash")
// @ts-expect-error Google model selectors only accept model ids.
Google.configure({ apiKey: "google-key" }).model("gemini-2.5-flash", {})
AmazonBedrock.configure({ apiKey: "bedrock-key" }).model("anthropic.claude")
// @ts-expect-error Bedrock model selectors only accept model ids.
AmazonBedrock.configure({ apiKey: "bedrock-key" }).model("anthropic.claude", {})
OpenRouter.configure({ apiKey: "openrouter-key" }).model("openai/gpt-4o-mini")
// @ts-expect-error OpenRouter model selectors only accept model ids.
OpenRouter.configure({ apiKey: "openrouter-key" }).model("openai/gpt-4o-mini", {})
XAI.configure({ apiKey: "xai-key" }).responses("grok-4")
XAI.configure({ apiKey: "xai-key" }).chat("grok-4")
// @ts-expect-error xAI Responses selectors only accept model ids.
XAI.configure({ apiKey: "xai-key" }).responses("grok-4", {})
// @ts-expect-error xAI Chat selectors only accept model ids.
XAI.configure({ apiKey: "xai-key" }).chat("grok-4", {})
OpenAICompatible.deepseek.configure({ apiKey: "deepseek-key" }).model("deepseek-chat")
// @ts-expect-error OpenAI-compatible family selectors only accept model ids.
OpenAICompatible.deepseek.configure({ apiKey: "deepseek-key" }).model("deepseek-chat", {})
Cloudflare.CloudflareWorkersAI.configure({ accountId: "account", apiKey: "cf-key" }).model("@cf/meta/llama")
// @ts-expect-error Cloudflare Workers AI model selectors only accept model ids.
Cloudflare.CloudflareWorkersAI.configure({ accountId: "account", apiKey: "cf-key" }).model("@cf/meta/llama", {})
GitHubCopilot.configure({ baseURL: "https://copilot.test", apiKey: "copilot-key" }).model("gpt-4.1")
// @ts-expect-error GitHub Copilot model selectors only accept model ids.
GitHubCopilot.configure({ baseURL: "https://copilot.test", apiKey: "copilot-key" }).model("gpt-4.1", {})

4
packages/llm/test/auth.test.ts
View file

@ -3,11 +3,13 @@ import { ConfigProvider, Effect } from "effect"
import { Headers } from "effect/unstable/http"
import { LLM } from "../src"
import { Auth } from "../src/route/auth"
import * as OpenAIChat from "../src/protocols/openai-chat"
import { Model } from "../src/schema"
import { it } from "./lib/effect"
const request = LLM.request({
id: "req_auth",
model: LLM.model({ id: "fake-model", provider: "fake", route: "fake", baseURL: "https://fake.local" }),
model: Model.make({ id: "fake-model", provider: "fake", route: OpenAIChat.route }),
prompt: "hello",
})

36
packages/llm/test/cache-policy.test.ts
View file

@ -1,36 +1,32 @@
import { describe, expect, test } from "bun:test"
import { Effect } from "effect"
import { CacheHint, LLM, Message } from "../src"
import { LLMClient } from "../src/route"
import { Auth, LLMClient } from "../src/route"
import { AmazonBedrock } from "../src/providers"
import * as AnthropicMessages from "../src/protocols/anthropic-messages"
import * as BedrockConverse from "../src/protocols/bedrock-converse"
import * as Gemini from "../src/protocols/gemini"
import * as OpenAIChat from "../src/protocols/openai-chat"
import { applyCachePolicy } from "../src/cache-policy"
import { it } from "./lib/effect"
const anthropicModel = AnthropicMessages.model({
id: "claude-sonnet-4-5",
baseURL: "https://api.anthropic.test/v1/",
headers: { "x-api-key": "test" },
})
const anthropicModel = AnthropicMessages.route
.with({ endpoint: { baseURL: "https://api.anthropic.test/v1/" }, auth: Auth.header("x-api-key", "test") })
.model({ id: "claude-sonnet-4-5" })
const bedrockModel = BedrockConverse.model({
id: "anthropic.claude-3-5-sonnet-20241022-v2:0",
const bedrockModel = AmazonBedrock.configure({
credentials: { region: "us-east-1", accessKeyId: "fixture", secretAccessKey: "fixture" },
})
}).model("anthropic.claude-3-5-sonnet-20241022-v2:0")
const openaiModel = OpenAIChat.model({
id: "gpt-4o-mini",
baseURL: "https://api.openai.test/v1/",
headers: { authorization: "Bearer test" },
})
const openaiModel = OpenAIChat.route
.with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("test") })
.model({ id: "gpt-4o-mini" })
const geminiModel = Gemini.model({
id: "gemini-2.5-flash",
baseURL: "https://generativelanguage.test/v1beta/",
headers: { "x-goog-api-key": "test" },
})
const geminiModel = Gemini.route
.with({
endpoint: { baseURL: "https://generativelanguage.test/v1beta/" },
auth: Auth.header("x-goog-api-key", "test"),
})
.model({ id: "gemini-2.5-flash" })
describe("applyCachePolicy", () => {
it.effect("undefined cache resolves to 'auto' (the recommended default)", () =>

32
packages/llm/test/endpoint.test.ts
View file

@ -1,37 +1,40 @@
import { describe, expect, test } from "bun:test"
import { LLM } from "../src"
import * as OpenAIChat from "../src/protocols/openai-chat"
import { Endpoint } from "../src/route"
import { Model } from "../src/schema"
const request = (input: { readonly baseURL: string; readonly queryParams?: Record<string, string> }) =>
const request = () =>
LLM.request({
model: LLM.model({
model: Model.make({
id: "model-1",
provider: "test",
route: "test-route",
baseURL: input.baseURL,
queryParams: input.queryParams,
route: OpenAIChat.route,
}),
prompt: "hello",
})
describe("Endpoint", () => {
test("appends a static path to the model's baseURL", () => {
const url = Endpoint.render(Endpoint.path("/chat"), {
request: request({ baseURL: "https://api.example.test/v1/" }),
const url = Endpoint.render(Endpoint.path("/chat", { baseURL: "https://api.example.test/v1/" }), {
request: request(),
body: {},
})
expect(url.toString()).toBe("https://api.example.test/v1/chat")
})
test("model query params are appended to the rendered URL", () => {
const url = Endpoint.render(Endpoint.path("/chat?alt=sse"), {
request: request({
test("endpoint query params are appended to the rendered URL", () => {
const url = Endpoint.render(
Endpoint.path("/chat?alt=sse", {
baseURL: "https://custom.example.test/root/",
queryParams: { "api-version": "2026-01-01", alt: "json" },
query: { "api-version": "2026-01-01", alt: "json" },
}),
body: {},
})
{
request: request(),
body: {},
},
)
expect(url.toString()).toBe("https://custom.example.test/root/chat?alt=json&api-version=2026-01-01")
})
@ -40,9 +43,10 @@ describe("Endpoint", () => {
const url = Endpoint.render(
Endpoint.path<{ readonly modelId: string }>(
({ body }) => `/model/${encodeURIComponent(body.modelId)}/converse-stream`,
{ baseURL: "https://bedrock-runtime.us-east-1.amazonaws.com" },
),
{
request: request({ baseURL: "https://bedrock-runtime.us-east-1.amazonaws.com" }),
request: request(),
body: { modelId: "us.amazon.nova-micro-v1:0" },
},
)

10
packages/llm/test/executor.test.ts
View file

@ -106,8 +106,8 @@ describe("RequestExecutor", () => {
expect(errorHttp(error)?.body).toBe("rate limited")
}).pipe(
Effect.provide(
responsesLayer([
...Array.from(
responsesLayer(
Array.from(
{ length: 3 },
() =>
new Response("rate limited", {
@ -115,7 +115,7 @@ describe("RequestExecutor", () => {
headers: { "retry-after-ms": "0", "x-request-id": "req_123", "x-api-key": "secret" },
}),
),
]),
),
),
),
)
@ -388,7 +388,9 @@ describe("RequestExecutor", () => {
it.effect("does not retry after a successful response reaches stream parsing", () =>
Effect.gen(function* () {
const attempts = yield* Ref.make(0)
const model = OpenAIChat.model({ id: "gpt-4o-mini", baseURL: "https://api.openai.test/v1" })
const model = OpenAIChat.route
.with({ endpoint: { baseURL: "https://api.openai.test/v1" } })
.model({ id: "gpt-4o-mini" })
const error = yield* LLMClient.generate(LLM.request({ model, prompt: "Say hello." })).pipe(
Effect.provide(
dynamicResponse((input) =>

38
packages/llm/test/exports.test.ts
View file

@ -2,7 +2,14 @@ import { describe, expect, test } from "bun:test"
import { LLM, LLMClient, Provider } from "@opencode-ai/llm"
import { Route, Protocol } from "@opencode-ai/llm/route"
import { Provider as ProviderSubpath } from "@opencode-ai/llm/provider"
import { Cloudflare, OpenAI, OpenAICompatible, OpenRouter, XAI } from "@opencode-ai/llm/providers"
import {
CloudflareAIGateway,
CloudflareWorkersAI,
OpenAI,
OpenAICompatible,
OpenRouter,
XAI,
} from "@opencode-ai/llm/providers"
import * as GitHubCopilot from "@opencode-ai/llm/providers/github-copilot"
import { OpenAIChat, OpenAICompatibleChat, OpenAIResponses } from "@opencode-ai/llm/protocols"
import * as AnthropicMessages from "@opencode-ai/llm/protocols/anthropic-messages"
@ -24,26 +31,25 @@ describe("public exports", () => {
test("provider barrels expose user-facing facades", () => {
expect(OpenAI.model).toBeFunction()
expect(OpenAI.provider.model).toBe(OpenAI.model)
expect(OpenAI.apis.responses).toBe(OpenAI.responses)
expect(OpenAI.apis.responsesWebSocket).toBe(OpenAI.responsesWebSocket)
expect(OpenAI.provider.responses).toBe(OpenAI.responses)
expect(OpenAI.provider.responsesWebSocket).toBe(OpenAI.responsesWebSocket)
expect(OpenAI.configure({ apiKey: "fixture" }).responses).toBeFunction()
expect(OpenAICompatible.deepseek.model).toBeFunction()
expect(Cloudflare.model).toBeFunction()
expect(Cloudflare.provider.model).toBe(Cloudflare.model)
expect(Cloudflare.aiGateway).toBeFunction()
expect(Cloudflare.workersAI).toBeFunction()
expect(CloudflareAIGateway.configure).toBeFunction()
expect(CloudflareAIGateway.configure({ accountId: "fixture", gatewayApiKey: "fixture" }).model).toBeFunction()
expect(CloudflareWorkersAI.configure).toBeFunction()
expect(CloudflareWorkersAI.configure({ accountId: "fixture", apiKey: "fixture" }).model).toBeFunction()
expect(OpenRouter.model).toBeFunction()
expect(OpenRouter.provider.model).toBe(OpenRouter.model)
expect(XAI.model).toBeFunction()
expect(XAI.provider.model).toBe(XAI.model)
expect(XAI.apis.responses).toBe(XAI.responses)
expect(XAI.apis.chat).toBe(XAI.chat)
expect(XAI.responses("grok-4.3", { apiKey: "fixture" })).toMatchObject({
route: "openai-responses",
})
expect(XAI.chat("grok-4.3", { apiKey: "fixture" })).toMatchObject({
route: "openai-compatible-chat",
})
expect(GitHubCopilot.model).toBeFunction()
expect(XAI.provider.responses).toBe(XAI.responses)
expect(XAI.provider.chat).toBe(XAI.chat)
expect(XAI.configure({ apiKey: "fixture" }).responses("grok-4.3").route.id).toBe("openai-responses")
expect(XAI.configure({ apiKey: "fixture" }).chat("grok-4.3").route.id).toBe("openai-compatible-chat")
expect(
GitHubCopilot.configure({ baseURL: "https://api.githubcopilot.test", apiKey: "fixture" }).model,
).toBeFunction()
})
test("protocol barrels expose supported low-level routes", () => {

BIN
packages/llm/test/fixtures/media/restroom.png vendored Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 14 KiB

32
packages/llm/test/fixtures/recordings/gemini/gemini-2-5-flash-image.json vendored Normal file
View file

File diff suppressed because one or more lines are too long

4
packages/llm/test/fixtures/recordings/openai-responses-cache/reports-cached-tokens-on-identical-second-call.json vendored
View file

File diff suppressed because one or more lines are too long

9
packages/llm/test/generate-object.test.ts
View file

@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test"
import { Effect, Schema } from "effect"
import { LLM } from "../src"
import * as OpenAIChat from "../src/protocols/openai-chat"
import { Auth } from "../src/route"
import { Tool, toDefinitions } from "../src/tool"
import { it } from "./lib/effect"
import { dynamicResponse } from "./lib/http"
@ -17,11 +18,9 @@ type OpenAIChatBody = {
}>
}
const model = OpenAIChat.model({
id: "gpt-4o-mini",
baseURL: "https://api.openai.test/v1/",
headers: { authorization: "Bearer test" },
})
const model = OpenAIChat.route
.with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("test") })
.model({ id: "gpt-4o-mini" })
const Json = Schema.fromJsonString(Schema.Unknown)
const decodeJson = Schema.decodeUnknownSync(Json)

10
packages/llm/test/lib/http.ts
View file

@ -1,8 +1,9 @@
import { Effect, Layer, Ref } from "effect"
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
import { LLMClient, RequestExecutor } from "../../src/route"
import { LLMClient, RequestExecutor, WebSocketExecutor } from "../../src/route"
import type { Service as LLMClientService } from "../../src/route/client"
import type { Service as RequestExecutorService } from "../../src/route/executor"
import type { Service as WebSocketExecutorService } from "../../src/route/transport/websocket"
export type HandlerInput = {
readonly request: HttpClientRequest.HttpClientRequest
@ -31,12 +32,13 @@ const handlerLayer = (handler: Handler): Layer.Layer<HttpClient.HttpClient> =>
),
)
export type RuntimeEnv = RequestExecutorService | LLMClientService
export type RuntimeEnv = RequestExecutorService | WebSocketExecutorService | LLMClientService
export const runtimeLayer = (layer: Layer.Layer<HttpClient.HttpClient>): Layer.Layer<RuntimeEnv> => {
const requestExecutorLayer = RequestExecutor.layer.pipe(Layer.provide(layer))
const llmClientLayer = LLMClient.layer.pipe(Layer.provide(requestExecutorLayer))
return Layer.mergeAll(requestExecutorLayer, llmClientLayer)
const deps = Layer.mergeAll(requestExecutorLayer, WebSocketExecutor.layer)
const llmClientLayer = LLMClient.layer.pipe(Layer.provide(deps))
return Layer.mergeAll(deps, llmClientLayer)
}
const SSE_HEADERS = { "content-type": "text/event-stream" } as const

51
packages/llm/test/llm.test.ts
View file

@ -1,18 +1,23 @@
import { describe, expect, test } from "bun:test"
import { LLM, LLMResponse } from "../src"
import { LLMRequest, Message, ModelRef, ToolCallPart, ToolChoice, ToolDefinition, ToolResultPart } from "../src/schema"
import * as OpenAIChat from "../src/protocols/openai-chat"
import * as OpenAIResponses from "../src/protocols/openai-responses"
import { LLMRequest, Message, Model, ToolCallPart, ToolChoice, ToolDefinition, ToolResultPart } from "../src/schema"
const chatRoute = OpenAIChat.route
const responsesRoute = OpenAIResponses.route
describe("llm constructors", () => {
test("builds canonical schema classes from ergonomic input", () => {
const request = LLM.request({
id: "req_1",
model: LLM.model({ id: "fake-model", provider: "fake", route: "openai-chat", baseURL: "https://fake.local" }),
model: Model.make({ id: "fake-model", provider: "fake", route: chatRoute }),
system: "You are concise.",
prompt: "Say hello.",
})
expect(request).toBeInstanceOf(LLMRequest)
expect(request.model).toBeInstanceOf(ModelRef)
expect(request.model).toBeInstanceOf(Model)
expect(request.messages[0]).toBeInstanceOf(Message)
expect(request.system).toEqual([{ type: "text", text: "You are concise." }])
expect(request.messages[0]?.content).toEqual([{ type: "text", text: "Say hello." }])
@ -23,7 +28,7 @@ describe("llm constructors", () => {
test("updates requests without spreading schema class instances", () => {
const base = LLM.request({
id: "req_1",
model: LLM.model({ id: "fake-model", provider: "fake", route: "openai-chat", baseURL: "https://fake.local" }),
model: Model.make({ id: "fake-model", provider: "fake", route: chatRoute }),
prompt: "Say hello.",
})
const updated = LLM.updateRequest(base, {
@ -38,16 +43,16 @@ describe("llm constructors", () => {
expect(updated.messages.map((message) => message.role)).toEqual(["user", "assistant"])
})
test("keeps request options separate from model defaults", () => {
test("keeps request options separate from route defaults", () => {
const request = LLM.request({
model: LLM.model({
model: Model.make({
id: "fake-model",
provider: "fake",
route: "openai-chat",
baseURL: "https://fake.local",
generation: { maxTokens: 100, temperature: 1 },
providerOptions: { openai: { store: false, metadata: { model: true } } },
http: { body: { metadata: { model: true } }, headers: { "x-shared": "model" }, query: { model: "1" } },
route: chatRoute.with({
generation: { maxTokens: 100, temperature: 1 },
providerOptions: { openai: { store: false, metadata: { model: true } } },
http: { body: { metadata: { model: true } }, headers: { "x-shared": "model" }, query: { model: "1" } },
}),
}),
prompt: "Say hello.",
generation: { temperature: 0 },
@ -67,7 +72,7 @@ describe("llm constructors", () => {
test("updates canonical requests from the request datatype", () => {
const base = LLM.request({
id: "req_1",
model: LLM.model({ id: "fake-model", provider: "fake", route: "openai-chat", baseURL: "https://fake.local" }),
model: Model.make({ id: "fake-model", provider: "fake", route: chatRoute }),
prompt: "Say hello.",
})
const updated = LLMRequest.update(base, { messages: [...base.messages, Message.assistant("Hi.")] })
@ -80,14 +85,18 @@ describe("llm constructors", () => {
})
test("updates canonical models from the model datatype", () => {
const base = LLM.model({ id: "fake-model", provider: "fake", route: "openai-chat", baseURL: "https://fake.local" })
const updated = ModelRef.update(base, { route: "openai-responses" })
const base = Model.make({
id: "fake-model",
provider: "fake",
route: chatRoute,
})
const updated = Model.update(base, { route: responsesRoute })
expect(updated).toBeInstanceOf(ModelRef)
expect(updated).toBeInstanceOf(Model)
expect(String(updated.id)).toBe("fake-model")
expect(updated.route).toBe("openai-responses")
expect(String(ModelRef.input(updated).provider)).toBe("fake")
expect(ModelRef.update(updated, {})).toBe(updated)
expect(updated.route).toBe(responsesRoute)
expect(String(Model.input(updated).provider)).toBe("fake")
expect(Model.update(updated, {})).toBe(updated)
})
test("builds tool choices from names and tools", () => {
@ -105,7 +114,11 @@ describe("llm constructors", () => {
expect(ToolChoice.make("required")).toEqual(new ToolChoice({ type: "required" }))
expect(
LLM.request({
model: LLM.model({ id: "fake-model", provider: "fake", route: "openai-chat", baseURL: "https://fake.local" }),
model: Model.make({
id: "fake-model",
provider: "fake",
route: chatRoute,
}),
prompt: "Use tools if needed.",
toolChoice: "required",
}).toolChoice,

10
packages/llm/test/provider.types.ts
View file

@ -1,9 +1,9 @@
import { Provider } from "../src/provider"
import { ProviderID, type ModelRef } from "../src/schema"
import { ProviderID, type Model } from "../src/schema"
declare const model: (id: string) => ModelRef
declare const requiredModel: (id: string, options: { readonly baseURL: string }) => ModelRef
declare const chat: (id: string, options: { readonly apiKey: string }) => ModelRef
declare const model: (id: string) => Model
declare const requiredModel: (id: string, options: { readonly baseURL: string }) => Model
declare const chat: (id: string, options: { readonly apiKey: string }) => Model
Provider.make({
id: ProviderID.make("example"),
@ -22,6 +22,8 @@ const requiredProvider = Provider.make({
model: requiredModel,
})
// Provider.make is advanced structural typing coverage; built-in providers use
// configure(...).model(id) facades instead of second-argument selectors.
requiredProvider.model("custom", { baseURL: "https://example.com/v1" })
// @ts-expect-error Provider.make preserves required model options.

7
packages/llm/test/provider/anthropic-messages-cache.recorded.test.ts
View file

@ -3,14 +3,13 @@ import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { CacheHint, LLM } from "../../src"
import { LLMClient } from "../../src/route"
import * as AnthropicMessages from "../../src/protocols/anthropic-messages"
import * as Anthropic from "../../src/providers/anthropic"
import { LARGE_CACHEABLE_SYSTEM } from "../recorded-scenarios"
import { recordedTests } from "../recorded-test"
const model = AnthropicMessages.model({
id: "claude-haiku-4-5-20251001",
const model = Anthropic.configure({
apiKey: process.env.ANTHROPIC_API_KEY ?? "fixture",
})
}).model("claude-haiku-4-5-20251001")
// Two identical generations in a row. The first call writes the prefix into
// Anthropic's cache; the second should report a cache read against the same

7
packages/llm/test/provider/anthropic-messages.recorded.test.ts
View file

@ -3,14 +3,13 @@ import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { LLM, LLMError, Message, ToolCallPart } from "../../src"
import { LLMClient } from "../../src/route"
import * as AnthropicMessages from "../../src/protocols/anthropic-messages"
import * as Anthropic from "../../src/providers/anthropic"
import { weatherToolName } from "../recorded-scenarios"
import { recordedTests } from "../recorded-test"
const model = AnthropicMessages.model({
id: "claude-haiku-4-5-20251001",
const model = Anthropic.configure({
apiKey: process.env.ANTHROPIC_API_KEY ?? "fixture",
})
}).model("claude-haiku-4-5-20251001")
const malformedToolOrderRequest = LLM.request({
id: "recorded_anthropic_malformed_tool_order",

10
packages/llm/test/provider/anthropic-messages.test.ts
View file

@ -1,17 +1,15 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { CacheHint, LLM, LLMError, Message, ToolCallPart, Usage } from "../../src"
import { LLMClient } from "../../src/route"
import { Auth, LLMClient } from "../../src/route"
import * as AnthropicMessages from "../../src/protocols/anthropic-messages"
import { it } from "../lib/effect"
import { fixedResponse } from "../lib/http"
import { sseEvents } from "../lib/sse"
const model = AnthropicMessages.model({
id: "claude-sonnet-4-5",
baseURL: "https://api.anthropic.test/v1/",
headers: { "x-api-key": "test" },
})
const model = AnthropicMessages.route
.with({ endpoint: { baseURL: "https://api.anthropic.test/v1/" }, auth: Auth.header("x-api-key", "test") })
.model({ id: "claude-sonnet-4-5" })
const request = LLM.request({
id: "req_1",

7
packages/llm/test/provider/bedrock-converse-cache.recorded.test.ts
View file

@ -2,7 +2,7 @@ import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { CacheHint, LLM } from "../../src"
import { LLMClient } from "../../src/route"
import * as BedrockConverse from "../../src/protocols/bedrock-converse"
import { AmazonBedrock } from "../../src/providers"
import { LARGE_CACHEABLE_SYSTEM } from "../recorded-scenarios"
import { recordedTests } from "../recorded-test"
@ -12,15 +12,14 @@ const RECORDING_REGION = process.env.BEDROCK_RECORDING_REGION ?? "us-east-1"
// doesn't reliably surface `cacheRead`/`cacheWrite` in usage, so the second
// call wouldn't deterministically prove cache mapping works. Override with
// BEDROCK_CACHE_MODEL_ID if your account has access elsewhere.
const model = BedrockConverse.model({
id: process.env.BEDROCK_CACHE_MODEL_ID ?? "us.anthropic.claude-haiku-4-5-20251001-v1:0",
const model = AmazonBedrock.configure({
credentials: {
region: RECORDING_REGION,
accessKeyId: process.env.AWS_ACCESS_KEY_ID ?? "fixture",
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY ?? "fixture",
sessionToken: process.env.AWS_SESSION_TOKEN,
},
})
}).model(process.env.BEDROCK_CACHE_MODEL_ID ?? "us.anthropic.claude-haiku-4-5-20251001-v1:0")
const cacheRequest = LLM.request({
id: "recorded_bedrock_cache",

80
packages/llm/test/provider/bedrock-converse.test.ts
View file

@ -4,6 +4,7 @@ import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { CacheHint, LLM, Message, ToolCallPart, ToolChoice } from "../../src"
import { LLMClient } from "../../src/route"
import { AmazonBedrock } from "../../src/providers"
import * as BedrockConverse from "../../src/protocols/bedrock-converse"
import { it } from "../lib/effect"
import { fixedResponse } from "../lib/http"
@ -52,11 +53,10 @@ const eventStreamBody = (...payloads: ReadonlyArray<readonly [string, object]>)
const fixedBytes = (bytes: Uint8Array) =>
fixedResponse(bytes.slice().buffer, { headers: { "content-type": "application/vnd.amazon.eventstream" } })
const model = BedrockConverse.model({
id: "anthropic.claude-3-5-sonnet-20240620-v1:0",
const model = AmazonBedrock.configure({
baseURL: "https://bedrock-runtime.test",
apiKey: "test-bearer",
})
}).model("anthropic.claude-3-5-sonnet-20240620-v1:0")
const baseRequest = LLM.request({
id: "req_1",
@ -156,6 +156,55 @@ describe("Bedrock Converse route", () => {
}),
)
it.effect("lowers image content in tool-result messages", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(
LLM.request({
id: "req_tool_image",
model,
messages: [
Message.user("Capture the screen."),
Message.assistant([ToolCallPart.make({ id: "tool_1", name: "screenshot", input: {} })]),
Message.tool({
id: "tool_1",
name: "screenshot",
result: {
type: "content",
value: [
{ type: "text", text: "Screenshot captured." },
{ type: "media", mediaType: "image/png", data: "AAAA" },
],
},
}),
],
cache: "none",
}),
)
expect(prepared.body).toMatchObject({
messages: [
{ role: "user", content: [{ text: "Capture the screen." }] },
{
role: "assistant",
content: [{ toolUse: { toolUseId: "tool_1", name: "screenshot", input: {} } }],
},
{
role: "user",
content: [
{
toolResult: {
toolUseId: "tool_1",
content: [{ text: "Screenshot captured." }, { image: { format: "png", source: { bytes: "AAAA" } } }],
status: "success",
},
},
],
},
],
})
}),
)
it.effect("decodes text-delta + messageStop + metadata usage from binary event stream", () =>
Effect.gen(function* () {
const body = eventStreamBody(
@ -249,39 +298,32 @@ describe("Bedrock Converse route", () => {
it.effect("rejects requests with no auth path", () =>
Effect.gen(function* () {
const unsignedModel = BedrockConverse.model({
id: "anthropic.claude-3-5-sonnet-20240620-v1:0",
const unsignedModel = AmazonBedrock.configure({
baseURL: "https://bedrock-runtime.test",
})
}).model("anthropic.claude-3-5-sonnet-20240620-v1:0")
const error = yield* LLMClient.generate(LLM.updateRequest(baseRequest, { model: unsignedModel })).pipe(
Effect.provide(fixedBytes(eventStreamBody(["messageStop", { stopReason: "end_turn" }]))),
Effect.flip,
)
expect(error.message).toContain("Bedrock Converse requires either model.apiKey")
expect(error.message).toContain("Bedrock Converse requires either route bearer auth or AWS credentials")
}),
)
it.effect("signs requests with SigV4 when AWS credentials are provided (deterministic plumbing check)", () =>
Effect.gen(function* () {
const signed = BedrockConverse.model({
id: "anthropic.claude-3-5-sonnet-20240620-v1:0",
const signed = AmazonBedrock.configure({
baseURL: "https://bedrock-runtime.test",
credentials: {
region: "us-east-1",
accessKeyId: "AKIAIOSFODNN7EXAMPLE",
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
},
})
}).model("anthropic.claude-3-5-sonnet-20240620-v1:0")
const prepared = yield* LLMClient.prepare(LLM.updateRequest(baseRequest, { model: signed }))
expect(prepared.route).toBe("bedrock-converse")
// The prepare phase doesn't sign — toHttp does. We assert the credential
// is plumbed onto the model native field for the signer to find.
expect(prepared.model.native).toMatchObject({
aws_credentials: { region: "us-east-1", accessKeyId: "AKIAIOSFODNN7EXAMPLE" },
aws_region: "us-east-1",
})
expect(prepared.model).toBe(signed)
}),
)
@ -531,18 +573,17 @@ describe("Bedrock Converse route", () => {
const RECORDING_REGION = process.env.BEDROCK_RECORDING_REGION ?? "us-east-1"
const recordedModel = () =>
BedrockConverse.model({
AmazonBedrock.configure({
// Most newer Anthropic models on Bedrock require a cross-region inference
// profile (`us.` prefix). Nova does not require an Anthropic use-case form
// and is on-demand-throughput accessible by default for most accounts.
id: process.env.BEDROCK_MODEL_ID ?? "us.amazon.nova-micro-v1:0",
credentials: {
region: RECORDING_REGION,
accessKeyId: process.env.AWS_ACCESS_KEY_ID ?? "fixture",
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY ?? "fixture",
sessionToken: process.env.AWS_SESSION_TOKEN,
},
})
}).model(process.env.BEDROCK_MODEL_ID ?? "us.amazon.nova-micro-v1:0")
const recorded = recordedTests({
prefix: "bedrock-converse",
@ -598,7 +639,6 @@ describe("Bedrock Converse recorded", () => {
recorded.effect.with("drives a tool loop", { tags: ["tool", "tool-loop", "golden"] }, () =>
Effect.gen(function* () {
const llm = yield* LLMClient.Service
expectWeatherToolLoop(
yield* runWeatherToolLoop(
weatherToolLoopRequest({

44
packages/llm/test/provider/cloudflare.test.ts
View file

@ -2,7 +2,7 @@ import { describe, expect } from "bun:test"
import { ConfigProvider, Effect, Schema } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { LLM } from "../../src"
import * as Cloudflare from "../../src/providers/cloudflare"
import { CloudflareAIGateway, CloudflareWorkersAI } from "../../src/providers/cloudflare"
import { LLMClient } from "../../src/route"
import { it } from "../lib/effect"
import { dynamicResponse } from "../lib/http"
@ -21,18 +21,18 @@ const deltaChunk = (delta: object, finishReason: string | null = null) => ({
describe("Cloudflare", () => {
it.effect("prepares AI Gateway models through the OpenAI-compatible Chat protocol", () =>
Effect.gen(function* () {
const model = Cloudflare.aiGateway("workers-ai/@cf/meta/llama-3.3-70b-instruct", {
const model = CloudflareAIGateway.configure({
accountId: "test-account",
gatewayId: "test-gateway",
apiKey: "test-token",
})
}).model("workers-ai/@cf/meta/llama-3.3-70b-instruct")
expect(model).toMatchObject({
id: "workers-ai/@cf/meta/llama-3.3-70b-instruct",
provider: "cloudflare-ai-gateway",
route: "cloudflare-ai-gateway",
baseURL: "https://gateway.ai.cloudflare.com/v1/test-account/test-gateway/compat",
route: { id: "cloudflare-ai-gateway" },
})
expect(model.route.endpoint.baseURL).toBe("https://gateway.ai.cloudflare.com/v1/test-account/test-gateway/compat")
const prepared = yield* LLMClient.prepare(LLM.request({ model, prompt: "Say hello." }))
@ -49,11 +49,11 @@ describe("Cloudflare", () => {
Effect.gen(function* () {
const response = yield* LLM.generate(
LLM.request({
model: Cloudflare.aiGateway("openai/gpt-4o-mini", {
model: CloudflareAIGateway.configure({
accountId: "test-account",
gatewayId: "test-gateway",
apiKey: "test-token",
}),
}).model("openai/gpt-4o-mini"),
prompt: "Say hello.",
}),
).pipe(
@ -86,11 +86,11 @@ describe("Cloudflare", () => {
it.effect("defaults AI Gateway id to default when omitted or blank", () =>
Effect.gen(function* () {
expect(
Cloudflare.aiGateway("workers-ai/@cf/meta/llama-3.3-70b-instruct", {
CloudflareAIGateway.configure({
accountId: "test-account",
gatewayId: "",
gatewayApiKey: "test-token",
}).baseURL,
}).model("workers-ai/@cf/meta/llama-3.3-70b-instruct").route.endpoint.baseURL,
).toBe("https://gateway.ai.cloudflare.com/v1/test-account/default/compat")
}),
)
@ -99,11 +99,11 @@ describe("Cloudflare", () => {
Effect.gen(function* () {
yield* LLM.generate(
LLM.request({
model: Cloudflare.aiGateway("openai/gpt-4o-mini", {
model: CloudflareAIGateway.configure({
accountId: "test-account",
gatewayApiKey: "gateway-token",
apiKey: "provider-token",
}),
}).model("openai/gpt-4o-mini"),
prompt: "Say hello.",
}),
).pipe(
@ -129,31 +129,31 @@ describe("Cloudflare", () => {
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(
LLM.request({
model: Cloudflare.aiGateway("openai/gpt-4o-mini", {
model: CloudflareAIGateway.configure({
baseURL: "https://gateway.proxy.test/v1/custom/compat",
apiKey: "test-token",
}),
}).model("openai/gpt-4o-mini"),
prompt: "Say hello.",
}),
)
expect(prepared.model.baseURL).toBe("https://gateway.proxy.test/v1/custom/compat")
expect(prepared.model.route.endpoint.baseURL).toBe("https://gateway.proxy.test/v1/custom/compat")
}),
)
it.effect("prepares direct Workers AI models through the OpenAI-compatible Chat protocol", () =>
Effect.gen(function* () {
const model = Cloudflare.workersAI("@cf/meta/llama-3.1-8b-instruct", {
const model = CloudflareWorkersAI.configure({
accountId: "test-account",
apiKey: "test-token",
})
}).model("@cf/meta/llama-3.1-8b-instruct")
expect(model).toMatchObject({
id: "@cf/meta/llama-3.1-8b-instruct",
provider: "cloudflare-workers-ai",
route: "cloudflare-workers-ai",
baseURL: "https://api.cloudflare.com/client/v4/accounts/test-account/ai/v1",
route: { id: "cloudflare-workers-ai" },
})
expect(model.route.endpoint.baseURL).toBe("https://api.cloudflare.com/client/v4/accounts/test-account/ai/v1")
const prepared = yield* LLMClient.prepare(LLM.request({ model, prompt: "Say hello." }))
@ -170,10 +170,10 @@ describe("Cloudflare", () => {
Effect.gen(function* () {
const response = yield* LLM.generate(
LLM.request({
model: Cloudflare.workersAI("@cf/meta/llama-3.1-8b-instruct", {
model: CloudflareWorkersAI.configure({
accountId: "test-account",
apiKey: "test-token",
}),
}).model("@cf/meta/llama-3.1-8b-instruct"),
prompt: "Say hello.",
}),
).pipe(
@ -205,9 +205,9 @@ describe("Cloudflare", () => {
Effect.gen(function* () {
yield* LLM.generate(
LLM.request({
model: Cloudflare.workersAI("@cf/meta/llama-3.1-8b-instruct", {
model: CloudflareWorkersAI.configure({
accountId: "test-account",
}),
}).model("@cf/meta/llama-3.1-8b-instruct"),
prompt: "Say hello.",
}),
).pipe(

7
packages/llm/test/provider/gemini-cache.recorded.test.ts
View file

@ -2,14 +2,13 @@ import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { LLM } from "../../src"
import { LLMClient } from "../../src/route"
import * as Gemini from "../../src/protocols/gemini"
import * as Google from "../../src/providers/google"
import { LARGE_CACHEABLE_SYSTEM } from "../recorded-scenarios"
import { recordedTests } from "../recorded-test"
const model = Gemini.model({
id: "gemini-2.5-flash",
const model = Google.configure({
apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY ?? process.env.GEMINI_API_KEY ?? "fixture",
})
}).model("gemini-2.5-flash")
// Gemini does implicit prefix caching on 2.5+ models above ~1024 tokens. The
// `CacheHint` is currently a no-op for Gemini (the explicit `CachedContent`

13
packages/llm/test/provider/gemini.test.ts
View file

@ -1,17 +1,18 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { LLM, LLMError, Message, ToolCallPart, Usage } from "../../src"
import { LLMClient } from "../../src/route"
import { Auth, LLMClient } from "../../src/route"
import * as Gemini from "../../src/protocols/gemini"
import { it } from "../lib/effect"
import { fixedResponse } from "../lib/http"
import { sseEvents, sseRaw } from "../lib/sse"
const model = Gemini.model({
id: "gemini-2.5-flash",
baseURL: "https://generativelanguage.test/v1beta/",
headers: { "x-goog-api-key": "test" },
})
const model = Gemini.route
.with({
endpoint: { baseURL: "https://generativelanguage.test/v1beta/" },
auth: Auth.header("x-goog-api-key", "test"),
})
.model({ id: "gemini-2.5-flash" })
const request = LLM.request({
id: "req_1",

79
packages/llm/test/provider/golden.recorded.test.ts
View file

@ -1,32 +1,30 @@
import { Redactor } from "@opencode-ai/http-recorder"
import * as AnthropicMessages from "../../src/protocols/anthropic-messages"
import * as Gemini from "../../src/protocols/gemini"
import * as OpenAIChat from "../../src/protocols/openai-chat"
import * as OpenAIResponses from "../../src/protocols/openai-responses"
import * as Cloudflare from "../../src/providers/cloudflare"
import * as Anthropic from "../../src/providers/anthropic"
import { CloudflareAIGateway, CloudflareWorkersAI } from "../../src/providers/cloudflare"
import * as Google from "../../src/providers/google"
import * as OpenAI from "../../src/providers/openai"
import * as OpenAICompatible from "../../src/providers/openai-compatible"
import * as OpenRouter from "../../src/providers/openrouter"
import * as XAI from "../../src/providers/xai"
import { describeRecordedGoldenScenarios } from "../recorded-golden"
const openAIChat = OpenAIChat.model({ id: "gpt-4o-mini", apiKey: process.env.OPENAI_API_KEY ?? "fixture" })
const openAIResponses = OpenAIResponses.model({ id: "gpt-5.5", apiKey: process.env.OPENAI_API_KEY ?? "fixture" })
const openAIResponsesWebSocket = OpenAI.responsesWebSocket("gpt-4.1-mini", {
const openAI = OpenAI.configure({
apiKey: process.env.OPENAI_API_KEY ?? "fixture",
})
const anthropicHaiku = AnthropicMessages.model({
id: "claude-haiku-4-5-20251001",
const openAIChat = openAI.chat("gpt-4o-mini")
const openAIResponses = openAI.responses("gpt-5.5")
const openAIResponsesWebSocket = openAI.responsesWebSocket("gpt-4.1-mini")
const anthropic = Anthropic.configure({
apiKey: process.env.ANTHROPIC_API_KEY ?? "fixture",
})
const anthropicOpus = AnthropicMessages.model({
id: "claude-opus-4-7",
apiKey: process.env.ANTHROPIC_API_KEY ?? "fixture",
})
const gemini = Gemini.model({ id: "gemini-2.5-flash", apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY ?? "fixture" })
const xaiBasic = XAI.model("grok-3-mini", { apiKey: process.env.XAI_API_KEY ?? "fixture" })
const xaiFlagship = XAI.model("grok-4.3", { apiKey: process.env.XAI_API_KEY ?? "fixture" })
const cloudflareAIGatewayWorkers = Cloudflare.aiGateway("workers-ai/@cf/meta/llama-3.1-8b-instruct", {
const anthropicHaiku = anthropic.model("claude-haiku-4-5-20251001")
const anthropicOpus = anthropic.model("claude-opus-4-7")
const google = Google.configure({ apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY ?? "fixture" })
const gemini = google.model("gemini-2.5-flash")
const xai = XAI.configure({ apiKey: process.env.XAI_API_KEY ?? "fixture" })
const xaiBasic = xai.model("grok-3-mini")
const xaiFlagship = xai.model("grok-4.3")
const cloudflareAIGateway = CloudflareAIGateway.configure({
accountId: process.env.CLOUDFLARE_ACCOUNT_ID ?? "fixture-account",
gatewayId:
process.env.CLOUDFLARE_GATEWAY_ID && process.env.CLOUDFLARE_GATEWAY_ID !== process.env.CLOUDFLARE_ACCOUNT_ID
@ -34,32 +32,31 @@ const cloudflareAIGatewayWorkers = Cloudflare.aiGateway("workers-ai/@cf/meta/lla
: undefined,
gatewayApiKey: process.env.CLOUDFLARE_API_TOKEN ?? "fixture",
})
const cloudflareAIGatewayWorkersTools = Cloudflare.aiGateway("workers-ai/@cf/openai/gpt-oss-20b", {
accountId: process.env.CLOUDFLARE_ACCOUNT_ID ?? "fixture-account",
gatewayId:
process.env.CLOUDFLARE_GATEWAY_ID && process.env.CLOUDFLARE_GATEWAY_ID !== process.env.CLOUDFLARE_ACCOUNT_ID
? process.env.CLOUDFLARE_GATEWAY_ID
: undefined,
gatewayApiKey: process.env.CLOUDFLARE_API_TOKEN ?? "fixture",
})
const cloudflareWorkersAI = Cloudflare.workersAI("@cf/meta/llama-3.1-8b-instruct", {
const cloudflareWorkers = CloudflareWorkersAI.configure({
accountId: process.env.CLOUDFLARE_ACCOUNT_ID ?? "fixture-account",
apiKey: process.env.CLOUDFLARE_API_KEY ?? "fixture",
})
const cloudflareWorkersAITools = Cloudflare.workersAI("@cf/openai/gpt-oss-20b", {
accountId: process.env.CLOUDFLARE_ACCOUNT_ID ?? "fixture-account",
apiKey: process.env.CLOUDFLARE_API_KEY ?? "fixture",
})
const deepseek = OpenAICompatible.deepseek.model("deepseek-chat", { apiKey: process.env.DEEPSEEK_API_KEY ?? "fixture" })
const together = OpenAICompatible.togetherai.model("meta-llama/Llama-3.3-70B-Instruct-Turbo", {
apiKey: process.env.TOGETHER_AI_API_KEY ?? "fixture",
})
const groq = OpenAICompatible.groq.model("llama-3.3-70b-versatile", { apiKey: process.env.GROQ_API_KEY ?? "fixture" })
const openrouter = OpenRouter.model("openai/gpt-4o-mini", { apiKey: process.env.OPENROUTER_API_KEY ?? "fixture" })
const openrouterGpt55 = OpenRouter.model("openai/gpt-5.5", { apiKey: process.env.OPENROUTER_API_KEY ?? "fixture" })
const openrouterOpus = OpenRouter.model("anthropic/claude-opus-4.7", {
const cloudflareAIGatewayWorkers = cloudflareAIGateway.model("workers-ai/@cf/meta/llama-3.1-8b-instruct")
const cloudflareAIGatewayWorkersTools = cloudflareAIGateway.model("workers-ai/@cf/openai/gpt-oss-20b")
const cloudflareWorkersAI = cloudflareWorkers.model("@cf/meta/llama-3.1-8b-instruct")
const cloudflareWorkersAITools = cloudflareWorkers.model("@cf/openai/gpt-oss-20b")
const deepseek = OpenAICompatible.deepseek
.configure({ apiKey: process.env.DEEPSEEK_API_KEY ?? "fixture" })
.model("deepseek-chat")
const together = OpenAICompatible.togetherai
.configure({
apiKey: process.env.TOGETHER_AI_API_KEY ?? "fixture",
})
.model("meta-llama/Llama-3.3-70B-Instruct-Turbo")
const groq = OpenAICompatible.groq
.configure({ apiKey: process.env.GROQ_API_KEY ?? "fixture" })
.model("llama-3.3-70b-versatile")
const openRouter = OpenRouter.configure({ apiKey: process.env.OPENROUTER_API_KEY ?? "fixture" })
const openrouter = openRouter.model("openai/gpt-4o-mini")
const openrouterGpt55 = openRouter.model("openai/gpt-5.5")
const openrouterOpus = OpenRouter.configure({
apiKey: process.env.OPENROUTER_API_KEY ?? "fixture",
})
}).model("anthropic/claude-opus-4.7")
const redactCloudflareURL = (url: string) =>
url
@ -120,7 +117,7 @@ describeRecordedGoldenScenarios([
prefix: "gemini",
model: gemini,
requires: ["GOOGLE_GENERATIVE_AI_API_KEY"],
scenarios: [{ id: "text", maxTokens: 80 }, "tool-call"],
scenarios: [{ id: "text", maxTokens: 80 }, "tool-call", { id: "image", maxTokens: 160 }],
},
{
name: "xAI Grok 3 Mini",

27
packages/llm/test/provider/openai-chat.test.ts
View file

@ -1,11 +1,11 @@
import { describe, expect } from "bun:test"
import { Effect, Schema, Stream } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { LLM, LLMError, Message, ToolCallPart, Usage } from "../../src"
import { LLM, LLMError, Message, Model, ToolCallPart, Usage } from "../../src"
import * as Azure from "../../src/providers/azure"
import * as OpenAI from "../../src/providers/openai"
import * as OpenAIChat from "../../src/protocols/openai-chat"
import { LLMClient } from "../../src/route"
import { Auth, LLMClient } from "../../src/route"
import { it } from "../lib/effect"
import { dynamicResponse, fixedResponse, truncatedStream } from "../lib/http"
import { deltaChunk, usageChunk } from "../lib/openai-chunks"
@ -15,11 +15,9 @@ const TargetJson = Schema.fromJsonString(Schema.Unknown)
const encodeJson = Schema.encodeSync(TargetJson)
const decodeJson = Schema.decodeUnknownSync(TargetJson)
const model = OpenAIChat.model({
id: "gpt-4o-mini",
baseURL: "https://api.openai.test/v1/",
headers: { authorization: "Bearer test" },
})
const model = OpenAIChat.route
.with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("test") })
.model({ id: "gpt-4o-mini" })
const request = LLM.request({
id: "req_1",
@ -56,7 +54,7 @@ describe("OpenAI Chat route", () => {
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIChat.OpenAIChatBody>(
LLM.request({
model: OpenAI.chat("gpt-4o-mini", { baseURL: "https://api.openai.test/v1/" }),
model: OpenAI.configure({ baseURL: "https://api.openai.test/v1/", apiKey: "test" }).chat("gpt-4o-mini"),
prompt: "think",
providerOptions: { openai: { reasoningEffort: "low" } },
}),
@ -69,7 +67,9 @@ describe("OpenAI Chat route", () => {
it.effect("adds native query params to the Chat Completions URL", () =>
LLMClient.generate(
LLM.updateRequest(request, { model: OpenAIChat.model({ ...model, queryParams: { "api-version": "v1" } }) }),
LLM.updateRequest(request, {
model: Model.update(model, { route: model.route.with({ endpoint: { query: { "api-version": "v1" } } }) }),
}),
).pipe(
Effect.provide(
dynamicResponse((input) =>
@ -88,17 +88,18 @@ describe("OpenAI Chat route", () => {
it.effect("uses Azure api-key header for static OpenAI Chat keys", () =>
LLMClient.generate(
LLM.updateRequest(request, {
model: Azure.chat("gpt-4o-mini", {
model: Azure.configure({
baseURL: "https://opencode-test.openai.azure.com/openai/v1/",
apiKey: "azure-key",
headers: { authorization: "Bearer stale" },
}),
}).chat("gpt-4o-mini"),
}),
).pipe(
Effect.provide(
dynamicResponse((input) =>
Effect.gen(function* () {
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
expect(web.url).toBe("https://opencode-test.openai.azure.com/openai/v1/chat/completions?api-version=v1")
expect(web.headers.get("api-key")).toBe("azure-key")
expect(web.headers.get("authorization")).toBeNull()
return input.respond(sseEvents(deltaChunk({}, "stop")), {
@ -113,7 +114,9 @@ describe("OpenAI Chat route", () => {
it.effect("applies serializable HTTP overlays after payload lowering", () =>
LLMClient.generate(
LLM.updateRequest(request, {
model: OpenAIChat.model({ ...model, apiKey: "fresh-key", headers: { authorization: "Bearer stale" } }),
model: model.route
.with({ auth: Auth.bearer("fresh-key"), headers: { authorization: "Bearer stale" } })
.model({ id: model.id }),
http: {
body: { metadata: { source: "test" } },
headers: { authorization: "Bearer request", "x-custom": "yes" },

45
packages/llm/test/provider/openai-compatible-chat.test.ts
View file

@ -2,7 +2,7 @@ import { describe, expect } from "bun:test"
import { Effect, Schema } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { LLM, Message, ToolCallPart } from "../../src"
import { LLMClient } from "../../src/route"
import { Auth, LLMClient } from "../../src/route"
import * as OpenAICompatible from "../../src/providers/openai-compatible"
import * as OpenAICompatibleChat from "../../src/protocols/openai-compatible-chat"
import { it } from "../lib/effect"
@ -12,13 +12,13 @@ import { sseEvents } from "../lib/sse"
const Json = Schema.fromJsonString(Schema.Unknown)
const decodeJson = Schema.decodeUnknownSync(Json)
const model = OpenAICompatibleChat.model({
id: "deepseek-chat",
provider: "deepseek",
baseURL: "https://api.deepseek.test/v1/",
apiKey: "test-key",
queryParams: { "api-version": "2026-01-01" },
})
const model = OpenAICompatibleChat.route
.with({
provider: "deepseek",
endpoint: { baseURL: "https://api.deepseek.test/v1/", query: { "api-version": "2026-01-01" } },
auth: Auth.bearer("test-key"),
})
.model({ id: "deepseek-chat" })
const request = LLM.request({
id: "req_1",
@ -63,10 +63,11 @@ describe("OpenAI-compatible Chat route", () => {
expect(prepared.model).toMatchObject({
id: "deepseek-chat",
provider: "deepseek",
route: "openai-compatible-chat",
route: { id: "openai-compatible-chat" },
})
expect(prepared.model.route.endpoint).toMatchObject({
baseURL: "https://api.deepseek.test/v1/",
apiKey: "test-key",
queryParams: { "api-version": "2026-01-01" },
query: { "api-version": "2026-01-01" },
})
expect(prepared.body).toEqual({
model: "deepseek-chat",
@ -93,13 +94,12 @@ describe("OpenAI-compatible Chat route", () => {
Effect.gen(function* () {
expect(
providerFamilies.map(([provider, family]) => {
const model = family.model(`${provider}-model`, { apiKey: "test-key" })
const model = family.configure({ apiKey: "test-key" }).model(`${provider}-model`)
return {
id: String(model.id),
provider: String(model.provider),
route: model.route,
baseURL: model.baseURL,
apiKey: model.apiKey,
route: model.route.id,
baseURL: model.route.endpoint.baseURL,
}
}),
).toEqual(
@ -108,19 +108,20 @@ describe("OpenAI-compatible Chat route", () => {
provider,
route: "openai-compatible-chat",
baseURL,
apiKey: "test-key",
})),
)
const custom = OpenAICompatible.deepseek.model("deepseek-chat", {
apiKey: "test-key",
baseURL: "https://custom.deepseek.test/v1",
})
const custom = OpenAICompatible.deepseek
.configure({
apiKey: "test-key",
baseURL: "https://custom.deepseek.test/v1",
})
.model("deepseek-chat")
expect(custom).toMatchObject({
provider: "deepseek",
route: "openai-compatible-chat",
baseURL: "https://custom.deepseek.test/v1",
route: { id: "openai-compatible-chat" },
})
expect(custom.route.endpoint.baseURL).toBe("https://custom.deepseek.test/v1")
}),
)

7
packages/llm/test/provider/openai-responses-cache.recorded.test.ts
View file

@ -2,14 +2,13 @@ import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { LLM } from "../../src"
import { LLMClient } from "../../src/route"
import * as OpenAIResponses from "../../src/protocols/openai-responses"
import * as OpenAI from "../../src/providers/openai"
import { LARGE_CACHEABLE_SYSTEM } from "../recorded-scenarios"
import { recordedTests } from "../recorded-test"
const model = OpenAIResponses.model({
id: "gpt-4.1-mini",
const model = OpenAI.configure({
apiKey: process.env.OPENAI_API_KEY ?? "fixture",
})
}).responses("gpt-4.1-mini")
// OpenAI caches prefixes automatically once they cross the 1024-token threshold;
// `CacheHint` is a no-op for the wire body. The stable signal is the

92
packages/llm/test/provider/openai-responses.test.ts
View file

@ -1,7 +1,7 @@
import { describe, expect } from "bun:test"
import { ConfigProvider, Effect, Layer, Stream } from "effect"
import { Headers, HttpClientRequest } from "effect/unstable/http"
import { LLM, LLMError, Message, ToolCallPart, Usage } from "../../src"
import { LLM, LLMError, Message, Model, ToolCallPart, Usage } from "../../src"
import { Auth, LLMClient, RequestExecutor, WebSocketExecutor } from "../../src/route"
import * as Azure from "../../src/providers/azure"
import * as OpenAI from "../../src/providers/openai"
@ -11,11 +11,9 @@ import { it } from "../lib/effect"
import { dynamicResponse, fixedResponse } from "../lib/http"
import { sseEvents } from "../lib/sse"
const model = OpenAIResponses.model({
id: "gpt-4.1-mini",
baseURL: "https://api.openai.test/v1/",
headers: { authorization: "Bearer test" },
})
const model = OpenAIResponses.route
.with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("test") })
.model({ id: "gpt-4.1-mini" })
const request = LLM.request({
id: "req_1",
@ -49,7 +47,9 @@ describe("OpenAI Responses route", () => {
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(
LLM.updateRequest(request, {
model: OpenAI.responsesWebSocket("gpt-4.1-mini", { baseURL: "https://api.openai.test/v1/", apiKey: "test" }),
model: OpenAI.configure({ baseURL: "https://api.openai.test/v1/", apiKey: "test" }).responsesWebSocket(
"gpt-4.1-mini",
),
}),
)
@ -95,10 +95,12 @@ describe("OpenAI Responses route", () => {
)
const response = yield* LLMClient.generate(
LLM.request({
model: OpenAI.responsesWebSocket("gpt-4.1-mini", { baseURL: "https://api.openai.test/v1/", apiKey: "test" }),
model: OpenAI.configure({ baseURL: "https://api.openai.test/v1/", apiKey: "test" }).responsesWebSocket(
"gpt-4.1-mini",
),
prompt: "Say hello.",
}),
).pipe(Effect.provide(LLMClient.layerWithWebSocket.pipe(Layer.provide(deps))))
).pipe(Effect.provide(LLMClient.layer.pipe(Layer.provide(deps))))
expect(response.text).toBe("Hi")
expect(opened).toEqual([{ url: "wss://api.openai.test/v1/responses", authorization: "Bearer test" }])
@ -113,33 +115,6 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("requires WebSocket runtime for OpenAI Responses WebSocket", () =>
Effect.gen(function* () {
const error = yield* LLMClient.generate(
LLM.request({
model: OpenAI.responsesWebSocket("gpt-4.1-mini", { baseURL: "https://api.openai.test/v1/", apiKey: "test" }),
prompt: "Say hello.",
}),
).pipe(
Effect.provide(
LLMClient.layer.pipe(
Layer.provide(
Layer.succeed(
RequestExecutor.Service,
RequestExecutor.Service.of({
execute: () => Effect.die("unexpected HTTP request"),
}),
),
),
),
),
Effect.flip,
)
expect(error.message).toContain("requires WebSocketExecutor.Service")
}),
)
it.effect("fails immediately when WebSocket is already closed", () =>
Effect.gen(function* () {
const error = yield* WebSocketExecutor.fromWebSocket(
@ -155,7 +130,7 @@ describe("OpenAI Responses route", () => {
Effect.gen(function* () {
yield* LLMClient.generate(
LLM.updateRequest(request, {
model: OpenAIResponses.model({ ...model, queryParams: { "api-version": "v1" } }),
model: Model.update(model, { route: model.route.with({ endpoint: { query: { "api-version": "v1" } } }) }),
}),
).pipe(
Effect.provide(
@ -177,17 +152,18 @@ describe("OpenAI Responses route", () => {
Effect.gen(function* () {
yield* LLMClient.generate(
LLM.updateRequest(request, {
model: Azure.responses("gpt-4.1-mini", {
model: Azure.configure({
baseURL: "https://opencode-test.openai.azure.com/openai/v1/",
apiKey: "azure-key",
headers: { authorization: "Bearer stale" },
}),
}).responses("gpt-4.1-mini"),
}),
).pipe(
Effect.provide(
dynamicResponse((input) =>
Effect.gen(function* () {
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
expect(web.url).toBe("https://opencode-test.openai.azure.com/openai/v1/responses?api-version=v1")
expect(web.headers.get("api-key")).toBe("azure-key")
expect(web.headers.get("authorization")).toBeNull()
return input.respond(sseEvents({ type: "response.completed", response: {} }), {
@ -203,7 +179,7 @@ describe("OpenAI Responses route", () => {
it.effect("loads OpenAI default auth from Effect Config", () =>
LLMClient.generate(
LLM.updateRequest(request, {
model: OpenAI.responses("gpt-4.1-mini", { baseURL: "https://api.openai.test/v1/" }),
model: OpenAI.configure({ baseURL: "https://api.openai.test/v1/" }).responses("gpt-4.1-mini"),
}),
).pipe(
configEnv({ OPENAI_API_KEY: "env-key" }),
@ -224,10 +200,10 @@ describe("OpenAI Responses route", () => {
it.effect("lets explicit auth override OpenAI default API key auth", () =>
LLMClient.generate(
LLM.updateRequest(request, {
model: OpenAI.responses("gpt-4.1-mini", {
model: OpenAI.configure({
baseURL: "https://api.openai.test/v1/",
auth: Auth.bearer("oauth-token"),
}),
}).responses("gpt-4.1-mini"),
}),
).pipe(
Effect.provide(
@ -274,7 +250,7 @@ describe("OpenAI Responses route", () => {
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
model: OpenAI.model("gpt-5.2", { baseURL: "https://api.openai.test/v1/" }),
model: OpenAI.configure({ baseURL: "https://api.openai.test/v1/", apiKey: "test" }).model("gpt-5.2"),
prompt: "think",
providerOptions: {
openai: {
@ -295,14 +271,15 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("request OpenAI provider options override model defaults", () =>
it.effect("request OpenAI provider options override route defaults", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
model: OpenAI.model("gpt-4.1-mini", {
model: OpenAI.configure({
baseURL: "https://api.openai.test/v1/",
apiKey: "test",
providerOptions: { openai: { promptCacheKey: "model_cache" } },
}),
}).model("gpt-4.1-mini"),
prompt: "no cache",
providerOptions: { openai: { promptCacheKey: "request_cache" } },
}),
@ -532,17 +509,36 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("lowers user image content", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
id: "req_media",
model,
messages: [Message.user({ type: "media", mediaType: "image/png", data: "AAECAw==" })],
}),
)
expect(prepared.body.input).toEqual([
{
role: "user",
content: [{ type: "input_image", image_url: "data:image/png;base64,AAECAw==" }],
},
])
}),
)
it.effect("rejects unsupported user media content", () =>
Effect.gen(function* () {
const error = yield* LLMClient.prepare(
LLM.request({
id: "req_media",
model,
messages: [Message.user({ type: "media", mediaType: "image/png", data: "AAECAw==" })],
messages: [Message.user({ type: "media", mediaType: "application/pdf", data: "AAECAw==" })],
}),
).pipe(Effect.flip)
expect(error.message).toContain("OpenAI Responses user messages only support text content for now")
expect(error.message).toContain("OpenAI Responses user media content only supports images")
}),
)

12
packages/llm/test/provider/openrouter.test.ts
View file

@ -8,15 +8,14 @@ import { it } from "../lib/effect"
describe("OpenRouter", () => {
it.effect("prepares OpenRouter models through the OpenAI-compatible Chat route", () =>
Effect.gen(function* () {
const model = OpenRouter.model("openai/gpt-4o-mini", { apiKey: "test-key" })
const model = OpenRouter.configure({ apiKey: "test-key" }).model("openai/gpt-4o-mini")
expect(model).toMatchObject({
id: "openai/gpt-4o-mini",
provider: "openrouter",
route: "openrouter",
baseURL: "https://openrouter.ai/api/v1",
apiKey: "test-key",
route: { id: "openrouter" },
})
expect(model.route.endpoint.baseURL).toBe("https://openrouter.ai/api/v1")
const prepared = yield* LLMClient.prepare(LLM.request({ model, prompt: "Say hello." }))
@ -33,7 +32,8 @@ describe("OpenRouter", () => {
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(
LLM.request({
model: OpenRouter.model("anthropic/claude-3.7-sonnet:thinking", {
model: OpenRouter.configure({
apiKey: "test-key",
providerOptions: {
openrouter: {
usage: true,
@ -41,7 +41,7 @@ describe("OpenRouter", () => {
promptCacheKey: "session_123",
},
},
}),
}).model("anthropic/claude-3.7-sonnet:thinking"),
prompt: "Think briefly.",
}),
)

9
packages/llm/test/recorded-golden.ts
View file

@ -1,7 +1,7 @@
import type { HttpRecorder } from "@opencode-ai/http-recorder"
import { describe, type TestOptions } from "bun:test"
import { Effect } from "effect"
import type { ModelRef } from "../src"
import type { Model } from "../src"
import { goldenScenarioTags, runGoldenScenario, type GoldenScenarioID } from "./recorded-scenarios"
import { recordedTests } from "./recorded-test"
import { kebab } from "./recorded-utils"
@ -22,7 +22,7 @@ type ScenarioInput =
type TargetInput = {
readonly name: string
readonly model: ModelRef
readonly model: Model
readonly protocol?: string
readonly requires?: ReadonlyArray<string>
readonly transport?: Transport
@ -38,19 +38,20 @@ const scenarioInput = (input: ScenarioInput) => (typeof input === "string" ? { i
const scenarioTitle = (id: GoldenScenarioID) => {
if (id === "text") return "streams text"
if (id === "tool-call") return "streams tool call"
if (id === "image") return "reads image text"
return "drives a tool loop"
}
const defaultPrefix = (target: TargetInput) => {
if (target.prefix) return target.prefix
const transport = target.transport === "websocket" ? "-websocket" : ""
return `${target.model.provider}-${target.protocol ?? target.model.route}${transport}`
return `${target.model.provider}-${target.protocol ?? target.model.route.id}${transport}`
}
const metadata = (target: TargetInput) => ({
provider: target.model.provider,
protocol: target.protocol,
route: target.model.route,
route: target.model.route.id,
transport: target.transport ?? "http",
model: target.model.id,
...target.metadata,

76
packages/llm/test/recorded-scenarios.ts
View file

@ -1,6 +1,6 @@
import { expect } from "bun:test"
import { Effect, Schema, Stream } from "effect"
import { LLM, LLMEvent, LLMResponse, ToolChoice, ToolDefinition, type LLMRequest, type ModelRef } from "../src"
import { LLM, LLMEvent, LLMResponse, Message, ToolChoice, ToolDefinition, type LLMRequest, type Model } from "../src"
import { LLMClient } from "../src/route"
import { tool } from "../src/tool"
@ -41,7 +41,7 @@ export const weatherRuntimeTool = tool({
export const textRequest = (input: {
readonly id: string
readonly model: ModelRef
readonly model: Model
readonly prompt?: string
readonly maxTokens?: number
readonly temperature?: number | false
@ -52,15 +52,17 @@ export const textRequest = (input: {
system: "You are concise.",
prompt: input.prompt ?? "Reply with exactly: Hello!",
cache: "none",
providerOptions:
input.model.route.id === "gemini" ? { gemini: { thinkingConfig: { thinkingBudget: 0 } } } : undefined,
generation:
input.temperature === false
? { maxTokens: input.maxTokens ?? 20 }
: { maxTokens: input.maxTokens ?? 20, temperature: input.temperature ?? 0 },
? { maxTokens: input.maxTokens ?? 80 }
: { maxTokens: input.maxTokens ?? 80, temperature: input.temperature ?? 0 },
})
export const weatherToolRequest = (input: {
readonly id: string
readonly model: ModelRef
readonly model: Model
readonly maxTokens?: number
readonly temperature?: number | false
}) =>
@ -80,7 +82,7 @@ export const weatherToolRequest = (input: {
export const weatherToolLoopRequest = (input: {
readonly id: string
readonly model: ModelRef
readonly model: Model
readonly system?: string
readonly maxTokens?: number
readonly temperature?: number | false
@ -99,7 +101,7 @@ export const weatherToolLoopRequest = (input: {
export const goldenWeatherToolLoopRequest = (input: {
readonly id: string
readonly model: ModelRef
readonly model: Model
readonly maxTokens?: number
readonly temperature?: number | false
}) =>
@ -108,6 +110,39 @@ export const goldenWeatherToolLoopRequest = (input: {
system: "Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.",
})
const RESTROOM_IMAGE_TEXT = "jiggling restroom prison"
const restroomImage = () =>
Effect.promise(() => Bun.file(new URL("./fixtures/media/restroom.png", import.meta.url)).bytes()).pipe(
Effect.map((bytes) => Buffer.from(bytes).toString("base64")),
)
export const imageRequest = (input: {
readonly id: string
readonly model: Model
readonly image: string
readonly maxTokens?: number
readonly temperature?: number | false
}) =>
LLM.request({
id: input.id,
model: input.model,
system: "Read images carefully. Reply only with the visible text.",
messages: [
Message.user([
{
type: "text",
text: "The image contains exactly three lowercase English words. Read them left to right and reply with only those words.",
},
{ type: "media", mediaType: "image/png", data: input.image },
]),
],
cache: "none",
generation:
input.temperature === false
? { maxTokens: input.maxTokens ?? 20 }
: { maxTokens: input.maxTokens ?? 20, temperature: input.temperature ?? 0 },
})
export const runWeatherToolLoop = (request: LLMRequest) =>
LLMClient.stream({
request,
@ -158,20 +193,28 @@ export const expectGoldenWeatherToolLoop = (events: ReadonlyArray<LLMEvent>) =>
expect(LLMResponse.text({ events }).trim()).toMatch(/^Paris is sunny\.?$/)
}
export type GoldenScenarioID = "text" | "tool-call" | "tool-loop"
export type GoldenScenarioID = "text" | "tool-call" | "tool-loop" | "image"
export interface GoldenScenarioContext {
readonly id: string
readonly model: ModelRef
readonly model: Model
readonly maxTokens?: number
readonly temperature?: number | false
}
const generate = (request: LLMRequest) => LLMClient.generate(request)
const normalizeImageText = (value: string) =>
value
.toLowerCase()
.replace(/[^a-z\s]/g, "")
.replace(/\s+/g, " ")
.trim()
export const goldenScenarioTags = (id: GoldenScenarioID) => {
if (id === "text") return ["text", "golden"]
if (id === "tool-call") return ["tool", "tool-call", "golden"]
if (id === "image") return ["media", "image", "vision", "golden"]
return ["tool", "tool-loop", "golden"]
}
@ -206,6 +249,21 @@ export const runGoldenScenario = (id: GoldenScenarioID, context: GoldenScenarioC
return
}
if (id === "image") {
const response = yield* generate(
imageRequest({
id: context.id,
model: context.model,
image: yield* restroomImage(),
maxTokens: context.maxTokens ?? 20,
temperature: context.temperature,
}),
)
expect(normalizeImageText(response.text)).toBe(RESTROOM_IMAGE_TEXT)
expectFinish(response.events, "stop")
return
}
expectGoldenWeatherToolLoop(
yield* runWeatherToolLoop(
goldenWeatherToolLoopRequest({

4
packages/llm/test/recorded-test.ts
View file

@ -69,8 +69,6 @@ export const recordedTests = (options: RecordedTestsOptions) =>
requestExecutor,
webSocketCassetteLayer(cassette, { metadata: recorderMetadata, mode }),
)
return Layer.mergeAll(deps, LLMClient.layerWithWebSocket.pipe(Layer.provide(deps))).pipe(
Layer.provide(cassetteService),
)
return Layer.mergeAll(deps, LLMClient.layer.pipe(Layer.provide(deps))).pipe(Layer.provide(cassetteService))
},
})

43
packages/llm/test/route.test.ts Normal file
View file

@ -0,0 +1,43 @@
import { describe, expect, test } from "bun:test"
import * as OpenAIChat from "../src/protocols/openai-chat"
import { Auth } from "../src/route"
describe("Route.with", () => {
test("merges endpoint query and header defaults while replacing auth and id", () => {
const auth = Auth.headers({ "x-auth": "new" })
const route = OpenAIChat.route
.with({
id: "base-chat",
endpoint: {
baseURL: "https://api.example.test/v1",
query: { keep: "base", base: "1" },
},
headers: { "x-base": "base", "x-override": "base" },
auth: Auth.headers({ "x-auth": "old" }),
})
.with({
id: "patched-chat",
endpoint: { query: { keep: "patch", patch: "1" } },
headers: { "x-override": "patch", "x-patch": "patch" },
auth,
})
expect(route.id).toBe("patched-chat")
expect(route.auth).toBe(auth)
expect(route.endpoint).toMatchObject({
baseURL: "https://api.example.test/v1",
path: "/chat/completions",
query: { keep: "patch", base: "1", patch: "1" },
})
expect(route.defaults.headers).toEqual({
"x-base": "base",
"x-override": "patch",
"x-patch": "patch",
})
expect(route.defaults.http?.headers).toEqual({
"x-base": "base",
"x-override": "patch",
"x-patch": "patch",
})
})
})

23
packages/llm/test/schema.test.ts
View file

@ -1,16 +1,19 @@
import { describe, expect, test } from "bun:test"
import { Schema } from "effect"
import { ContentPart, LLMEvent, LLMRequest, ModelID, ModelLimits, ModelRef, ProviderID, Usage } from "../src/schema"
import * as OpenAIChat from "../src/protocols/openai-chat"
import * as OpenAIResponses from "../src/protocols/openai-responses"
import { ContentPart, LLMEvent, LLMRequest, Model, ModelID, ProviderID, Usage } from "../src/schema"
import { ProviderShared } from "../src/protocols/shared"
const model = new ModelRef({
const model = new Model({
id: ModelID.make("fake-model"),
provider: ProviderID.make("fake-provider"),
route: "openai-chat",
baseURL: "https://fake.local",
limits: new ModelLimits({}),
route: OpenAIChat.route,
})
const decodeLLMRequest = Schema.decodeUnknownSync(LLMRequest as unknown as Schema.Decoder<LLMRequest>)
const decodeLLMEvent = Schema.decodeUnknownSync(LLMEvent as unknown as Schema.Decoder<LLMEvent>)
describe("llm schema", () => {
test("decodes a minimal request", () => {
const input: unknown = {
@ -22,26 +25,26 @@ describe("llm schema", () => {
generation: {},
}
const decoded = Schema.decodeUnknownSync(LLMRequest)(input)
const decoded = decodeLLMRequest(input)
expect(decoded.id).toBe("req_1")
expect(decoded.messages[0]?.content[0]?.type).toBe("text")
})
test("accepts custom route ids", () => {
const decoded = Schema.decodeUnknownSync(LLMRequest)({
model: { ...model, route: "custom-route" },
const decoded = decodeLLMRequest({
model: Model.update(model, { route: OpenAIResponses.route }),
system: [],
messages: [],
tools: [],
generation: {},
})
expect(decoded.model.route).toBe("custom-route")
expect(decoded.model.route.id).toBe("openai-responses")
})
test("rejects invalid event type", () => {
expect(() => Schema.decodeUnknownSync(LLMEvent)({ type: "bogus" })).toThrow()
expect(() => decodeLLMEvent({ type: "bogus" })).toThrow()
})
test("finish constructors accept usage input", () => {

57
packages/llm/test/tool-runtime.test.ts
View file

@ -1,7 +1,7 @@
import { describe, expect } from "bun:test"
import { Effect, Schema, Stream } from "effect"
import { GenerationOptions, LLM, LLMEvent, LLMRequest, LLMResponse, ToolChoice } from "../src"
import { LLMClient } from "../src/route"
import { Auth, LLMClient } from "../src/route"
import * as AnthropicMessages from "../src/protocols/anthropic-messages"
import * as OpenAIChat from "../src/protocols/openai-chat"
import { tool, ToolFailure, type ToolExecuteContext } from "../src/tool"
@ -12,11 +12,9 @@ import { dynamicResponse, scriptedResponses } from "./lib/http"
import { deltaChunk, finishChunk, toolCallChunk } from "./lib/openai-chunks"
import { sseEvents } from "./lib/sse"
const model = OpenAIChat.model({
id: "gpt-4o-mini",
baseURL: "https://api.openai.test/v1/",
headers: { authorization: "Bearer test" },
})
const model = OpenAIChat.route
.with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("test") })
.model({ id: "gpt-4o-mini" })
const Json = Schema.fromJsonString(Schema.Unknown)
const decodeJson = Schema.decodeUnknownSync(Json)
@ -141,6 +139,45 @@ describe("LLMClient tools", () => {
}),
)
it.effect("preserves content tool results from dynamic tools", () =>
Effect.gen(function* () {
const screenshot = tool({
description: "Capture a screenshot.",
jsonSchema: { type: "object", properties: {} },
execute: () =>
Effect.succeed({
type: "content" as const,
value: [
{ type: "text" as const, text: "Screenshot captured." },
{ type: "media" as const, mediaType: "image/png", data: "AAAA" },
],
}),
})
const events = Array.from(
yield* LLMClient.stream({ request: baseRequest, tools: { screenshot } }).pipe(
Stream.runCollect,
Effect.provide(
scriptedResponses([sseEvents(toolCallChunk("call_1", "screenshot", "{}"), finishChunk("tool_calls"))]),
),
),
)
expect(events.find(LLMEvent.is.toolResult)).toMatchObject({
type: "tool-result",
id: "call_1",
name: "screenshot",
result: {
type: "content",
value: [
{ type: "text", text: "Screenshot captured." },
{ type: "media", mediaType: "image/png", data: "AAAA" },
],
},
})
}),
)
it.effect("executes tool calls for one step without looping by default", () =>
Effect.gen(function* () {
const layer = scriptedResponses([
@ -249,7 +286,9 @@ describe("LLMClient tools", () => {
yield* TestToolRuntime.runTools({
request: LLM.updateRequest(baseRequest, {
model: AnthropicMessages.model({ id: "claude-sonnet-4-5", apiKey: "test" }),
model: AnthropicMessages.route
.with({ auth: Auth.header("x-api-key", "test") })
.model({ id: "claude-sonnet-4-5" }),
}),
tools: { get_weather },
}).pipe(Stream.runCollect, Effect.provide(layer))
@ -496,7 +535,9 @@ describe("LLMClient tools", () => {
const events = Array.from(
yield* TestToolRuntime.runTools({
request: LLM.updateRequest(baseRequest, {
model: AnthropicMessages.model({ id: "claude-sonnet-4-5", apiKey: "test" }),
model: AnthropicMessages.route
.with({ auth: Auth.header("x-api-key", "test") })
.model({ id: "claude-sonnet-4-5" }),
}),
tools: {},
}).pipe(Stream.runCollect, Effect.provide(layer)),

3
packages/llm/test/tool.types.ts
View file

@ -1,10 +1,11 @@
import { Effect, Schema } from "effect"
import { LLM } from "../src"
import * as OpenAIChat from "../src/protocols/openai-chat"
import { Auth } from "../src/route"
import { tool } from "../src/tool"
const request = LLM.request({
model: OpenAIChat.model({ id: "gpt-4o-mini", apiKey: "fixture" }),
model: OpenAIChat.route.with({ auth: Auth.bearer("fixture") }).model({ id: "gpt-4o-mini" }),
prompt: "Use the tool.",
})