ceos/opencode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(core): add embedded v2 session runtime and tool foundation (#30632)

Browse source
This commit is contained in:
Kit Langton 2026-06-03 23:02:17 -04:00 • committed by GitHub
commit 76ee87ead8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
215 changed files with 31398 additions and 3332 deletions
Download patch file Download diff file Expand all files Collapse all files

144
packages/llm/test/lib/tool-runtime.ts
View file

@ -1,8 +1,142 @@
import { Effect, Stream } from "effect"
import { LLMClient } from "../../src/route"
import type { Tools } from "../../src/tool"
import type { RunOptions } from "../../src/tool-runtime"
import {
LLMEvent,
LLMRequest,
Message,
type ContentPart,
type ProviderMetadata,
type ToolCallPart,
ToolResultPart,
type ToolResultValue,
type Usage,
} from "../../src/schema"
import { type Tools, toDefinitions } from "../../src/tool"
import { ToolRuntime } from "../../src/tool-runtime"
type CompatRunOptions<T extends Tools> = RunOptions<T> & { readonly maxSteps?: number }
interface RunOptions<T extends Tools> {
readonly request: LLMRequest
readonly tools: T
readonly maxSteps?: number
}
export const runTools = <T extends Tools>(options: CompatRunOptions<T>) =>
LLMClient.stream({ ...options, stopWhen: options.stopWhen ?? LLMClient.stepCountIs(options.maxSteps ?? 10) })
/** Test-owned continuation loop. Production callers must own durable history. */
export const runTools = <T extends Tools>(options: RunOptions<T>) =>
Stream.unwrap(
Effect.gen(function* () {
const names = new Set(Object.keys(options.tools))
let request = LLMRequest.update(options.request, {
tools: [...options.request.tools.filter((tool) => !names.has(tool.name)), ...toDefinitions(options.tools)],
})
let usage: Usage | undefined
const events: LLMEvent[] = []
for (let step = 0; step < (options.maxSteps ?? 10); step++) {
const streamed = Array.from(yield* LLMClient.stream(request).pipe(Stream.runCollect))
const state = stepState(streamed)
usage = addUsage(usage, state.usage)
events.push(...streamed.filter((event) => event.type !== "finish").map((event) => indexStep(event, step)))
if (state.toolCalls.length === 0) {
events.push(LLMEvent.finish({ reason: state.reason, usage, providerMetadata: state.providerMetadata }))
return Stream.fromIterable(events)
}
const dispatched = yield* Effect.forEach(
state.toolCalls,
(call) => ToolRuntime.dispatch(options.tools, call).pipe(Effect.map((result) => [call, result] as const)),
{ concurrency: 10 },
)
events.push(...dispatched.flatMap(([, result]) => result.events))
if (step + 1 >= (options.maxSteps ?? 10)) {
events.push(LLMEvent.finish({ reason: state.reason, usage, providerMetadata: state.providerMetadata }))
return Stream.fromIterable(events)
}
request = LLMRequest.update(request, {
messages: [
...request.messages,
Message.assistant(state.assistantContent),
...dispatched.map(([call, dispatched]) =>
Message.tool({ id: call.id, name: call.name, result: dispatched.result }),
),
],
})
}
return Stream.fromIterable(events)
}),
)
const indexStep = (event: LLMEvent, index: number): LLMEvent => {
if (event.type === "step-start") return LLMEvent.stepStart({ index })
if (event.type === "step-finish") return LLMEvent.stepFinish({ ...event, index })
return event
}
const stepState = (events: ReadonlyArray<LLMEvent>) => {
const assistantContent: ContentPart[] = []
const toolCalls: ToolCallPart[] = []
let reason: Extract<LLMEvent, { type: "finish" }>["reason"] = "unknown"
let usage: Usage | undefined
let providerMetadata: ProviderMetadata | undefined
for (const event of events) {
if (event.type === "text-delta" || event.type === "reasoning-delta") {
appendText(assistantContent, event.type === "text-delta" ? "text" : "reasoning", event.text)
} else if (event.type === "text-end" || event.type === "reasoning-end") {
appendText(assistantContent, event.type === "text-end" ? "text" : "reasoning", "", event.providerMetadata)
} else if (event.type === "tool-call") {
assistantContent.push(event)
if (!event.providerExecuted) toolCalls.push(event)
} else if (event.type === "tool-result" && event.providerExecuted && event.result !== undefined) {
assistantContent.push(
ToolResultPart.make({
id: event.id,
name: event.name,
result: event.result,
providerExecuted: true,
providerMetadata: event.providerMetadata,
}),
)
} else if (event.type === "finish") {
reason = event.reason
usage = event.usage
providerMetadata = event.providerMetadata
}
}
return { assistantContent, toolCalls, reason, usage, providerMetadata }
}
const appendText = (
content: ContentPart[],
type: "text" | "reasoning",
text: string,
providerMetadata?: ProviderMetadata,
) => {
const last = content.at(-1)
if (last?.type === type) {
content[content.length - 1] = { ...last, text: `${last.text}${text}`, providerMetadata: providerMetadata ?? last.providerMetadata }
return
}
content.push({ type, text, providerMetadata })
}
const addUsage = (left: Usage | undefined, right: Usage | undefined): Usage | undefined => {
if (!left) return right
if (!right) return left
const sum = (key: keyof Usage) =>
typeof left[key] !== "number" && typeof right[key] !== "number"
? undefined
: ((left[key] as number | undefined) ?? 0) + ((right[key] as number | undefined) ?? 0)
return {
inputTokens: sum("inputTokens"),
outputTokens: sum("outputTokens"),
nonCachedInputTokens: sum("nonCachedInputTokens"),
cacheReadInputTokens: sum("cacheReadInputTokens"),
cacheWriteInputTokens: sum("cacheWriteInputTokens"),
reasoningTokens: sum("reasoningTokens"),
totalTokens: sum("totalTokens"),
} as Usage
}

19
packages/llm/test/llm.test.ts
View file

@ -1,5 +1,5 @@
import { describe, expect, test } from "bun:test"
import { LLM, LLMResponse } from "../src"
import { CacheHint, LLM, LLMResponse } from "../src"
import * as OpenAIChat from "../src/protocols/openai-chat"
import * as OpenAIResponses from "../src/protocols/openai-responses"
import { LLMRequest, Message, Model, ToolCallPart, ToolChoice, ToolDefinition, ToolResultPart } from "../src/schema"
@ -135,6 +135,23 @@ describe("llm constructors", () => {
])
})
test("builds chronological text-only system updates separately from the initial system prompt", () => {
const update = Message.system([{ type: "text", text: "Use parameterized SQL.", cache: new CacheHint({ type: "ephemeral" }) }])
const request = LLM.request({
model: Model.make({ id: "fake-model", provider: "fake", route: chatRoute }),
system: "Initial operator prompt.",
messages: [Message.user("Review this."), update],
})
expect(update).toBeInstanceOf(Message)
expect(update).toEqual({
role: "system",
content: [{ type: "text", text: "Use parameterized SQL.", cache: { type: "ephemeral" } }],
})
expect(request.system).toEqual([{ type: "text", text: "Initial operator prompt." }])
expect(request.messages.map((message) => message.role)).toEqual(["user", "system"])
})
test("extracts output text from response events", () => {
expect(
LLMResponse.text({

91
packages/llm/test/provider/anthropic-messages.test.ts
View file

@ -13,6 +13,10 @@ const model = AnthropicMessages.route
.with({ endpoint: { baseURL: "https://api.anthropic.test/v1/" }, auth: Auth.header("x-api-key", "test") })
.model({ id: "claude-sonnet-4-5" })
const opus48 = AnthropicMessages.route
.with({ endpoint: { baseURL: "https://api.anthropic.test/v1/" }, auth: Auth.header("x-api-key", "test") })
.model({ id: "claude-opus-4-8" })
const request = LLM.request({
id: "req_1",
model,
@ -53,6 +57,93 @@ describe("Anthropic Messages route", () => {
}),
)
it.effect("lowers chronological system updates natively for Claude Opus 4.8 with cache hints", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<AnthropicMessages.AnthropicMessagesBody>(
LLM.request({
model: opus48,
messages: [
Message.user("Before."),
Message.system([{ type: "text", text: "Operator update.", cache: new CacheHint({ type: "ephemeral" }) }]),
Message.assistant("After."),
],
cache: "none",
}),
)
expect(prepared.body.messages).toEqual([
{ role: "user", content: [{ type: "text", text: "Before." }] },
{
role: "system",
content: [{ type: "text", text: "Operator update.", cache_control: { type: "ephemeral" } }],
},
{ role: "assistant", content: [{ type: "text", text: "After." }] },
])
}),
)
it.effect("lowers chronological system updates to wrapped user text for unsupported Anthropic models", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<AnthropicMessages.AnthropicMessagesBody>(
LLM.request({
model,
messages: [Message.user("Before."), Message.system("Treat </system-update> literally."), Message.assistant("After.")],
cache: "none",
}),
)
expect(prepared.body.messages).toEqual([
{
role: "user",
content: [
{ type: "text", text: "Before." },
{ type: "text", text: "<system-update>\nTreat &lt;/system-update&gt; literally.\n</system-update>" },
],
},
{ role: "assistant", content: [{ type: "text", text: "After." }] },
])
}),
)
it.effect("rejects non-text chronological system update content before send", () =>
Effect.gen(function* () {
const error = yield* LLMClient.prepare(
LLM.request({
model: opus48,
messages: [
Message.user("Before."),
Message.make({ role: "system", content: { type: "media", mediaType: "image/png", data: "AAECAw==" } }),
],
}),
).pipe(Effect.flip)
expect(error.message).toContain("Anthropic Messages system messages only support text content for now")
}),
)
it.effect("rejects invalid native chronological system update placement", () =>
Effect.gen(function* () {
const placementError = (messages: Parameters<typeof LLM.request>[0]["messages"]) =>
LLMClient.prepare(LLM.request({ model: opus48, messages, cache: "none" })).pipe(Effect.flip)
expect((yield* placementError([Message.system("First.")])).message).toContain("cannot be the first message")
expect((yield* placementError([Message.user("Before."), Message.system("One."), Message.system("Two.")])).message)
.toContain("cannot be consecutive")
expect((yield* placementError([Message.assistant("Plain."), Message.system("After plain assistant.")])).message)
.toContain("must follow a user message, tool result, or assistant server tool use")
expect(
(
yield* placementError([
Message.user("Use the tool."),
Message.assistant([ToolCallPart.make({ id: "call_1", name: "lookup", input: {} })]),
Message.system("Too early."),
Message.tool({ id: "call_1", name: "lookup", result: "Done." }),
])
).message,
).toContain("cannot appear between a local tool call and its tool result")
}),
)
it.effect("prepares tool call and tool result messages", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<AnthropicMessages.AnthropicMessagesBody>(

53
packages/llm/test/provider/bedrock-converse.test.ts
View file

@ -5,6 +5,7 @@ import { Effect } from "effect"
import { CacheHint, LLM, Message, ToolCallPart, ToolChoice } from "../../src"
import { LLMClient } from "../../src/route"
import { AmazonBedrock } from "../../src/providers"
import * as BedrockConverse from "../../src/protocols/bedrock-converse"
import { it } from "../lib/effect"
import { fixedResponse } from "../lib/http"
import {
@ -82,6 +83,23 @@ describe("Bedrock Converse route", () => {
}),
)
it.effect("lowers chronological system updates to wrapped user text in order", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<BedrockConverse.BedrockConverseBody>(
LLM.request({
model,
messages: [Message.user("Before."), Message.system("Update."), Message.assistant("After.")],
cache: "none",
}),
)
expect(prepared.body.messages).toEqual([
{ role: "user", content: [{ text: "Before." }, { text: "<system-update>\nUpdate.\n</system-update>" }] },
{ role: "assistant", content: [{ text: "After." }] },
])
}),
)
it.effect("prepares tool config with toolSpec and toolChoice", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(
@ -279,6 +297,41 @@ describe("Bedrock Converse route", () => {
}),
)
it.effect("preserves streamed reasoning signatures for continuation lowering", () =>
Effect.gen(function* () {
const body = eventStreamBody(
["messageStart", { role: "assistant" }],
["contentBlockDelta", { contentBlockIndex: 0, delta: { reasoningContent: { text: "Let me think." } } }],
["contentBlockDelta", { contentBlockIndex: 0, delta: { reasoningContent: { signature: "sig_1" } } }],
["contentBlockStop", { contentBlockIndex: 0 }],
["messageStop", { stopReason: "end_turn" }],
)
const response = yield* LLMClient.generate(baseRequest).pipe(Effect.provide(fixedBytes(body)))
const reasoning = response.events.find((event) => event.type === "reasoning-end")
expect(reasoning).toEqual({
type: "reasoning-end",
id: "reasoning-0",
providerMetadata: { bedrock: { signature: "sig_1" } },
})
const prepared = yield* LLMClient.prepare<BedrockConverse.BedrockConverseBody>(
LLM.request({
model,
messages: [
Message.assistant([
{ type: "reasoning", text: "Let me think.", providerMetadata: reasoning?.providerMetadata },
]),
],
cache: "none",
}),
)
expect(prepared.body.messages).toEqual([
{ role: "assistant", content: [{ reasoningContent: { reasoningText: { text: "Let me think.", signature: "sig_1" } } }] },
])
}),
)
it.effect("emits provider-error for throttlingException", () =>
Effect.gen(function* () {
const body = eventStreamBody(

82
packages/llm/test/provider/gemini.test.ts
View file

@ -35,6 +35,22 @@ describe("Gemini route", () => {
}),
)
it.effect("lowers chronological system updates to wrapped user text in order", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<Gemini.GeminiBody>(
LLM.request({
model,
messages: [Message.user("Before."), Message.system("Update."), Message.assistant("After.")],
}),
)
expect(prepared.body.contents).toEqual([
{ role: "user", parts: [{ text: "Before." }, { text: "<system-update>\nUpdate.\n</system-update>" }] },
{ role: "model", parts: [{ text: "After." }] },
])
}),
)
it.effect("prepares multimodal user input and tool history", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(
@ -241,6 +257,72 @@ describe("Gemini route", () => {
}),
)
it.effect("preserves thoughtSignature for reasoning and tool-call continuation", () =>
Effect.gen(function* () {
const body = sseEvents({
candidates: [
{
content: {
role: "model",
parts: [
{ text: "thinking", thought: true },
{ text: "", thought: true, thoughtSignature: "thought_sig" },
{ functionCall: { name: "lookup", args: { query: "weather" } }, thoughtSignature: "tool_sig" },
],
},
finishReason: "STOP",
},
],
})
const response = yield* LLMClient.generate(
LLM.updateRequest(request, {
tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }],
}),
).pipe(Effect.provide(fixedResponse(body)))
const reasoning = response.events.find((event) => event.type === "reasoning-start")
const reasoningEnd = response.events.find((event) => event.type === "reasoning-end")
const toolCall = response.events.find((event) => event.type === "tool-call")
expect(reasoning).toEqual({
type: "reasoning-start",
id: "reasoning-0",
providerMetadata: undefined,
})
expect(reasoningEnd).toEqual({
type: "reasoning-end",
id: "reasoning-0",
providerMetadata: { google: { thoughtSignature: "thought_sig" } },
})
expect(toolCall).toMatchObject({ providerMetadata: { google: { thoughtSignature: "tool_sig" } } })
const prepared = yield* LLMClient.prepare<Gemini.GeminiBody>(
LLM.request({
model,
messages: [
Message.assistant([
{ type: "reasoning", text: "thinking", providerMetadata: reasoningEnd?.providerMetadata },
ToolCallPart.make({
id: "tool_0",
name: "lookup",
input: { query: "weather" },
providerMetadata: toolCall?.providerMetadata,
}),
]),
],
}),
)
expect(prepared.body.contents).toEqual([
{
role: "model",
parts: [
{ text: "thinking", thought: true, thoughtSignature: "thought_sig" },
{ functionCall: { name: "lookup", args: { query: "weather" } }, thoughtSignature: "tool_sig" },
],
},
])
}),
)
it.effect("emits streamed tool calls and maps finish reason", () =>
Effect.gen(function* () {
const body = sseEvents({

37
packages/llm/test/provider/openai-chat.test.ts
View file

@ -50,6 +50,35 @@ describe("OpenAI Chat route", () => {
}),
)
it.effect("lowers chronological system updates to escaped user wrappers in order", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIChat.OpenAIChatBody>(
LLM.request({
model,
messages: [Message.user("Before."), Message.system("Treat <admin> & data literally."), Message.assistant("After.")],
}),
)
expect(prepared.body.messages).toEqual([
{ role: "user", content: "Before.\n<system-update>\nTreat &lt;admin&gt; &amp; data literally.\n</system-update>" },
{ role: "assistant", content: "After." },
])
}),
)
it.effect("replays canonical reasoning as OpenAI-compatible reasoning_content", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIChat.OpenAIChatBody>(
LLM.request({
model,
messages: [Message.assistant([{ type: "reasoning", text: "thinking" }, { type: "text", text: "Hello" }])],
}),
)
expect(prepared.body.messages).toEqual([{ role: "assistant", content: "Hello", reasoning_content: "thinking" }])
}),
)
it.effect("maps OpenAI provider options to Chat options", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIChat.OpenAIChatBody>(
@ -196,17 +225,17 @@ describe("OpenAI Chat route", () => {
}),
)
it.effect("rejects unsupported assistant reasoning content", () =>
it.effect("lowers reasoning-only assistant history", () =>
Effect.gen(function* () {
const error = yield* LLMClient.prepare(
const prepared = yield* LLMClient.prepare<OpenAIChat.OpenAIChatBody>(
LLM.request({
id: "req_reasoning",
model,
messages: [Message.assistant({ type: "reasoning", text: "hidden" })],
}),
).pipe(Effect.flip)
)
expect(error.message).toContain("OpenAI Chat assistant messages only support text and tool-call content for now")
expect(prepared.body.messages).toEqual([{ role: "assistant", content: null, reasoning_content: "hidden" }])
}),
)

58
packages/llm/test/provider/openai-responses.test.ts
View file

@ -57,6 +57,28 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("lowers chronological system updates to escaped user wrappers in order", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
model,
messages: [Message.user("Before."), Message.system("Treat </system-update> literally."), Message.assistant("After.")],
}),
)
expect(prepared.body.input).toEqual([
{
role: "user",
content: [
{ type: "input_text", text: "Before." },
{ type: "input_text", text: "<system-update>\nTreat &lt;/system-update&gt; literally.\n</system-update>" },
],
},
{ role: "assistant", content: [{ type: "output_text", text: "After." }] },
])
}),
)
it.effect("prepares OpenAI Responses WebSocket target", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(
@ -857,6 +879,42 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("references stored provider-executed hosted tool results by id", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
model,
messages: [
Message.assistant([
ToolCallPart.make({
id: "ws_1",
name: "web_search",
input: { query: "effect 4" },
providerExecuted: true,
providerMetadata: { openai: { itemId: "ws_1" } },
}),
{
type: "tool-result",
id: "ws_1",
name: "web_search",
result: { type: "json", value: { type: "web_search_call", id: "ws_1", status: "completed" } },
providerExecuted: true,
providerMetadata: { openai: { itemId: "ws_1" } },
},
]),
Message.user("Continue."),
],
providerOptions: { openai: { store: true } },
}),
)
expect(prepared.body.input).toEqual([
{ type: "item_reference", id: "ws_1" },
{ role: "user", content: [{ type: "input_text", text: "Continue." }] },
])
}),
)
it.effect("joins streamed summary blocks into one continuation reasoning item", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(

68
packages/llm/test/recorded-scenarios.ts
View file

@ -5,15 +5,17 @@ import {
LLMEvent,
LLMResponse,
Message,
ToolRuntime,
ToolChoice,
ToolDefinition,
toDefinitions,
type ContentPart,
type FinishReason,
type LLMRequest,
type Model,
} from "../src"
import { LLMClient } from "../src/route"
import { tool } from "../src/tool"
import { Tool } from "../src/tool"
export const weatherToolName = "get_weather"
@ -40,7 +42,7 @@ export const weatherTool = ToolDefinition.make({
},
})
export const weatherRuntimeTool = tool({
export const weatherRuntimeTool = Tool.make({
description: weatherTool.description,
parameters: Schema.Struct({ city: Schema.String }),
success: Schema.Struct({ temperature: Schema.Number, condition: Schema.String }),
@ -87,14 +89,60 @@ const restroomImage = () =>
)
export const runWeatherToolLoop = (request: LLMRequest) =>
LLMClient.stream({
request,
tools: { [weatherToolName]: weatherRuntimeTool },
stopWhen: LLMClient.stepCountIs(10),
}).pipe(
Stream.runCollect,
Effect.map((events) => Array.from(events)),
)
Effect.gen(function* () {
const tools = { [weatherToolName]: weatherRuntimeTool }
let next = LLM.updateRequest(request, { tools: toDefinitions(tools) })
const events: LLMEvent[] = []
for (let step = 0; step < 10; step++) {
const response = yield* LLMClient.generate(next)
events.push(...response.events.filter((event) => event.type !== "finish"))
const calls = response.events.filter(LLMEvent.is.toolCall).filter((call) => !call.providerExecuted)
if (calls.length === 0) {
const finish = response.events.find(LLMEvent.is.finish)
if (finish) events.push(finish)
return events
}
const dispatched = yield* Effect.forEach(calls, (call) =>
ToolRuntime.dispatch(tools, call).pipe(Effect.map((result) => [call, result] as const)),
)
events.push(...dispatched.flatMap(([, result]) => result.events))
next = LLM.updateRequest(next, {
messages: [
...next.messages,
Message.assistant(assistantContent(response.events)),
...dispatched.map(([call, result]) => Message.tool({ id: call.id, name: call.name, result: result.result })),
],
})
}
throw new Error("Weather tool loop exceeded 10 steps")
})
const assistantContent = (events: ReadonlyArray<LLMEvent>) => {
const content: ContentPart[] = []
for (const event of events) {
if (event.type === "text-delta" || event.type === "reasoning-delta") {
const type = event.type === "text-delta" ? "text" : "reasoning"
const last = content.at(-1)
if (last?.type === type) {
content[content.length - 1] = { ...last, text: `${last.text}${event.text}` }
} else {
content.push({ type, text: event.text })
}
continue
}
if (event.type === "text-end" || event.type === "reasoning-end") {
const type = event.type === "text-end" ? "text" : "reasoning"
const last = content.at(-1)
if (last?.type === type) content[content.length - 1] = { ...last, providerMetadata: event.providerMetadata }
continue
}
if (event.type === "tool-call") content.push(event)
}
return content
}
export const expectFinish = (
events: ReadonlyArray<LLMEvent>,

11
packages/llm/test/schema.test.ts
View file

@ -43,6 +43,17 @@ describe("llm schema", () => {
expect(decoded.model.route.id).toBe("openai-responses")
})
test("decodes chronological system messages", () => {
const decoded = decodeLLMRequest({
model,
system: [],
messages: [{ role: "system", content: [{ type: "text", text: "Operator update." }] }],
tools: [],
})
expect(decoded.messages[0]).toMatchObject({ role: "system", content: [{ type: "text", text: "Operator update." }] })
})
test("rejects invalid event type", () => {
expect(() => decodeLLMEvent({ type: "bogus" })).toThrow()
})

270
packages/llm/test/tool-runtime.test.ts
View file

@ -1,11 +1,11 @@
import { describe, expect } from "bun:test"
import { Effect, Schema, Stream } from "effect"
import { GenerationOptions, LLM, LLMEvent, LLMRequest, LLMResponse, ToolChoice } from "../src"
import { GenerationOptions, LLM, LLMEvent, LLMRequest, LLMResponse, ToolChoice, ToolContent, ToolOutput, toolFileSourceFromUri, toDefinitions } from "../src"
import { Auth, LLMClient } from "../src/route"
import * as AnthropicMessages from "../src/protocols/anthropic-messages"
import * as OpenAIChat from "../src/protocols/openai-chat"
import * as OpenAIResponses from "../src/protocols/openai-responses"
import { tool, ToolFailure, type ToolExecuteContext } from "../src/tool"
import { Tool, ToolFailure, type ToolExecuteContext } from "../src/tool"
import { ToolRuntime } from "../src/tool-runtime"
import { it } from "./lib/effect"
import * as TestToolRuntime from "./lib/tool-runtime"
@ -26,7 +26,7 @@ const baseRequest = LLM.request({
})
const weatherFailureCause = new Error("weather lookup denied")
const get_weather = tool({
const get_weather = Tool.make({
description: "Get current weather for a city.",
parameters: Schema.Struct({ city: Schema.String }),
success: Schema.Struct({ temperature: Schema.Number, condition: Schema.String }),
@ -38,7 +38,7 @@ const get_weather = tool({
}),
})
const schema_only_weather = tool({
const schema_only_weather = Tool.make({
description: "Get current weather for a city.",
parameters: Schema.Struct({ city: Schema.String }),
success: Schema.Struct({ temperature: Schema.Number, condition: Schema.String }),
@ -140,9 +140,161 @@ describe("LLMClient tools", () => {
}),
)
it.effect("projects encoded typed tool success into canonical model content", () =>
Effect.gen(function* () {
const calls: unknown[] = []
const projected = Tool.make({
description: "Project an encoded success.",
parameters: Schema.Struct({ prefix: Schema.String }),
success: Schema.Struct({ count: Schema.NumberFromString }),
execute: () => Effect.succeed({ count: 2 }),
toModelOutput: (input) => {
calls.push(input)
return [{ type: "text", text: `${input.parameters.prefix}:${input.output.count}` }]
},
})
const dispatched = yield* ToolRuntime.dispatch(
{ projected },
LLMEvent.toolCall({ id: "call_projected", name: "projected", input: { prefix: "count" } }),
)
expect(calls).toEqual([{ callID: "call_projected", parameters: { prefix: "count" }, output: { count: "2" } }])
expect(dispatched.result).toEqual({ type: "text", value: "count:2" })
expect(dispatched.output).toEqual({ structured: { count: "2" }, content: [{ type: "text", text: "count:2" }] })
expect(dispatched.events).toEqual([
LLMEvent.toolResult({
id: "call_projected",
name: "projected",
result: { type: "text", value: "count:2" },
output: { structured: { count: "2" }, content: [{ type: "text", text: "count:2" }] },
}),
])
}),
)
it.effect("uses the narrow default projection for encoded typed success", () =>
Effect.gen(function* () {
const text = Tool.make({
description: "Return text.",
parameters: Schema.Struct({}),
success: Schema.String,
execute: () => Effect.succeed("hello"),
})
const json = Tool.make({
description: "Return JSON.",
parameters: Schema.Struct({}),
success: Schema.Struct({ ok: Schema.Boolean }),
execute: () => Effect.succeed({ ok: true }),
})
expect((yield* ToolRuntime.dispatch({ text }, LLMEvent.toolCall({ id: "call_text", name: "text", input: {} }))).output)
.toEqual({ structured: "hello", content: [{ type: "text", text: "hello" }] })
expect((yield* ToolRuntime.dispatch({ json }, LLMEvent.toolCall({ id: "call_json", name: "json", input: {} }))).output)
.toEqual({ structured: { ok: true }, content: [] })
}),
)
it.effect("models canonical tool files with explicit data, url, and file sources", () =>
Effect.sync(() => {
const decode = Schema.decodeUnknownSync(ToolContent)
expect(decode({ type: "file", source: { type: "data", data: "AAAA" }, mime: "image/png" })).toEqual({
type: "file",
source: { type: "data", data: "AAAA" },
mime: "image/png",
})
expect(decode({ type: "file", source: { type: "url", url: "https://example.test/image.png" }, mime: "image/png" })).toEqual({
type: "file",
source: { type: "url", url: "https://example.test/image.png" },
mime: "image/png",
})
expect(decode({ type: "file", source: { type: "file", uri: "file:///tmp/image.png" }, mime: "image/png" })).toEqual({
type: "file",
source: { type: "file", uri: "file:///tmp/image.png" },
mime: "image/png",
})
}),
)
it.effect("converts canonical data files deliberately and rejects unmaterialized sources", () =>
Effect.sync(() => {
expect(
ToolOutput.toResultValue(
ToolOutput.make({}, [{ type: "file", source: { type: "data", data: "AAAA" }, mime: "image/png" }]),
),
).toEqual({ type: "content", value: [{ type: "media", mediaType: "image/png", data: "AAAA" }] })
expect(
ToolOutput.toResultValue(
ToolOutput.make({}, [{ type: "file", source: { type: "url", url: "https://example.test/image.png" }, mime: "image/png" }]),
),
).toEqual({ type: "error", value: 'Tool file source "url" must be materialized to inline data before provider conversion' })
expect(
ToolOutput.toResultValue(
ToolOutput.make({}, [{ type: "file", source: { type: "file", uri: "file:///tmp/image.png" }, mime: "image/png" }]),
),
).toEqual({ type: "error", value: 'Tool file source "file" must be materialized to inline data before provider conversion' })
expect(toolFileSourceFromUri("data:image/png;base64,AAAA")).toEqual({ type: "data", data: "AAAA" })
expect(toolFileSourceFromUri("https://example.test/image.png")).toEqual({ type: "url", url: "https://example.test/image.png" })
expect(toolFileSourceFromUri("file:///tmp/image.png")).toEqual({ type: "file", uri: "file:///tmp/image.png" })
expect(() => toolFileSourceFromUri("opaque-value")).toThrow("Unsupported tool file URI")
expect(() =>
ToolOutput.fromResultValue({
type: "content",
value: [{ type: "media", mediaType: "image/png", data: "https://example.test/image.png" }],
}),
).toThrow("Legacy tool-result media must contain raw base64 bytes or a base64 data URI")
}),
)
it.effect("settles projected url files as materialization errors", () =>
Effect.gen(function* () {
const remote = Tool.make({
description: "Return a remote file.",
parameters: Schema.Struct({}),
success: Schema.Struct({ ok: Schema.Boolean }),
execute: () => Effect.succeed({ ok: true }),
toModelOutput: () => [
{ type: "file", source: { type: "url", url: "https://example.test/image.png" }, mime: "image/png" },
],
})
const dispatched = yield* ToolRuntime.dispatch(
{ remote },
LLMEvent.toolCall({ id: "call_remote", name: "remote", input: {} }),
)
expect(dispatched.output).toBeUndefined()
expect(dispatched.result).toEqual({
type: "error",
value: 'Tool file source "url" must be materialized to inline data before provider conversion',
})
expect(dispatched.events.map((event) => event.type)).toEqual(["tool-error", "tool-result"])
}),
)
it.effect("derives typed output schemas and preserves dynamic output schemas", () =>
Effect.sync(() => {
const [typed] = toDefinitions({ get_weather })
const schema = { type: "object", properties: { result: { type: "string" } } } as const
const [dynamic] = toDefinitions({
dynamic: Tool.make({ description: "Dynamic tool.", jsonSchema: { type: "object" }, outputSchema: schema }),
})
expect(typed?.outputSchema).toMatchObject({
type: "object",
properties: { condition: { type: "string" } },
required: ["temperature", "condition"],
additionalProperties: false,
})
expect(Reflect.get(Reflect.get(typed?.outputSchema ?? {}, "properties") as object, "temperature")).toBeDefined()
expect(dynamic?.outputSchema).toEqual(schema)
}),
)
it.effect("preserves content tool results from dynamic tools", () =>
Effect.gen(function* () {
const screenshot = tool({
const screenshot = Tool.make({
description: "Capture a screenshot.",
jsonSchema: { type: "object", properties: {} },
execute: () =>
@ -156,7 +308,7 @@ describe("LLMClient tools", () => {
})
const events = Array.from(
yield* LLMClient.stream({ request: baseRequest, tools: { screenshot } }).pipe(
yield* TestToolRuntime.runTools({ request: baseRequest, tools: { screenshot }, maxSteps: 1 }).pipe(
Stream.runCollect,
Effect.provide(
scriptedResponses([sseEvents(toolCallChunk("call_1", "screenshot", "{}"), finishChunk("tool_calls"))]),
@ -179,6 +331,32 @@ describe("LLMClient tools", () => {
}),
)
it.effect("does not mistake dynamic tool output fields for dispatcher state", () =>
Effect.gen(function* () {
const callerOwned = { type: "json" as const, value: { ok: true }, events: ["caller-owned"] }
const eventful = Tool.make({
description: "Return an events field.",
jsonSchema: { type: "object", properties: {} },
execute: () => Effect.succeed(callerOwned),
})
const dispatched = yield* ToolRuntime.dispatch(
{ eventful },
LLMEvent.toolCall({ id: "call_1", name: "eventful", input: {} }),
)
expect(dispatched.result).toEqual(callerOwned)
expect(dispatched.events).toEqual([
LLMEvent.toolResult({
id: "call_1",
name: "eventful",
result: callerOwned,
output: { structured: { ok: true }, content: [] },
}),
])
}),
)
it.effect("executes tool calls for one step without looping by default", () =>
Effect.gen(function* () {
const layer = scriptedResponses([
@ -187,7 +365,7 @@ describe("LLMClient tools", () => {
])
const events = Array.from(
yield* LLMClient.stream({ request: baseRequest, tools: { get_weather } }).pipe(
yield* TestToolRuntime.runTools({ request: baseRequest, tools: { get_weather }, maxSteps: 1 }).pipe(
Stream.runCollect,
Effect.provide(layer),
),
@ -201,7 +379,7 @@ describe("LLMClient tools", () => {
it.effect("passes tool call context to execute", () =>
Effect.gen(function* () {
let context: ToolExecuteContext | undefined
const contextual = tool({
const contextual = Tool.make({
description: "Capture tool context.",
parameters: Schema.Struct({ value: Schema.String }),
success: Schema.Struct({ ok: Schema.Boolean }),
@ -234,11 +412,9 @@ describe("LLMClient tools", () => {
])
const events = Array.from(
yield* LLMClient.stream({
request: baseRequest,
tools: { get_weather: schema_only_weather },
toolExecution: "none",
}).pipe(Stream.runCollect, Effect.provide(layer)),
yield* LLMClient.stream(
LLMRequest.update(baseRequest, { tools: toDefinitions({ get_weather: schema_only_weather }) }),
).pipe(Stream.runCollect, Effect.provide(layer)),
)
expect(events.find(LLMEvent.is.toolCall)).toMatchObject({ type: "tool-call", id: "call_1" })
@ -500,74 +676,6 @@ describe("LLMClient tools", () => {
}),
)
it.effect("emits one final finish with aggregate usage", () =>
Effect.gen(function* () {
let calls = 0
const events = Array.from(
yield* ToolRuntime.stream({
request: baseRequest,
tools: { get_weather },
stopWhen: ToolRuntime.stepCountIs(2),
stream: () =>
Stream.fromIterable<LLMEvent>(
calls++ === 0
? [
LLMEvent.stepStart({ index: 0 }),
LLMEvent.toolCall({ id: "call_1", name: "get_weather", input: { city: "Paris" } }),
LLMEvent.stepFinish({
index: 0,
reason: "tool-calls",
usage: { inputTokens: 1, outputTokens: 2, totalTokens: 3 },
}),
LLMEvent.finish({
reason: "tool-calls",
usage: { inputTokens: 1, outputTokens: 2, totalTokens: 3 },
}),
]
: [
LLMEvent.stepStart({ index: 0 }),
LLMEvent.textDelta({ id: "text_1", text: "Done." }),
LLMEvent.stepFinish({
index: 0,
reason: "stop",
usage: { inputTokens: 4, outputTokens: 5, totalTokens: 9 },
}),
LLMEvent.finish({ reason: "stop", usage: { inputTokens: 4, outputTokens: 5, totalTokens: 9 } }),
],
),
}).pipe(Stream.runCollect),
)
expect(events.filter(LLMEvent.is.stepFinish).map((event) => event.index)).toEqual([0, 1])
expect(events.filter(LLMEvent.is.finish)).toHaveLength(1)
expect(events.find(LLMEvent.is.finish)?.usage).toMatchObject({
inputTokens: 5,
outputTokens: 7,
totalTokens: 12,
})
}),
)
it.effect("stops follow-up when stopWhen returns true after the first step", () =>
Effect.gen(function* () {
const layer = scriptedResponses([
sseEvents(toolCallChunk("call_1", "get_weather", '{"city":"Paris"}'), finishChunk("tool_calls")),
sseEvents(deltaChunk({ role: "assistant", content: "Should not run." }), finishChunk("stop")),
])
const events = Array.from(
yield* TestToolRuntime.runTools({
request: baseRequest,
tools: { get_weather },
stopWhen: (state) => state.step >= 0,
}).pipe(Stream.runCollect, Effect.provide(layer)),
)
expect(events.filter(LLMEvent.is.finish)).toHaveLength(1)
expect(events.find(LLMEvent.is.toolResult)).toMatchObject({ type: "tool-result", id: "call_1" })
}),
)
it.effect("does not dispatch provider-executed tool calls", () =>
Effect.gen(function* () {
let streams = 0

24
packages/llm/test/tool.types.ts
View file

@ -1,30 +1,38 @@
import { Effect, Schema } from "effect"
import { LLM } from "../src"
import { LLM, LLMRequest, ToolRuntime, toDefinitions } from "../src"
import * as OpenAIChat from "../src/protocols/openai-chat"
import { Auth } from "../src/route"
import { tool } from "../src/tool"
import { Tool } from "../src/tool"
const request = LLM.request({
model: OpenAIChat.route.with({ auth: Auth.bearer("fixture") }).model({ id: "gpt-4o-mini" }),
prompt: "Use the tool.",
})
const executable = tool({
const executable = Tool.make({
description: "Get weather.",
parameters: Schema.Struct({ city: Schema.String }),
success: Schema.Struct({ forecast: Schema.String }),
execute: (input) => Effect.succeed({ forecast: input.city }),
})
const schemaOnly = tool({
const schemaOnly = Tool.make({
description: "Get weather.",
parameters: Schema.Struct({ city: Schema.String }),
success: Schema.Struct({ forecast: Schema.String }),
})
LLM.stream({ request, tools: { executable } })
LLM.generate({ request, tools: { executable }, stopWhen: LLM.stepCountIs(2) })
LLM.stream({ request, tools: { schemaOnly }, toolExecution: "none" })
Tool.make({
description: "Encode success before projection.",
parameters: Schema.Struct({ city: Schema.String }),
success: Schema.Struct({ forecast: Schema.NumberFromString }),
execute: () => Effect.succeed({ forecast: 1 }),
toModelOutput: ({ callID, parameters, output }) => [{ type: "text", text: `${callID}:${parameters.city}:${output.forecast}` }],
})
// @ts-expect-error Handler-less tools can only be passed with toolExecution: "none".
LLM.stream(request)
LLM.generate(LLMRequest.update(request, { tools: toDefinitions({ schemaOnly }) }))
ToolRuntime.dispatch({ executable }, { type: "tool-call", id: "call_1", name: "executable", input: { city: "Paris" } })
// @ts-expect-error High-level tool orchestration overloads are intentionally not supported.
LLM.stream({ request, tools: { schemaOnly } })