ceos/opencode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

test(core): migrate runner tool scenarios

Browse source
This commit is contained in:
Kit Langton 2026-07-07 11:50:11 -04:00
commit 0125b73145
1 changed files with 245 additions and 228 deletions
Download patch file Download diff file Expand all files Collapse all files

475
packages/core/test/session-runner.test.ts
View file

@ -1743,65 +1743,62 @@ describe("SessionRunnerLLM", () => {
}), }),
) )
it.effect("projects reasoning and tool events without executing or continuing tools", () => scenarioIt("projects reasoning and tool events without executing or continuing tools", (scenario) =>
Effect.gen(function* () { Effect.gen(function* () {
yield* setup yield* setup
const session = yield* SessionV2.Service const session = yield* SessionV2.Service
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Use tools" }), resume: false }) yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Use tools" }), resume: false })
requests.length = 0 yield* scenario.run(function* () {
responses = undefined const call = yield* scenario.llm.next()
streamGate = undefined expect(call.request.tools.map((tool) => tool.name)).toEqual(["echo", "defect", "storefail"])
streamStarted = undefined yield* call.respond.events(
response = [ LLMEvent.stepStart({ index: 0 }),
LLMEvent.stepStart({ index: 0 }), LLMEvent.reasoningStart({ id: "reasoning-1" }),
LLMEvent.reasoningStart({ id: "reasoning-1" }), LLMEvent.reasoningDelta({ id: "reasoning-1", text: "Think" }),
LLMEvent.reasoningDelta({ id: "reasoning-1", text: "Think" }), LLMEvent.reasoningEnd({ id: "reasoning-1" }),
LLMEvent.reasoningEnd({ id: "reasoning-1" }), LLMEvent.toolInputStart({ id: "call-error", name: "write" }),
LLMEvent.toolInputStart({ id: "call-error", name: "write" }), LLMEvent.toolInputDelta({ id: "call-error", name: "write", text: '{"path":"README.md"}' }),
LLMEvent.toolInputDelta({ id: "call-error", name: "write", text: '{"path":"README.md"}' }), LLMEvent.toolInputEnd({ id: "call-error", name: "write" }),
LLMEvent.toolInputEnd({ id: "call-error", name: "write" }), LLMEvent.toolCall({ id: "call-error", name: "write", input: { path: "README.md" }, providerExecuted: true }),
LLMEvent.toolCall({ id: "call-error", name: "write", input: { path: "README.md" }, providerExecuted: true }), LLMEvent.toolError({ id: "call-error", name: "write", message: "Denied" }),
LLMEvent.toolError({ id: "call-error", name: "write", message: "Denied" }), LLMEvent.toolResult({ id: "call-error", name: "write", result: { type: "error", value: "Denied" } }),
LLMEvent.toolResult({ id: "call-error", name: "write", result: { type: "error", value: "Denied" } }), LLMEvent.toolCall({
LLMEvent.toolCall({ id: "call-provider",
id: "call-provider", name: "web_search",
name: "web_search", input: { query: "hello" },
input: { query: "hello" }, providerExecuted: true,
providerExecuted: true, providerMetadata: { fake: { source: "provider" } },
providerMetadata: { fake: { source: "provider" } }, }),
}), LLMEvent.toolResult({
LLMEvent.toolResult({ id: "call-provider",
id: "call-provider", name: "web_search",
name: "web_search", result: {
result: { type: "content",
type: "content", value: [
value: [ { type: "text", text: "Hello" },
{ type: "text", text: "Hello" }, { type: "file", uri: "data:image/png;base64,aGVsbG8=", mime: "image/png", name: "hello.png" },
{ type: "file", uri: "data:image/png;base64,aGVsbG8=", mime: "image/png", name: "hello.png" }, ],
], },
}, providerExecuted: true,
providerExecuted: true, providerMetadata: { fake: { source: "provider" } },
providerMetadata: { fake: { source: "provider" } }, }),
}), LLMEvent.stepFinish({
LLMEvent.stepFinish({ index: 0,
index: 0, reason: "tool-calls",
reason: "tool-calls", usage: {
usage: { inputTokens: 10,
inputTokens: 10, nonCachedInputTokens: 8,
nonCachedInputTokens: 8, outputTokens: 4,
outputTokens: 4, reasoningTokens: 1,
reasoningTokens: 1, cacheReadInputTokens: 2,
cacheReadInputTokens: 2, },
}, }),
}), LLMEvent.finish({ reason: "tool-calls" }),
LLMEvent.finish({ reason: "tool-calls" }), )
] })
yield* session.resume(sessionID) expect(yield* scenario.llm.requests).toHaveLength(1)
expect(requests).toHaveLength(1)
expect(requests[0]?.tools.map((tool) => tool.name)).toEqual(["echo", "defect", "storefail"])
expect(yield* session.context(sessionID)).toMatchObject([ expect(yield* session.context(sessionID)).toMatchObject([
{ type: "user", text: "Use tools" }, { type: "user", text: "Use tools" },
{ {
@ -1896,40 +1893,41 @@ describe("SessionRunnerLLM", () => {
}), }),
) )
it.effect("reloads a model switch before a tool-driven continuation turn", () => scenarioIt("reloads a model switch before a tool-driven continuation turn", (scenario) =>
Effect.gen(function* () { Effect.gen(function* () {
yield* setup yield* setup
const session = yield* SessionV2.Service const session = yield* SessionV2.Service
const events = yield* EventV2.Service const events = yield* EventV2.Service
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Echo this" }), resume: false }) yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Echo this" }), resume: false })
requests.length = 0 const executionGate = yield* Deferred.make<void>()
responses = [ const executionsStarted = yield* Deferred.make<void>()
[ toolExecutionGate = executionGate
LLMEvent.stepStart({ index: 0 }), toolExecutionsStarted = executionsStarted
LLMEvent.toolCall({ id: "call-echo", name: "echo", input: { text: "hello" } }), toolExecutionsReady = 1
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), yield* scenario.run(function* () {
LLMEvent.finish({ reason: "tool-calls" }), const first = yield* scenario.llm.next()
], yield* first.respond.toolCall("echo", { text: "hello" }, { id: "call-echo" })
[ yield* Deferred.await(executionsStarted)
yield* events.publish(SessionEvent.ModelSelected, {
sessionID,
model: { id: ModelV2.ID.make("replacement"), providerID: ProviderV2.ID.make("fake") },
})
systemBaseline = "Replacement context"
yield* Deferred.succeed(executionGate, undefined)
const second = yield* scenario.llm.next()
expect(second.request.model).toBe(replacementModel)
expect(second.request.system.map((part) => part.text)).toEqual([defaultSystem, "Initial context"])
expect(systemTexts(second.request)).toContain("Replacement context")
yield* second.respond.events(
LLMEvent.stepStart({ index: 0 }), LLMEvent.stepStart({ index: 0 }),
LLMEvent.stepFinish({ index: 0, reason: "stop" }), LLMEvent.stepFinish({ index: 0, reason: "stop" }),
LLMEvent.finish({ reason: "stop" }), LLMEvent.finish({ reason: "stop" }),
], )
]
toolExecutionGate = yield* Deferred.make<void>()
toolExecutionsStarted = yield* Deferred.make<void>()
toolExecutionsReady = 1
const run = yield* Effect.forkChild(session.resume(sessionID))
yield* Deferred.await(toolExecutionsStarted)
yield* events.publish(SessionEvent.ModelSelected, {
sessionID,
model: { id: ModelV2.ID.make("replacement"), providerID: ProviderV2.ID.make("fake") },
}) })
systemBaseline = "Replacement context"
yield* Deferred.succeed(toolExecutionGate, undefined)
yield* Fiber.join(run)
const requests = yield* scenario.llm.requests
expect(requests.map((request) => request.model)).toEqual([model, replacementModel]) expect(requests.map((request) => request.model)).toEqual([model, replacementModel])
expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([ expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([
[defaultSystem, "Initial context"], [defaultSystem, "Initial context"],
@ -1939,140 +1937,169 @@ describe("SessionRunnerLLM", () => {
}), }),
) )
it.effect("restores durable reasoning provider metadata in a second-turn request", () => scenarioIt("restores durable reasoning provider metadata in a second-turn request", (scenario) =>
Effect.gen(function* () { Effect.gen(function* () {
yield* setup yield* setup
const session = yield* SessionV2.Service const session = yield* SessionV2.Service
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Think first" }), resume: false }) yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Think first" }), resume: false })
requests.length = 0 const streamed = yield* Deferred.make<void>()
response = [ const release = yield* Deferred.make<void>()
LLMEvent.stepStart({ index: 0 }), yield* scenario.run(function* () {
LLMEvent.reasoningStart({ id: "reasoning-anthropic" }), const first = yield* scenario.llm.next()
LLMEvent.reasoningDelta({ id: "reasoning-anthropic", text: "Signed thought" }), yield* first.respond.stream(
LLMEvent.reasoningEnd({ Stream.concat(
id: "reasoning-anthropic", Stream.fromIterable([
providerMetadata: { fake: { signature: "sig_1" }, anthropic: { ignored: true } }, LLMEvent.stepStart({ index: 0 }),
}), LLMEvent.reasoningStart({ id: "reasoning-anthropic" }),
LLMEvent.reasoningStart({ LLMEvent.reasoningDelta({ id: "reasoning-anthropic", text: "Signed thought" }),
id: "reasoning-openai", LLMEvent.reasoningEnd({
providerMetadata: { id: "reasoning-anthropic",
fake: { itemId: "rs_1", reasoningEncryptedContent: null }, providerMetadata: { fake: { signature: "sig_1" }, anthropic: { ignored: true } },
openai: { ignored: true }, }),
LLMEvent.reasoningStart({
id: "reasoning-openai",
providerMetadata: {
fake: { itemId: "rs_1", reasoningEncryptedContent: null },
openai: { ignored: true },
},
}),
LLMEvent.reasoningDelta({ id: "reasoning-openai", text: "Encrypted thought" }),
LLMEvent.reasoningEnd({
id: "reasoning-openai",
providerMetadata: {
fake: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" },
openai: { ignored: true },
},
}),
LLMEvent.stepFinish({ index: 0, reason: "stop" }),
LLMEvent.finish({ reason: "stop" }),
]),
Stream.unwrap(
Deferred.succeed(streamed, undefined).pipe(
Effect.andThen(Deferred.await(release)),
Effect.as(Stream.empty),
),
),
),
)
yield* Deferred.await(streamed)
yield* replaySessionProjection(sessionID)
expect(yield* session.context(sessionID)).toMatchObject([
{ type: "user", text: "Think first" },
{
type: "assistant",
content: [
{ type: "reasoning", text: "Signed thought", state: { signature: "sig_1" } },
{
type: "reasoning",
text: "Encrypted thought",
state: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" },
},
],
}, },
}), ])
LLMEvent.reasoningDelta({ id: "reasoning-openai", text: "Encrypted thought" }),
LLMEvent.reasoningEnd({ yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Continue" }), resume: false })
id: "reasoning-openai", yield* Deferred.succeed(release, undefined)
providerMetadata: {
fake: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" }, const second = yield* scenario.llm.next()
openai: { ignored: true }, expect(second.request.messages[1]?.content).toEqual([
{ type: "reasoning", text: "Signed thought", providerMetadata: { fake: { signature: "sig_1" } } },
{
type: "reasoning",
text: "Encrypted thought",
providerMetadata: { fake: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } },
}, },
}), ])
LLMEvent.stepFinish({ index: 0, reason: "stop" }), yield* second.respond.events()
LLMEvent.finish({ reason: "stop" }), })
]
yield* session.resume(sessionID)
yield* replaySessionProjection(sessionID)
expect(yield* session.context(sessionID)).toMatchObject([
{ type: "user", text: "Think first" },
{
type: "assistant",
content: [
{ type: "reasoning", text: "Signed thought", state: { signature: "sig_1" } },
{
type: "reasoning",
text: "Encrypted thought",
state: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" },
},
],
},
])
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Continue" }), resume: false })
response = []
yield* session.resume(sessionID)
expect(requests[1]?.messages[1]?.content).toEqual([
{ type: "reasoning", text: "Signed thought", providerMetadata: { fake: { signature: "sig_1" } } },
{
type: "reasoning",
text: "Encrypted thought",
providerMetadata: { fake: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } },
},
])
}), }),
) )
it.effect("replays durable provider-executed tool results inline in a second-turn request", () => scenarioIt("replays durable provider-executed tool results inline in a second-turn request", (scenario) =>
Effect.gen(function* () { Effect.gen(function* () {
yield* setup yield* setup
const session = yield* SessionV2.Service const session = yield* SessionV2.Service
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Search first" }), resume: false }) yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Search first" }), resume: false })
requests.length = 0 const streamed = yield* Deferred.make<void>()
response = [ const release = yield* Deferred.make<void>()
LLMEvent.stepStart({ index: 0 }), yield* scenario.run(function* () {
LLMEvent.toolCall({ const first = yield* scenario.llm.next()
id: "hosted-search", yield* first.respond.stream(
name: "web_search", Stream.concat(
input: { query: "Effect" }, Stream.fromIterable([
providerExecuted: true, LLMEvent.stepStart({ index: 0 }),
providerMetadata: { fake: { itemId: "hosted-search" }, openai: { ignored: true } }, LLMEvent.toolCall({
}), id: "hosted-search",
LLMEvent.toolResult({ name: "web_search",
id: "hosted-search", input: { query: "Effect" },
name: "web_search", providerExecuted: true,
result: { type: "json", value: [{ title: "Effect" }] }, providerMetadata: { fake: { itemId: "hosted-search" }, openai: { ignored: true } },
providerExecuted: true, }),
providerMetadata: { fake: { blockType: "web_search_tool_result" }, anthropic: { ignored: true } }, LLMEvent.toolResult({
}), id: "hosted-search",
LLMEvent.stepFinish({ index: 0, reason: "stop" }), name: "web_search",
LLMEvent.finish({ reason: "stop" }), result: { type: "json", value: [{ title: "Effect" }] },
] providerExecuted: true,
yield* session.resume(sessionID) providerMetadata: { fake: { blockType: "web_search_tool_result" }, anthropic: { ignored: true } },
yield* replaySessionProjection(sessionID) }),
LLMEvent.stepFinish({ index: 0, reason: "stop" }),
LLMEvent.finish({ reason: "stop" }),
]),
Stream.unwrap(
Deferred.succeed(streamed, undefined).pipe(
Effect.andThen(Deferred.await(release)),
Effect.as(Stream.empty),
),
),
),
)
yield* Deferred.await(streamed)
yield* replaySessionProjection(sessionID)
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Continue" }), resume: false }) yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Continue" }), resume: false })
response = [] yield* Deferred.succeed(release, undefined)
yield* session.resume(sessionID)
expect(requests[1]?.messages.map((message) => message.role)).toEqual(["user", "assistant", "user"]) const second = yield* scenario.llm.next()
expect(requests[1]?.messages[1]?.content).toMatchObject([ expect(second.request.messages.map((message) => message.role)).toEqual(["user", "assistant", "user"])
{ expect(second.request.messages[1]?.content).toMatchObject([
type: "tool-call", {
id: "hosted-search", type: "tool-call",
name: "web_search", id: "hosted-search",
input: { query: "Effect" }, name: "web_search",
providerExecuted: true, input: { query: "Effect" },
providerMetadata: { fake: { itemId: "hosted-search" } }, providerExecuted: true,
}, providerMetadata: { fake: { itemId: "hosted-search" } },
{ },
type: "tool-result", {
id: "hosted-search", type: "tool-result",
name: "web_search", id: "hosted-search",
result: { type: "json", value: [{ title: "Effect" }] }, name: "web_search",
providerExecuted: true, result: { type: "json", value: [{ title: "Effect" }] },
providerMetadata: { fake: { blockType: "web_search_tool_result" } }, providerExecuted: true,
}, providerMetadata: { fake: { blockType: "web_search_tool_result" } },
]) },
])
yield* second.respond.events()
})
}), }),
) )
it.effect("starts recorded local tools eagerly and awaits settlement before continuing", () => scenarioIt("starts recorded local tools eagerly and awaits settlement before continuing", (scenario) =>
Effect.gen(function* () { Effect.gen(function* () {
yield* setup yield* setup
const session = yield* SessionV2.Service const session = yield* SessionV2.Service
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Echo five times" }), resume: false }) yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Echo five times" }), resume: false })
requests.length = 0
executions.length = 0 executions.length = 0
toolExecutionGate = yield* Deferred.make<void>() const executionGate = yield* Deferred.make<void>()
toolExecutionsStarted = yield* Deferred.make<void>() const executionsStarted = yield* Deferred.make<void>()
toolExecutionGate = executionGate
toolExecutionsStarted = executionsStarted
const providerGate = yield* Deferred.make<void>() const providerGate = yield* Deferred.make<void>()
response = []
responses = undefined
const initial = Stream.fromIterable([ const initial = Stream.fromIterable([
LLMEvent.stepStart({ index: 0 }), LLMEvent.stepStart({ index: 0 }),
...Array.from({ length: 5 }, (_, index) => ...Array.from({ length: 5 }, (_, index) =>
@ -2083,72 +2110,62 @@ describe("SessionRunnerLLM", () => {
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
LLMEvent.finish({ reason: "tool-calls" }), LLMEvent.finish({ reason: "tool-calls" }),
]) ])
streamGate = undefined
responseStream = Stream.concat(
initial,
Stream.fromEffect(Deferred.await(providerGate)).pipe(Stream.flatMap(() => final)),
)
const run = yield* session.resume(sessionID).pipe(Effect.forkChild) yield* scenario.run(function* () {
yield* Deferred.await(toolExecutionsStarted) const first = yield* scenario.llm.next()
yield* first.respond.stream(
Stream.concat(
initial,
Stream.unwrap(Deferred.await(providerGate).pipe(Effect.as(final))),
),
)
yield* Deferred.await(executionsStarted)
expect(executions).toHaveLength(5) expect(executions).toHaveLength(5)
expect(maxActiveToolExecutions).toBe(5) expect(maxActiveToolExecutions).toBe(5)
expect(yield* session.context(sessionID)).toMatchObject([ expect(yield* session.context(sessionID)).toMatchObject([
{ type: "user", text: "Echo five times" }, { type: "user", text: "Echo five times" },
{ {
type: "assistant", type: "assistant",
content: Array.from({ length: 5 }, (_, index) => ({ content: Array.from({ length: 5 }, (_, index) => ({
type: "tool", type: "tool",
id: `call-echo-${index}`, id: `call-echo-${index}`,
state: { status: "running", input: { text: `${index}` } }, state: { status: "running", input: { text: `${index}` } },
})), })),
}, },
]) ])
yield* Deferred.succeed(providerGate, undefined) yield* Deferred.succeed(providerGate, undefined)
yield* Effect.yieldNow yield* Effect.yieldNow
expect(requests).toHaveLength(1) expect(yield* scenario.llm.requests).toHaveLength(1)
yield* Deferred.succeed(toolExecutionGate, undefined) yield* Deferred.succeed(executionGate, undefined)
yield* Fiber.join(run) yield* (yield* scenario.llm.next()).respond.events()
})
toolExecutionGate = undefined toolExecutionGate = undefined
toolExecutionsStarted = undefined toolExecutionsStarted = undefined
expect(executions).toHaveLength(5) expect(executions).toHaveLength(5)
expect(maxActiveToolExecutions).toBe(5) expect(maxActiveToolExecutions).toBe(5)
expect(requests).toHaveLength(2) expect(yield* scenario.llm.requests).toHaveLength(2)
}), }),
) )
it.effect("settles repeated provider-local tool call IDs against their owning assistant messages", () => scenarioIt("settles repeated provider-local tool call IDs against their owning assistant messages", (scenario) =>
Effect.gen(function* () { Effect.gen(function* () {
yield* setup yield* setup
const session = yield* SessionV2.Service const session = yield* SessionV2.Service
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Echo twice" }), resume: false }) yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Echo twice" }), resume: false })
requests.length = 0
executions.length = 0 executions.length = 0
responses = [ yield* scenario.run(function* () {
[ yield* (yield* scenario.llm.next()).respond.toolCall("echo", { text: "first" }, { id: "tool_0" })
LLMEvent.stepStart({ index: 0 }), yield* (yield* scenario.llm.next()).respond.toolCall("echo", { text: "second" }, { id: "tool_0" })
LLMEvent.toolCall({ id: "tool_0", name: "echo", input: { text: "first" } }), yield* (yield* scenario.llm.next()).respond.events()
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), })
LLMEvent.finish({ reason: "tool-calls" }),
],
[
LLMEvent.stepStart({ index: 0 }),
LLMEvent.toolCall({ id: "tool_0", name: "echo", input: { text: "second" } }),
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
LLMEvent.finish({ reason: "tool-calls" }),
],
[],
]
yield* session.resume(sessionID)
expect(executions).toEqual(["first", "second"]) expect(executions).toEqual(["first", "second"])
expect(requests).toHaveLength(3) expect(yield* scenario.llm.requests).toHaveLength(3)
expect(yield* session.context(sessionID)).toMatchObject([ expect(yield* session.context(sessionID)).toMatchObject([
{ type: "user", text: "Echo twice" }, { type: "user", text: "Echo twice" },
{ {