test(core): migrate runner tool scenarios

This commit is contained in:
Kit Langton 2026-07-07 11:50:11 -04:00
commit 0125b73145

View file

@ -1743,17 +1743,16 @@ describe("SessionRunnerLLM", () => {
}), }),
) )
it.effect("projects reasoning and tool events without executing or continuing tools", () => scenarioIt("projects reasoning and tool events without executing or continuing tools", (scenario) =>
Effect.gen(function* () { Effect.gen(function* () {
yield* setup yield* setup
const session = yield* SessionV2.Service const session = yield* SessionV2.Service
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Use tools" }), resume: false }) yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Use tools" }), resume: false })
requests.length = 0 yield* scenario.run(function* () {
responses = undefined const call = yield* scenario.llm.next()
streamGate = undefined expect(call.request.tools.map((tool) => tool.name)).toEqual(["echo", "defect", "storefail"])
streamStarted = undefined yield* call.respond.events(
response = [
LLMEvent.stepStart({ index: 0 }), LLMEvent.stepStart({ index: 0 }),
LLMEvent.reasoningStart({ id: "reasoning-1" }), LLMEvent.reasoningStart({ id: "reasoning-1" }),
LLMEvent.reasoningDelta({ id: "reasoning-1", text: "Think" }), LLMEvent.reasoningDelta({ id: "reasoning-1", text: "Think" }),
@ -1796,12 +1795,10 @@ describe("SessionRunnerLLM", () => {
}, },
}), }),
LLMEvent.finish({ reason: "tool-calls" }), LLMEvent.finish({ reason: "tool-calls" }),
] )
})
yield* session.resume(sessionID) expect(yield* scenario.llm.requests).toHaveLength(1)
expect(requests).toHaveLength(1)
expect(requests[0]?.tools.map((tool) => tool.name)).toEqual(["echo", "defect", "storefail"])
expect(yield* session.context(sessionID)).toMatchObject([ expect(yield* session.context(sessionID)).toMatchObject([
{ type: "user", text: "Use tools" }, { type: "user", text: "Use tools" },
{ {
@ -1896,40 +1893,41 @@ describe("SessionRunnerLLM", () => {
}), }),
) )
it.effect("reloads a model switch before a tool-driven continuation turn", () => scenarioIt("reloads a model switch before a tool-driven continuation turn", (scenario) =>
Effect.gen(function* () { Effect.gen(function* () {
yield* setup yield* setup
const session = yield* SessionV2.Service const session = yield* SessionV2.Service
const events = yield* EventV2.Service const events = yield* EventV2.Service
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Echo this" }), resume: false }) yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Echo this" }), resume: false })
requests.length = 0 const executionGate = yield* Deferred.make<void>()
responses = [ const executionsStarted = yield* Deferred.make<void>()
[ toolExecutionGate = executionGate
LLMEvent.stepStart({ index: 0 }), toolExecutionsStarted = executionsStarted
LLMEvent.toolCall({ id: "call-echo", name: "echo", input: { text: "hello" } }),
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
LLMEvent.finish({ reason: "tool-calls" }),
],
[
LLMEvent.stepStart({ index: 0 }),
LLMEvent.stepFinish({ index: 0, reason: "stop" }),
LLMEvent.finish({ reason: "stop" }),
],
]
toolExecutionGate = yield* Deferred.make<void>()
toolExecutionsStarted = yield* Deferred.make<void>()
toolExecutionsReady = 1 toolExecutionsReady = 1
const run = yield* Effect.forkChild(session.resume(sessionID)) yield* scenario.run(function* () {
yield* Deferred.await(toolExecutionsStarted) const first = yield* scenario.llm.next()
yield* first.respond.toolCall("echo", { text: "hello" }, { id: "call-echo" })
yield* Deferred.await(executionsStarted)
yield* events.publish(SessionEvent.ModelSelected, { yield* events.publish(SessionEvent.ModelSelected, {
sessionID, sessionID,
model: { id: ModelV2.ID.make("replacement"), providerID: ProviderV2.ID.make("fake") }, model: { id: ModelV2.ID.make("replacement"), providerID: ProviderV2.ID.make("fake") },
}) })
systemBaseline = "Replacement context" systemBaseline = "Replacement context"
yield* Deferred.succeed(toolExecutionGate, undefined) yield* Deferred.succeed(executionGate, undefined)
yield* Fiber.join(run)
const second = yield* scenario.llm.next()
expect(second.request.model).toBe(replacementModel)
expect(second.request.system.map((part) => part.text)).toEqual([defaultSystem, "Initial context"])
expect(systemTexts(second.request)).toContain("Replacement context")
yield* second.respond.events(
LLMEvent.stepStart({ index: 0 }),
LLMEvent.stepFinish({ index: 0, reason: "stop" }),
LLMEvent.finish({ reason: "stop" }),
)
})
const requests = yield* scenario.llm.requests
expect(requests.map((request) => request.model)).toEqual([model, replacementModel]) expect(requests.map((request) => request.model)).toEqual([model, replacementModel])
expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([ expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([
[defaultSystem, "Initial context"], [defaultSystem, "Initial context"],
@ -1939,14 +1937,19 @@ describe("SessionRunnerLLM", () => {
}), }),
) )
it.effect("restores durable reasoning provider metadata in a second-turn request", () => scenarioIt("restores durable reasoning provider metadata in a second-turn request", (scenario) =>
Effect.gen(function* () { Effect.gen(function* () {
yield* setup yield* setup
const session = yield* SessionV2.Service const session = yield* SessionV2.Service
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Think first" }), resume: false }) yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Think first" }), resume: false })
requests.length = 0 const streamed = yield* Deferred.make<void>()
response = [ const release = yield* Deferred.make<void>()
yield* scenario.run(function* () {
const first = yield* scenario.llm.next()
yield* first.respond.stream(
Stream.concat(
Stream.fromIterable([
LLMEvent.stepStart({ index: 0 }), LLMEvent.stepStart({ index: 0 }),
LLMEvent.reasoningStart({ id: "reasoning-anthropic" }), LLMEvent.reasoningStart({ id: "reasoning-anthropic" }),
LLMEvent.reasoningDelta({ id: "reasoning-anthropic", text: "Signed thought" }), LLMEvent.reasoningDelta({ id: "reasoning-anthropic", text: "Signed thought" }),
@ -1971,8 +1974,16 @@ describe("SessionRunnerLLM", () => {
}), }),
LLMEvent.stepFinish({ index: 0, reason: "stop" }), LLMEvent.stepFinish({ index: 0, reason: "stop" }),
LLMEvent.finish({ reason: "stop" }), LLMEvent.finish({ reason: "stop" }),
] ]),
yield* session.resume(sessionID) Stream.unwrap(
Deferred.succeed(streamed, undefined).pipe(
Effect.andThen(Deferred.await(release)),
Effect.as(Stream.empty),
),
),
),
)
yield* Deferred.await(streamed)
yield* replaySessionProjection(sessionID) yield* replaySessionProjection(sessionID)
expect(yield* session.context(sessionID)).toMatchObject([ expect(yield* session.context(sessionID)).toMatchObject([
@ -1991,10 +2002,10 @@ describe("SessionRunnerLLM", () => {
]) ])
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Continue" }), resume: false }) yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Continue" }), resume: false })
response = [] yield* Deferred.succeed(release, undefined)
yield* session.resume(sessionID)
expect(requests[1]?.messages[1]?.content).toEqual([ const second = yield* scenario.llm.next()
expect(second.request.messages[1]?.content).toEqual([
{ type: "reasoning", text: "Signed thought", providerMetadata: { fake: { signature: "sig_1" } } }, { type: "reasoning", text: "Signed thought", providerMetadata: { fake: { signature: "sig_1" } } },
{ {
type: "reasoning", type: "reasoning",
@ -2002,17 +2013,24 @@ describe("SessionRunnerLLM", () => {
providerMetadata: { fake: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } }, providerMetadata: { fake: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } },
}, },
]) ])
yield* second.respond.events()
})
}), }),
) )
it.effect("replays durable provider-executed tool results inline in a second-turn request", () => scenarioIt("replays durable provider-executed tool results inline in a second-turn request", (scenario) =>
Effect.gen(function* () { Effect.gen(function* () {
yield* setup yield* setup
const session = yield* SessionV2.Service const session = yield* SessionV2.Service
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Search first" }), resume: false }) yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Search first" }), resume: false })
requests.length = 0 const streamed = yield* Deferred.make<void>()
response = [ const release = yield* Deferred.make<void>()
yield* scenario.run(function* () {
const first = yield* scenario.llm.next()
yield* first.respond.stream(
Stream.concat(
Stream.fromIterable([
LLMEvent.stepStart({ index: 0 }), LLMEvent.stepStart({ index: 0 }),
LLMEvent.toolCall({ LLMEvent.toolCall({
id: "hosted-search", id: "hosted-search",
@ -2030,16 +2048,24 @@ describe("SessionRunnerLLM", () => {
}), }),
LLMEvent.stepFinish({ index: 0, reason: "stop" }), LLMEvent.stepFinish({ index: 0, reason: "stop" }),
LLMEvent.finish({ reason: "stop" }), LLMEvent.finish({ reason: "stop" }),
] ]),
yield* session.resume(sessionID) Stream.unwrap(
Deferred.succeed(streamed, undefined).pipe(
Effect.andThen(Deferred.await(release)),
Effect.as(Stream.empty),
),
),
),
)
yield* Deferred.await(streamed)
yield* replaySessionProjection(sessionID) yield* replaySessionProjection(sessionID)
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Continue" }), resume: false }) yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Continue" }), resume: false })
response = [] yield* Deferred.succeed(release, undefined)
yield* session.resume(sessionID)
expect(requests[1]?.messages.map((message) => message.role)).toEqual(["user", "assistant", "user"]) const second = yield* scenario.llm.next()
expect(requests[1]?.messages[1]?.content).toMatchObject([ expect(second.request.messages.map((message) => message.role)).toEqual(["user", "assistant", "user"])
expect(second.request.messages[1]?.content).toMatchObject([
{ {
type: "tool-call", type: "tool-call",
id: "hosted-search", id: "hosted-search",
@ -2057,22 +2083,23 @@ describe("SessionRunnerLLM", () => {
providerMetadata: { fake: { blockType: "web_search_tool_result" } }, providerMetadata: { fake: { blockType: "web_search_tool_result" } },
}, },
]) ])
yield* second.respond.events()
})
}), }),
) )
it.effect("starts recorded local tools eagerly and awaits settlement before continuing", () => scenarioIt("starts recorded local tools eagerly and awaits settlement before continuing", (scenario) =>
Effect.gen(function* () { Effect.gen(function* () {
yield* setup yield* setup
const session = yield* SessionV2.Service const session = yield* SessionV2.Service
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Echo five times" }), resume: false }) yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Echo five times" }), resume: false })
requests.length = 0
executions.length = 0 executions.length = 0
toolExecutionGate = yield* Deferred.make<void>() const executionGate = yield* Deferred.make<void>()
toolExecutionsStarted = yield* Deferred.make<void>() const executionsStarted = yield* Deferred.make<void>()
toolExecutionGate = executionGate
toolExecutionsStarted = executionsStarted
const providerGate = yield* Deferred.make<void>() const providerGate = yield* Deferred.make<void>()
response = []
responses = undefined
const initial = Stream.fromIterable([ const initial = Stream.fromIterable([
LLMEvent.stepStart({ index: 0 }), LLMEvent.stepStart({ index: 0 }),
...Array.from({ length: 5 }, (_, index) => ...Array.from({ length: 5 }, (_, index) =>
@ -2083,14 +2110,16 @@ describe("SessionRunnerLLM", () => {
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
LLMEvent.finish({ reason: "tool-calls" }), LLMEvent.finish({ reason: "tool-calls" }),
]) ])
streamGate = undefined
responseStream = Stream.concat(
initial,
Stream.fromEffect(Deferred.await(providerGate)).pipe(Stream.flatMap(() => final)),
)
const run = yield* session.resume(sessionID).pipe(Effect.forkChild) yield* scenario.run(function* () {
yield* Deferred.await(toolExecutionsStarted) const first = yield* scenario.llm.next()
yield* first.respond.stream(
Stream.concat(
initial,
Stream.unwrap(Deferred.await(providerGate).pipe(Effect.as(final))),
),
)
yield* Deferred.await(executionsStarted)
expect(executions).toHaveLength(5) expect(executions).toHaveLength(5)
expect(maxActiveToolExecutions).toBe(5) expect(maxActiveToolExecutions).toBe(5)
@ -2108,47 +2137,35 @@ describe("SessionRunnerLLM", () => {
yield* Deferred.succeed(providerGate, undefined) yield* Deferred.succeed(providerGate, undefined)
yield* Effect.yieldNow yield* Effect.yieldNow
expect(requests).toHaveLength(1) expect(yield* scenario.llm.requests).toHaveLength(1)
yield* Deferred.succeed(toolExecutionGate, undefined) yield* Deferred.succeed(executionGate, undefined)
yield* Fiber.join(run) yield* (yield* scenario.llm.next()).respond.events()
})
toolExecutionGate = undefined toolExecutionGate = undefined
toolExecutionsStarted = undefined toolExecutionsStarted = undefined
expect(executions).toHaveLength(5) expect(executions).toHaveLength(5)
expect(maxActiveToolExecutions).toBe(5) expect(maxActiveToolExecutions).toBe(5)
expect(requests).toHaveLength(2) expect(yield* scenario.llm.requests).toHaveLength(2)
}), }),
) )
it.effect("settles repeated provider-local tool call IDs against their owning assistant messages", () => scenarioIt("settles repeated provider-local tool call IDs against their owning assistant messages", (scenario) =>
Effect.gen(function* () { Effect.gen(function* () {
yield* setup yield* setup
const session = yield* SessionV2.Service const session = yield* SessionV2.Service
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Echo twice" }), resume: false }) yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Echo twice" }), resume: false })
requests.length = 0
executions.length = 0 executions.length = 0
responses = [ yield* scenario.run(function* () {
[ yield* (yield* scenario.llm.next()).respond.toolCall("echo", { text: "first" }, { id: "tool_0" })
LLMEvent.stepStart({ index: 0 }), yield* (yield* scenario.llm.next()).respond.toolCall("echo", { text: "second" }, { id: "tool_0" })
LLMEvent.toolCall({ id: "tool_0", name: "echo", input: { text: "first" } }), yield* (yield* scenario.llm.next()).respond.events()
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), })
LLMEvent.finish({ reason: "tool-calls" }),
],
[
LLMEvent.stepStart({ index: 0 }),
LLMEvent.toolCall({ id: "tool_0", name: "echo", input: { text: "second" } }),
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
LLMEvent.finish({ reason: "tool-calls" }),
],
[],
]
yield* session.resume(sessionID)
expect(executions).toEqual(["first", "second"]) expect(executions).toEqual(["first", "second"])
expect(requests).toHaveLength(3) expect(yield* scenario.llm.requests).toHaveLength(3)
expect(yield* session.context(sessionID)).toMatchObject([ expect(yield* session.context(sessionID)).toMatchObject([
{ type: "user", text: "Echo twice" }, { type: "user", text: "Echo twice" },
{ {