test(core): remove legacy runner LLM harness

This commit is contained in:
Kit Langton 2026-07-07 19:48:28 -04:00
commit 68b23a80eb

View file

@ -1,6 +1,5 @@
import { describe, expect, test } from "bun:test" import { describe, expect, test } from "bun:test"
import { import {
LLMClient,
LLMError, LLMError,
LLMEvent, LLMEvent,
Model, Model,
@ -8,7 +7,7 @@ import {
TransportReason, TransportReason,
InvalidRequestReason, InvalidRequestReason,
RateLimitReason, RateLimitReason,
type LLMClientShape, type LLMClientService,
type LLMRequest, type LLMRequest,
} from "@opencode-ai/llm" } from "@opencode-ai/llm"
import * as OpenAIChat from "@opencode-ai/llm/protocols/openai-chat" import * as OpenAIChat from "@opencode-ai/llm/protocols/openai-chat"
@ -65,46 +64,14 @@ import { Cause, DateTime, Deferred, Effect, Exit, Fiber, Layer, Schema, Stream }
import type { Scope } from "effect/Scope" import type { Scope } from "effect/Scope"
import { TestClock } from "effect/testing" import { TestClock } from "effect/testing"
import { asc, eq } from "drizzle-orm" import { asc, eq } from "drizzle-orm"
import { it as effectIt, testEffect } from "./lib/effect" import { it as effectIt } from "./lib/effect"
import { RunnerScenario } from "./lib/runner-scenario" import { RunnerScenario } from "./lib/runner-scenario"
const requests: LLMRequest[] = []
let response: LLMEvent[] = []
let responses: LLMEvent[][] | undefined
let responseStream: Stream.Stream<LLMEvent, LLMError> | undefined
let streamGate: Deferred.Deferred<void> | undefined
let streamStarted: Deferred.Deferred<void> | undefined
let streamFailure: LLMError | undefined
let toolExecutionGate: Deferred.Deferred<void> | undefined let toolExecutionGate: Deferred.Deferred<void> | undefined
let toolExecutionsStarted: Deferred.Deferred<void> | undefined let toolExecutionsStarted: Deferred.Deferred<void> | undefined
let toolExecutionsReady = 5 let toolExecutionsReady = 5
let activeToolExecutions = 0 let activeToolExecutions = 0
let maxActiveToolExecutions = 0 let maxActiveToolExecutions = 0
const client = Layer.succeed(
LLMClient.Service,
LLMClient.Service.of({
prepare: () => Effect.die("unused"),
stream: ((request: LLMRequest) => {
requests.push(request)
if (responseStream) {
const stream = responseStream
responseStream = undefined
return stream
}
const events = streamFailure
? Stream.fail(streamFailure)
: Stream.fromIterable(responses === undefined ? response : (responses.shift() ?? []))
if (!streamGate) return events
return Stream.unwrap(
(streamStarted ? Deferred.succeed(streamStarted, undefined) : Effect.void).pipe(
Effect.andThen(Deferred.await(streamGate)),
Effect.as(events),
),
)
}) as unknown as LLMClientShape["stream"],
generate: () => Effect.die("unused"),
}),
)
const model = Model.make({ id: "fake-model", provider: "fake", route: OpenAIChat.route }) const model = Model.make({ id: "fake-model", provider: "fake", route: OpenAIChat.route })
const defaultSystem = SessionRunnerSystemPrompt.provider(model) const defaultSystem = SessionRunnerSystemPrompt.provider(model)
const replacementModel = Model.make({ id: "replacement", provider: "fake", route: OpenAIChat.route }) const replacementModel = Model.make({ id: "replacement", provider: "fake", route: OpenAIChat.route })
@ -269,7 +236,8 @@ const config = Layer.succeed(
]), ]),
}), }),
) )
const runnerLayerWith = (clientLayer: typeof client) => AppNodeBuilder.build(SessionRunnerLLM.node, [ const runnerLayerWith = (clientLayer: Layer.Layer<LLMClientService>) =>
AppNodeBuilder.build(SessionRunnerLLM.node, [
[Snapshot.node, Snapshot.noopLayer], [Snapshot.node, Snapshot.noopLayer],
[LayerNodePlatform.llmClient, clientLayer], [LayerNodePlatform.llmClient, clientLayer],
[SessionRunnerModel.node, models], [SessionRunnerModel.node, models],
@ -283,7 +251,8 @@ const runnerLayerWith = (clientLayer: typeof client) => AppNodeBuilder.build(Ses
[McpGuidance.node, mcpGuidance], [McpGuidance.node, mcpGuidance],
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
]) ])
const executionWith = (runnerLayer: ReturnType<typeof runnerLayerWith>) => Layer.effect( const executionWith = (runnerLayer: ReturnType<typeof runnerLayerWith>) =>
Layer.effect(
SessionExecution.Service, SessionExecution.Service,
Effect.gen(function* () { Effect.gen(function* () {
const sessionRunner = yield* SessionRunner.Service const sessionRunner = yield* SessionRunner.Service
@ -299,7 +268,7 @@ const executionWith = (runnerLayer: ReturnType<typeof runnerLayerWith>) => Layer
}) })
}), }),
).pipe(Layer.provide(runnerLayer)) ).pipe(Layer.provide(runnerLayer))
const testLayerWith = (clientLayer: typeof client) => { const testLayerWith = (clientLayer: Layer.Layer<LLMClientService>) => {
const runnerLayer = runnerLayerWith(clientLayer) const runnerLayer = runnerLayerWith(clientLayer)
const execution = executionWith(runnerLayer) const execution = executionWith(runnerLayer)
return AppNodeBuilder.build( return AppNodeBuilder.build(
@ -341,14 +310,12 @@ const testLayerWith = (clientLayer: typeof client) => {
], ],
) )
} }
const testLayer = testLayerWith(client)
const it = testEffect(testLayer)
const sessionID = SessionV2.ID.make("ses_runner_test") const sessionID = SessionV2.ID.make("ses_runner_test")
const otherSessionID = SessionV2.ID.make("ses_runner_other") const otherSessionID = SessionV2.ID.make("ses_runner_other")
const makeSessionScenario = () => const makeSessionScenario = () =>
RunnerScenario.make(() => SessionV2.Service.use((session) => session.resume(sessionID))) RunnerScenario.make(() => SessionV2.Service.use((session) => session.resume(sessionID)))
type SessionScenario = Effect.Success<ReturnType<typeof makeSessionScenario>> type SessionScenario = Effect.Success<ReturnType<typeof makeSessionScenario>>
type TestServices = Layer.Success<typeof testLayer> type TestServices = Layer.Success<ReturnType<typeof testLayerWith>>
const scenarioIt = <A, E>( const scenarioIt = <A, E>(
name: string, name: string,
body: (scenario: SessionScenario) => Effect.Effect<A, E, TestServices | Scope>, body: (scenario: SessionScenario) => Effect.Effect<A, E, TestServices | Scope>,
@ -380,7 +347,6 @@ const insertSession = (id: SessionV2.ID) =>
const setup = Effect.gen(function* () { const setup = Effect.gen(function* () {
const { db } = yield* Database.Service const { db } = yield* Database.Service
response = []
systemBaseline = "Initial context" systemBaseline = "Initial context"
systemRemoved = false systemRemoved = false
systemUnavailable = false systemUnavailable = false
@ -388,11 +354,6 @@ const setup = Effect.gen(function* () {
modelResolveHook = Effect.void modelResolveHook = Effect.void
currentModel = model currentModel = model
skillBaselines.clear() skillBaselines.clear()
responses = undefined
streamFailure = undefined
responseStream = undefined
streamGate = undefined
streamStarted = undefined
toolExecutionGate = undefined toolExecutionGate = undefined
toolExecutionsStarted = undefined toolExecutionsStarted = undefined
toolExecutionsReady = 5 toolExecutionsReady = 5
@ -591,97 +552,8 @@ const fragmentFixture = (kind: FragmentKind, id: string, chunks: readonly string
} }
} }
const verifyEphemeralDeltas = (kind: FragmentKind) =>
Effect.gen(function* () {
yield* setup
requests.length = 0
const session = yield* SessionV2.Service
const prompt = `Stream ${kind}`
const chunks = Array.from({ length: 32 }, (_, index) => `${index},`)
const fixture = fragmentFixture(kind, fragmentID(kind, "many"), chunks)
const expectedContext = [{ type: "user", text: prompt }, fixture.expectedAssistant]
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: prompt }), resume: false })
const events = yield* EventV2.Service
const live = yield* events.subscribe(fixture.delta).pipe(Stream.take(32), Stream.runCollect, Effect.forkScoped)
yield* Effect.yieldNow
response = fixture.completeEvents
yield* session.resume(sessionID)
const { db } = yield* Database.Service
const deltas = yield* db
.select({ type: EventTable.type })
.from(EventTable)
.where(eq(EventTable.type, EventV2.versionedType(fixture.delta.type, 1)))
.all()
.pipe(Effect.orDie)
expect(Array.from(yield* Fiber.join(live))).toHaveLength(32)
expect(deltas).toHaveLength(0)
expect(yield* session.context(sessionID)).toMatchObject(expectedContext)
yield* replaySessionProjection(sessionID)
expect(yield* session.context(sessionID)).toMatchObject(expectedContext)
})
const verifyPartialFlushOnFailure = (kind: FragmentKind) =>
Effect.gen(function* () {
yield* setup
requests.length = 0
const session = yield* SessionV2.Service
const prompt = `Fail after ${kind}`
const fixture = fragmentFixture(kind, fragmentID(kind, "partial"), ["Partial"])
const failure = providerUnavailable()
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: prompt }), resume: false })
responseStream = Stream.concat(Stream.fromIterable(fixture.partialEvents), Stream.fail(failure))
expect(yield* session.resume(sessionID).pipe(Effect.flip)).toBe(failure)
expect(yield* session.context(sessionID)).toMatchObject([
{ type: "user", text: prompt },
{
type: "assistant",
finish: "error",
error: { type: "provider.transport", message: "Provider unavailable" },
content: [fixture.expectedContent],
},
])
expect(requests).toHaveLength(1)
})
const verifyPartialFlushOnInterruption = (kind: FragmentKind) =>
Effect.gen(function* () {
yield* setup
const session = yield* SessionV2.Service
const prompt = `Interrupt after ${kind}`
const fixture = fragmentFixture(kind, fragmentID(kind, "interrupted"), ["Partial"])
const streamed = yield* Deferred.make<void>()
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: prompt }), resume: false })
responseStream = Stream.concat(
Stream.fromIterable(fixture.partialEvents),
Stream.fromEffect(Deferred.succeed(streamed, undefined)).pipe(Stream.flatMap(() => Stream.never)),
)
const runner = yield* SessionRunner.Service
const fiber = yield* runner.drain({ sessionID, force: true }).pipe(Effect.forkChild)
yield* Deferred.await(streamed)
yield* Fiber.interrupt(fiber)
expect(yield* session.context(sessionID)).toMatchObject([
{ type: "user", text: prompt },
{
type: "assistant",
finish: "error",
error: { type: "aborted", message: "Step interrupted" },
content: [
kind === "tool input"
? { type: "tool", id: fragmentID(kind, "interrupted"), state: { status: "error" } }
: fixture.expectedContent,
],
},
])
})
describe("SessionRunnerLLM", () => { describe("SessionRunnerLLM", () => {
it.effect("advertises and executes a location registered tool", () => scenarioIt("advertises and executes a location registered tool", (scenario) =>
Effect.gen(function* () { Effect.gen(function* () {
yield* setup yield* setup
const registry = yield* ToolRegistry.Service const registry = yield* ToolRegistry.Service
@ -704,19 +576,12 @@ describe("SessionRunnerLLM", () => {
prompt: PromptInput.Prompt.make({ text: "Use application context" }), prompt: PromptInput.Prompt.make({ text: "Use application context" }),
resume: false, resume: false,
}) })
responses = [ yield* scenario.run(function* () {
[ const call = yield* scenario.llm.next()
LLMEvent.stepStart({ index: 0 }), expect(call.request.tools.map((tool) => tool.name)).toContain("location_context")
LLMEvent.toolCall({ id: "call-location", name: "location_context", input: { query: "hello" } }), yield* call.respond.toolCall("location_context", { query: "hello" }, { id: "call-location" })
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), yield* (yield* scenario.llm.next()).respond.events()
LLMEvent.finish({ reason: "tool-calls" }), })
],
[],
]
yield* session.resume(sessionID)
expect(requests[0]?.tools.map((tool) => tool.name)).toContain("location_context")
expect(contexts).toEqual([ expect(contexts).toEqual([
{ {
sessionID, sessionID,
@ -1157,10 +1022,7 @@ describe("SessionRunnerLLM", () => {
yield* scenario.run(function* () { yield* scenario.run(function* () {
const call = yield* scenario.llm.next() const call = yield* scenario.llm.next()
expect(call.request.system.map((part) => part.text)).toEqual([ expect(call.request.system.map((part) => part.text)).toEqual([defaultSystem, "Initial context\n\nBuild skills"])
defaultSystem,
"Initial context\n\nBuild skills",
])
yield* call.respond.events() yield* call.respond.events()
}) })
}), }),
@ -2114,10 +1976,7 @@ describe("SessionRunnerLLM", () => {
yield* scenario.run(function* () { yield* scenario.run(function* () {
const first = yield* scenario.llm.next() const first = yield* scenario.llm.next()
yield* first.respond.stream( yield* first.respond.stream(
Stream.concat( Stream.concat(initial, Stream.unwrap(Deferred.await(providerGate).pipe(Effect.as(final)))),
initial,
Stream.unwrap(Deferred.await(providerGate).pipe(Effect.as(final))),
),
) )
yield* Deferred.await(executionsStarted) yield* Deferred.await(executionsStarted)
@ -2405,8 +2264,16 @@ describe("SessionRunnerLLM", () => {
yield* scenario.run(function* () { yield* scenario.run(function* () {
const first = yield* scenario.llm.next() const first = yield* scenario.llm.next()
expect(userTexts(first.request)).toEqual(["Start working"]) expect(userTexts(first.request)).toEqual(["Start working"])
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Queue first" }), delivery: "queue" }) yield* session.prompt({
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Queue second" }), delivery: "queue" }) sessionID,
prompt: PromptInput.Prompt.make({ text: "Queue first" }),
delivery: "queue",
})
yield* session.prompt({
sessionID,
prompt: PromptInput.Prompt.make({ text: "Queue second" }),
delivery: "queue",
})
yield* first.respond.stop() yield* first.respond.stop()
const second = yield* scenario.llm.next() const second = yield* scenario.llm.next()
@ -2457,13 +2324,24 @@ describe("SessionRunnerLLM", () => {
yield* scenario.run(function* () { yield* scenario.run(function* () {
const first = yield* scenario.llm.next() const first = yield* scenario.llm.next()
expect(userTexts(first.request)).toEqual(["Start working"]) expect(userTexts(first.request)).toEqual(["Start working"])
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Queue first" }), delivery: "queue" }) yield* session.prompt({
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Queue second" }), delivery: "queue" }) sessionID,
prompt: PromptInput.Prompt.make({ text: "Queue first" }),
delivery: "queue",
})
yield* session.prompt({
sessionID,
prompt: PromptInput.Prompt.make({ text: "Queue second" }),
delivery: "queue",
})
yield* first.respond.stop() yield* first.respond.stop()
const second = yield* scenario.llm.next() const second = yield* scenario.llm.next()
expect(userTexts(second.request)).toEqual(["Start working", "Queue first"]) expect(userTexts(second.request)).toEqual(["Start working", "Queue first"])
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Steer before next queued input" }) }) yield* session.prompt({
sessionID,
prompt: PromptInput.Prompt.make({ text: "Steer before next queued input" }),
})
yield* session.prompt({ yield* session.prompt({
sessionID, sessionID,
prompt: PromptInput.Prompt.make({ text: "Also steer before next queued input" }), prompt: PromptInput.Prompt.make({ text: "Also steer before next queued input" }),
@ -2749,10 +2627,7 @@ describe("SessionRunnerLLM", () => {
const scenario = yield* RunnerScenario.make(() => const scenario = yield* RunnerScenario.make(() =>
Effect.gen(function* () { Effect.gen(function* () {
const session = yield* SessionV2.Service const session = yield* SessionV2.Service
yield* Deferred.succeed( yield* Deferred.succeed(rolledBack, yield* session.resume(sessionID).pipe(Effect.catchDefect(Effect.succeed)))
rolledBack,
yield* session.resume(sessionID).pipe(Effect.catchDefect(Effect.succeed)),
)
yield* Deferred.await(retry) yield* Deferred.await(retry)
const execution = yield* SessionExecution.Service const execution = yield* SessionExecution.Service
yield* execution.wake(sessionID) yield* execution.wake(sessionID)
@ -2808,8 +2683,17 @@ describe("SessionRunnerLLM", () => {
}), }),
) )
it.effect("runs different sessions concurrently", () => effectIt.effect("runs different sessions concurrently", () =>
Effect.gen(function* () { Effect.gen(function* () {
const scenario = yield* RunnerScenario.make(() =>
SessionV2.Service.use((session) =>
Effect.all([session.resume(sessionID), session.resume(otherSessionID)], {
concurrency: "unbounded",
discard: true,
}),
),
)
yield* Effect.gen(function* () {
yield* setup yield* setup
yield* insertSession(otherSessionID) yield* insertSession(otherSessionID)
const session = yield* SessionV2.Service const session = yield* SessionV2.Service
@ -2820,28 +2704,18 @@ describe("SessionRunnerLLM", () => {
resume: false, resume: false,
}) })
requests.length = 0 yield* scenario.run(function* () {
responses = undefined const first = yield* scenario.llm.next()
response = [] const second = yield* scenario.llm.next()
streamGate = yield* Deferred.make<void>() expect(
streamStarted = yield* Deferred.make<void>() [first, second].map((call) => call.request.providerOptions?.openai?.promptCacheKey).toSorted(),
).toEqual([sessionID, otherSessionID].toSorted())
yield* first.respond.events()
yield* second.respond.events()
})
const first = yield* session.resume(sessionID).pipe(Effect.forkChild) expect(yield* scenario.llm.requests).toHaveLength(2)
yield* Deferred.await(streamStarted) }).pipe(Effect.provide(testLayerWith(scenario.llm.layer)))
streamStarted = yield* Deferred.make<void>()
const second = yield* session.resume(otherSessionID).pipe(Effect.forkChild)
yield* Deferred.await(streamStarted)
expect(requests).toHaveLength(2)
expect(requests.map((request) => request.providerOptions?.openai?.promptCacheKey)).toEqual([
sessionID,
otherSessionID,
])
yield* Deferred.succeed(streamGate, undefined)
yield* Fiber.join(first)
yield* Fiber.join(second)
streamGate = undefined
streamStarted = undefined
}), }),
) )
@ -2875,9 +2749,7 @@ describe("SessionRunnerLLM", () => {
yield* (yield* scenario.llm.next()).respond.events() yield* (yield* scenario.llm.next()).respond.events()
}) })
const keys = (yield* scenario.llm.requests).map( const keys = (yield* scenario.llm.requests).map((request) => request.providerOptions?.openai?.promptCacheKey)
(request) => request.providerOptions?.openai?.promptCacheKey,
)
expect(keys).toEqual([longSessionID.slice(4), otherLongSessionID.slice(4)]) expect(keys).toEqual([longSessionID.slice(4), otherLongSessionID.slice(4)])
expect(keys.every((key) => typeof key === "string" && key.length === 64)).toBe(true) expect(keys.every((key) => typeof key === "string" && key.length === 64)).toBe(true)
expect(keys[0]).not.toBe(keys[1]) expect(keys[0]).not.toBe(keys[1])
@ -2888,7 +2760,8 @@ describe("SessionRunnerLLM", () => {
effectIt.effect("fans out one failed run and allows a later retry", () => effectIt.effect("fans out one failed run and allows a later retry", () =>
Effect.gen(function* () { Effect.gen(function* () {
const join = yield* Deferred.make<void>() const join = yield* Deferred.make<void>()
const joined = yield* Deferred.make< const joined =
yield* Deferred.make<
readonly [ readonly [
Exit.Exit<void, SessionRunner.RunError | SessionV2.NotFoundError>, Exit.Exit<void, SessionRunner.RunError | SessionV2.NotFoundError>,
Exit.Exit<void, SessionRunner.RunError | SessionV2.NotFoundError>, Exit.Exit<void, SessionRunner.RunError | SessionV2.NotFoundError>,
@ -3412,8 +3285,12 @@ describe("SessionRunnerLLM", () => {
}), }),
) )
it.effect("durably fails blocked local tools when interrupted while awaiting settlement", () => effectIt.effect("durably fails blocked local tools when interrupted while awaiting settlement", () =>
Effect.gen(function* () { Effect.gen(function* () {
const scenario = yield* RunnerScenario.make(() =>
SessionRunner.Service.use((runner) => runner.drain({ sessionID, force: true })),
)
yield* Effect.gen(function* () {
yield* setup yield* setup
const session = yield* SessionV2.Service const session = yield* SessionV2.Service
yield* session.prompt({ yield* session.prompt({
@ -3422,19 +3299,22 @@ describe("SessionRunnerLLM", () => {
resume: false, resume: false,
}) })
executions.length = 0 executions.length = 0
toolExecutionGate = yield* Deferred.make<void>() const executionGate = yield* Deferred.make<void>()
toolExecutionsStarted = yield* Deferred.make<void>() const executionsStarted = yield* Deferred.make<void>()
toolExecutionGate = executionGate
toolExecutionsStarted = executionsStarted
toolExecutionsReady = 1 toolExecutionsReady = 1
const executionsStarted = toolExecutionsStarted
response = [
LLMEvent.stepStart({ index: 0 }),
LLMEvent.toolCall({ id: "call-await-interrupt", name: "echo", input: { text: "blocked" } }),
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
LLMEvent.finish({ reason: "tool-calls" }),
]
const runner = yield* SessionRunner.Service const run = yield* scenario
const run = yield* runner.drain({ sessionID, force: true }).pipe(Effect.forkChild) .run(function* () {
yield* (yield* scenario.llm.next()).respond.toolCall(
"echo",
{ text: "blocked" },
{ id: "call-await-interrupt" },
)
yield* Effect.never
})
.pipe(Effect.forkChild)
yield* Deferred.await(executionsStarted) yield* Deferred.await(executionsStarted)
yield* Fiber.interrupt(run) yield* Fiber.interrupt(run)
toolExecutionGate = undefined toolExecutionGate = undefined
@ -3459,6 +3339,7 @@ describe("SessionRunnerLLM", () => {
const eventTypes = yield* recordedEventTypes(sessionID) const eventTypes = yield* recordedEventTypes(sessionID)
expect(eventTypes).toContain("session.step.failed.1") expect(eventTypes).toContain("session.step.failed.1")
expect(eventTypes).not.toContain("session.step.ended.1") expect(eventTypes).not.toContain("session.step.ended.1")
}).pipe(Effect.provide(testLayerWith(scenario.llm.layer)))
}), }),
) )
@ -3544,16 +3425,14 @@ describe("SessionRunnerLLM", () => {
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Fail durably" }), resume: false }) yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Fail durably" }), resume: false })
expect( expect(
( (yield* scenario
yield* scenario
.run(function* () { .run(function* () {
yield* (yield* scenario.llm.next()).respond.events( yield* (yield* scenario.llm.next()).respond.events(
LLMEvent.stepStart({ index: 0 }), LLMEvent.stepStart({ index: 0 }),
LLMEvent.providerError({ message: "Provider unavailable" }), LLMEvent.providerError({ message: "Provider unavailable" }),
) )
}) })
.pipe(Effect.flip) .pipe(Effect.flip)).message,
).message,
).toBe("Provider unavailable") ).toBe("Provider unavailable")
expect(yield* scenario.llm.requests).toHaveLength(1) expect(yield* scenario.llm.requests).toHaveLength(1)
@ -3571,15 +3450,13 @@ describe("SessionRunnerLLM", () => {
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Fail before step" }), resume: false }) yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Fail before step" }), resume: false })
expect( expect(
( (yield* scenario
yield* scenario
.run(function* () { .run(function* () {
yield* (yield* scenario.llm.next()).respond.events( yield* (yield* scenario.llm.next()).respond.events(
LLMEvent.providerError({ message: "Provider unavailable" }), LLMEvent.providerError({ message: "Provider unavailable" }),
) )
}) })
.pipe(Effect.flip) .pipe(Effect.flip)).message,
).message,
).toBe("Provider unavailable") ).toBe("Provider unavailable")
expect(yield* scenario.llm.requests).toHaveLength(1) expect(yield* scenario.llm.requests).toHaveLength(1)
@ -3646,8 +3523,7 @@ describe("SessionRunnerLLM", () => {
const started = (toolExecutionsStarted = yield* Deferred.make<void>()) const started = (toolExecutionsStarted = yield* Deferred.make<void>())
toolExecutionsReady = 1 toolExecutionsReady = 1
expect( expect(
( (yield* scenario
yield* scenario
.run(function* () { .run(function* () {
yield* (yield* scenario.llm.next()).respond.events( yield* (yield* scenario.llm.next()).respond.events(
LLMEvent.stepStart({ index: 0 }), LLMEvent.stepStart({ index: 0 }),
@ -3658,8 +3534,7 @@ describe("SessionRunnerLLM", () => {
yield* Deferred.await(started) yield* Deferred.await(started)
yield* Deferred.succeed(gate, undefined) yield* Deferred.succeed(gate, undefined)
}) })
.pipe(Effect.flip) .pipe(Effect.flip)).message,
).message,
).toBe("Provider blocked the response") ).toBe("Provider blocked the response")
toolExecutionGate = undefined toolExecutionGate = undefined
toolExecutionsStarted = undefined toolExecutionsStarted = undefined
@ -3689,8 +3564,7 @@ describe("SessionRunnerLLM", () => {
}) })
expect( expect(
( (yield* scenario
yield* scenario
.run(function* () { .run(function* () {
yield* (yield* scenario.llm.next()).respond.events( yield* (yield* scenario.llm.next()).respond.events(
LLMEvent.stepStart({ index: 0 }), LLMEvent.stepStart({ index: 0 }),
@ -3700,8 +3574,7 @@ describe("SessionRunnerLLM", () => {
LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" }), LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" }),
) )
}) })
.pipe(Effect.flip) .pipe(Effect.flip)).message,
).message,
).toBe("prompt too long") ).toBe("prompt too long")
expect(yield* scenario.llm.requests).toHaveLength(1) expect(yield* scenario.llm.requests).toHaveLength(1)
@ -3899,8 +3772,7 @@ describe("SessionRunnerLLM", () => {
const started = (toolExecutionsStarted = yield* Deferred.make<void>()) const started = (toolExecutionsStarted = yield* Deferred.make<void>())
toolExecutionsReady = 1 toolExecutionsReady = 1
expect( expect(
( (yield* scenario
yield* scenario
.run(function* () { .run(function* () {
yield* (yield* scenario.llm.next()).respond.events( yield* (yield* scenario.llm.next()).respond.events(
LLMEvent.stepStart({ index: 0 }), LLMEvent.stepStart({ index: 0 }),
@ -3910,8 +3782,7 @@ describe("SessionRunnerLLM", () => {
yield* Deferred.await(started) yield* Deferred.await(started)
yield* Deferred.succeed(gate, undefined) yield* Deferred.succeed(gate, undefined)
}) })
.pipe(Effect.flip) .pipe(Effect.flip)).message,
).message,
).toBe("Provider unavailable") ).toBe("Provider unavailable")
toolExecutionGate = undefined toolExecutionGate = undefined
toolExecutionsStarted = undefined toolExecutionsStarted = undefined
@ -3940,8 +3811,7 @@ describe("SessionRunnerLLM", () => {
}) })
expect( expect(
( (yield* scenario
yield* scenario
.run(function* () { .run(function* () {
yield* (yield* scenario.llm.next()).respond.events( yield* (yield* scenario.llm.next()).respond.events(
LLMEvent.stepStart({ index: 0 }), LLMEvent.stepStart({ index: 0 }),
@ -3954,8 +3824,7 @@ describe("SessionRunnerLLM", () => {
LLMEvent.providerError({ message: "Provider unavailable" }), LLMEvent.providerError({ message: "Provider unavailable" }),
) )
}) })
.pipe(Effect.flip) .pipe(Effect.flip)).message,
).message,
).toBe("Provider unavailable") ).toBe("Provider unavailable")
expect(yield* scenario.llm.requests).toHaveLength(1) expect(yield* scenario.llm.requests).toHaveLength(1)
@ -3988,8 +3857,7 @@ describe("SessionRunnerLLM", () => {
}) })
expect( expect(
( (yield* scenario
yield* scenario
.run(function* () { .run(function* () {
yield* (yield* scenario.llm.next()).respond.events( yield* (yield* scenario.llm.next()).respond.events(
LLMEvent.stepStart({ index: 0 }), LLMEvent.stepStart({ index: 0 }),
@ -3997,8 +3865,7 @@ describe("SessionRunnerLLM", () => {
LLMEvent.providerError({ message: "Provider unavailable" }), LLMEvent.providerError({ message: "Provider unavailable" }),
) )
}) })
.pipe(Effect.flip) .pipe(Effect.flip)).message,
).message,
).toBe("Provider unavailable") ).toBe("Provider unavailable")
const context = yield* session.context(sessionID) const context = yield* session.context(sessionID)
@ -4025,8 +3892,7 @@ describe("SessionRunnerLLM", () => {
}) })
expect( expect(
( (yield* scenario
yield* scenario
.run(function* () { .run(function* () {
yield* (yield* scenario.llm.next()).respond.events( yield* (yield* scenario.llm.next()).respond.events(
LLMEvent.stepStart({ index: 0 }), LLMEvent.stepStart({ index: 0 }),
@ -4038,8 +3904,7 @@ describe("SessionRunnerLLM", () => {
}), }),
) )
}) })
.pipe(Effect.flip) .pipe(Effect.flip)).message,
).message,
).toBe("Provider did not return a tool result") ).toBe("Provider did not return a tool result")
const assistant = requireAssistant(yield* session.context(sessionID)) const assistant = requireAssistant(yield* session.context(sessionID))
const events = yield* recordedStepSettlementEvents(sessionID, assistant.id) const events = yield* recordedStepSettlementEvents(sessionID, assistant.id)
@ -4408,7 +4273,12 @@ describe("SessionRunnerLLM", () => {
LLMEvent.toolInputStart({ id: "call-parsed", name: "web_search" }), LLMEvent.toolInputStart({ id: "call-parsed", name: "web_search" }),
LLMEvent.toolInputDelta({ id: "call-parsed", name: "web_search", text: '{"query":"hello"}' }), LLMEvent.toolInputDelta({ id: "call-parsed", name: "web_search", text: '{"query":"hello"}' }),
LLMEvent.toolInputEnd({ id: "call-parsed", name: "web_search" }), LLMEvent.toolInputEnd({ id: "call-parsed", name: "web_search" }),
LLMEvent.toolCall({ id: "call-parsed", name: "web_search", input: { query: "hello" }, providerExecuted: true }), LLMEvent.toolCall({
id: "call-parsed",
name: "web_search",
input: { query: "hello" },
providerExecuted: true,
}),
LLMEvent.stepFinish({ index: 0, reason: "stop" }), LLMEvent.stepFinish({ index: 0, reason: "stop" }),
LLMEvent.finish({ reason: "stop" }), LLMEvent.finish({ reason: "stop" }),
) )