ceos/opencode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

test(core): remove legacy runner LLM harness

Browse source
This commit is contained in:
Kit Langton 2026-07-07 19:48:28 -04:00
commit 68b23a80eb
1 changed files with 252 additions and 382 deletions
Download patch file Download diff file Expand all files Collapse all files

632
packages/core/test/session-runner.test.ts
View file

@ -1,6 +1,5 @@
import { describe, expect, test } from "bun:test" import { describe, expect, test } from "bun:test"
import { import {
LLMClient,
LLMError, LLMError,
LLMEvent, LLMEvent,
Model, Model,
@ -8,7 +7,7 @@ import {
TransportReason, TransportReason,
InvalidRequestReason, InvalidRequestReason,
RateLimitReason, RateLimitReason,
type LLMClientShape, type LLMClientService,
type LLMRequest, type LLMRequest,
} from "@opencode-ai/llm" } from "@opencode-ai/llm"
import * as OpenAIChat from "@opencode-ai/llm/protocols/openai-chat" import * as OpenAIChat from "@opencode-ai/llm/protocols/openai-chat"
@ -65,46 +64,14 @@ import { Cause, DateTime, Deferred, Effect, Exit, Fiber, Layer, Schema, Stream }
import type { Scope } from "effect/Scope" import type { Scope } from "effect/Scope"
import { TestClock } from "effect/testing" import { TestClock } from "effect/testing"
import { asc, eq } from "drizzle-orm" import { asc, eq } from "drizzle-orm"
import { it as effectIt, testEffect } from "./lib/effect" import { it as effectIt } from "./lib/effect"
import { RunnerScenario } from "./lib/runner-scenario" import { RunnerScenario } from "./lib/runner-scenario"
const requests: LLMRequest[] = []
let response: LLMEvent[] = []
let responses: LLMEvent[][] | undefined
let responseStream: Stream.Stream<LLMEvent, LLMError> | undefined
let streamGate: Deferred.Deferred<void> | undefined
let streamStarted: Deferred.Deferred<void> | undefined
let streamFailure: LLMError | undefined
let toolExecutionGate: Deferred.Deferred<void> | undefined let toolExecutionGate: Deferred.Deferred<void> | undefined
let toolExecutionsStarted: Deferred.Deferred<void> | undefined let toolExecutionsStarted: Deferred.Deferred<void> | undefined
let toolExecutionsReady = 5 let toolExecutionsReady = 5
let activeToolExecutions = 0 let activeToolExecutions = 0
let maxActiveToolExecutions = 0 let maxActiveToolExecutions = 0
const client = Layer.succeed(
LLMClient.Service,
LLMClient.Service.of({
prepare: () => Effect.die("unused"),
stream: ((request: LLMRequest) => {
requests.push(request)
if (responseStream) {
const stream = responseStream
responseStream = undefined
return stream
}
const events = streamFailure
? Stream.fail(streamFailure)
: Stream.fromIterable(responses === undefined ? response : (responses.shift() ?? []))
if (!streamGate) return events
return Stream.unwrap(
(streamStarted ? Deferred.succeed(streamStarted, undefined) : Effect.void).pipe(
Effect.andThen(Deferred.await(streamGate)),
Effect.as(events),
),
)
}) as unknown as LLMClientShape["stream"],
generate: () => Effect.die("unused"),
}),
)
const model = Model.make({ id: "fake-model", provider: "fake", route: OpenAIChat.route }) const model = Model.make({ id: "fake-model", provider: "fake", route: OpenAIChat.route })
const defaultSystem = SessionRunnerSystemPrompt.provider(model) const defaultSystem = SessionRunnerSystemPrompt.provider(model)
const replacementModel = Model.make({ id: "replacement", provider: "fake", route: OpenAIChat.route }) const replacementModel = Model.make({ id: "replacement", provider: "fake", route: OpenAIChat.route })
@ -269,37 +236,39 @@ const config = Layer.succeed(
]), ]),
}), }),
) )
const runnerLayerWith = (clientLayer: typeof client) => AppNodeBuilder.build(SessionRunnerLLM.node, [ const runnerLayerWith = (clientLayer: Layer.Layer<LLMClientService>) =>
[Snapshot.node, Snapshot.noopLayer], AppNodeBuilder.build(SessionRunnerLLM.node, [
[LayerNodePlatform.llmClient, clientLayer], [Snapshot.node, Snapshot.noopLayer],
[SessionRunnerModel.node, models], [LayerNodePlatform.llmClient, clientLayer],
[InstructionBuiltIns.node, systemContext], [SessionRunnerModel.node, models],
[InstructionDiscovery.node, instructionContext], [InstructionBuiltIns.node, systemContext],
[Location.node, Location.boundNode({ directory: AbsolutePath.make("/project") })], [InstructionDiscovery.node, instructionContext],
[SkillGuidance.node, skillGuidance], [Location.node, Location.boundNode({ directory: AbsolutePath.make("/project") })],
[ReferenceGuidance.node, referenceGuidance], [SkillGuidance.node, skillGuidance],
[PermissionV2.node, permission], [ReferenceGuidance.node, referenceGuidance],
[Config.node, config], [PermissionV2.node, permission],
[McpGuidance.node, mcpGuidance], [Config.node, config],
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], [McpGuidance.node, mcpGuidance],
]) [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
const executionWith = (runnerLayer: ReturnType<typeof runnerLayerWith>) => Layer.effect( ])
SessionExecution.Service, const executionWith = (runnerLayer: ReturnType<typeof runnerLayerWith>) =>
Effect.gen(function* () { Layer.effect(
const sessionRunner = yield* SessionRunner.Service SessionExecution.Service,
const coordinator = yield* SessionRunCoordinator.make<SessionV2.ID, SessionRunner.RunError>({ Effect.gen(function* () {
drain: (sessionID, force) => sessionRunner.drain({ sessionID, force }), const sessionRunner = yield* SessionRunner.Service
}) const coordinator = yield* SessionRunCoordinator.make<SessionV2.ID, SessionRunner.RunError>({
return SessionExecution.Service.of({ drain: (sessionID, force) => sessionRunner.drain({ sessionID, force }),
active: coordinator.active, })
resume: coordinator.run, return SessionExecution.Service.of({
wake: coordinator.wake, active: coordinator.active,
interrupt: coordinator.interrupt, resume: coordinator.run,
awaitIdle: coordinator.awaitIdle, wake: coordinator.wake,
}) interrupt: coordinator.interrupt,
}), awaitIdle: coordinator.awaitIdle,
).pipe(Layer.provide(runnerLayer)) })
const testLayerWith = (clientLayer: typeof client) => { }),
).pipe(Layer.provide(runnerLayer))
const testLayerWith = (clientLayer: Layer.Layer<LLMClientService>) => {
const runnerLayer = runnerLayerWith(clientLayer) const runnerLayer = runnerLayerWith(clientLayer)
const execution = executionWith(runnerLayer) const execution = executionWith(runnerLayer)
return AppNodeBuilder.build( return AppNodeBuilder.build(
@ -341,14 +310,12 @@ const testLayerWith = (clientLayer: typeof client) => {
], ],
) )
} }
const testLayer = testLayerWith(client)
const it = testEffect(testLayer)
const sessionID = SessionV2.ID.make("ses_runner_test") const sessionID = SessionV2.ID.make("ses_runner_test")
const otherSessionID = SessionV2.ID.make("ses_runner_other") const otherSessionID = SessionV2.ID.make("ses_runner_other")
const makeSessionScenario = () => const makeSessionScenario = () =>
RunnerScenario.make(() => SessionV2.Service.use((session) => session.resume(sessionID))) RunnerScenario.make(() => SessionV2.Service.use((session) => session.resume(sessionID)))
type SessionScenario = Effect.Success<ReturnType<typeof makeSessionScenario>> type SessionScenario = Effect.Success<ReturnType<typeof makeSessionScenario>>
type TestServices = Layer.Success<typeof testLayer> type TestServices = Layer.Success<ReturnType<typeof testLayerWith>>
const scenarioIt = <A, E>( const scenarioIt = <A, E>(
name: string, name: string,
body: (scenario: SessionScenario) => Effect.Effect<A, E, TestServices | Scope>, body: (scenario: SessionScenario) => Effect.Effect<A, E, TestServices | Scope>,
@ -380,7 +347,6 @@ const insertSession = (id: SessionV2.ID) =>
const setup = Effect.gen(function* () { const setup = Effect.gen(function* () {
const { db } = yield* Database.Service const { db } = yield* Database.Service
response = []
systemBaseline = "Initial context" systemBaseline = "Initial context"
systemRemoved = false systemRemoved = false
systemUnavailable = false systemUnavailable = false
@ -388,11 +354,6 @@ const setup = Effect.gen(function* () {
modelResolveHook = Effect.void modelResolveHook = Effect.void
currentModel = model currentModel = model
skillBaselines.clear() skillBaselines.clear()
responses = undefined
streamFailure = undefined
responseStream = undefined
streamGate = undefined
streamStarted = undefined
toolExecutionGate = undefined toolExecutionGate = undefined
toolExecutionsStarted = undefined toolExecutionsStarted = undefined
toolExecutionsReady = 5 toolExecutionsReady = 5
@ -591,97 +552,8 @@ const fragmentFixture = (kind: FragmentKind, id: string, chunks: readonly string
} }
} }
const verifyEphemeralDeltas = (kind: FragmentKind) =>
Effect.gen(function* () {
yield* setup
requests.length = 0
const session = yield* SessionV2.Service
const prompt = `Stream ${kind}`
const chunks = Array.from({ length: 32 }, (_, index) => `${index},`)
const fixture = fragmentFixture(kind, fragmentID(kind, "many"), chunks)
const expectedContext = [{ type: "user", text: prompt }, fixture.expectedAssistant]
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: prompt }), resume: false })
const events = yield* EventV2.Service
const live = yield* events.subscribe(fixture.delta).pipe(Stream.take(32), Stream.runCollect, Effect.forkScoped)
yield* Effect.yieldNow
response = fixture.completeEvents
yield* session.resume(sessionID)
const { db } = yield* Database.Service
const deltas = yield* db
.select({ type: EventTable.type })
.from(EventTable)
.where(eq(EventTable.type, EventV2.versionedType(fixture.delta.type, 1)))
.all()
.pipe(Effect.orDie)
expect(Array.from(yield* Fiber.join(live))).toHaveLength(32)
expect(deltas).toHaveLength(0)
expect(yield* session.context(sessionID)).toMatchObject(expectedContext)
yield* replaySessionProjection(sessionID)
expect(yield* session.context(sessionID)).toMatchObject(expectedContext)
})
const verifyPartialFlushOnFailure = (kind: FragmentKind) =>
Effect.gen(function* () {
yield* setup
requests.length = 0
const session = yield* SessionV2.Service
const prompt = `Fail after ${kind}`
const fixture = fragmentFixture(kind, fragmentID(kind, "partial"), ["Partial"])
const failure = providerUnavailable()
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: prompt }), resume: false })
responseStream = Stream.concat(Stream.fromIterable(fixture.partialEvents), Stream.fail(failure))
expect(yield* session.resume(sessionID).pipe(Effect.flip)).toBe(failure)
expect(yield* session.context(sessionID)).toMatchObject([
{ type: "user", text: prompt },
{
type: "assistant",
finish: "error",
error: { type: "provider.transport", message: "Provider unavailable" },
content: [fixture.expectedContent],
},
])
expect(requests).toHaveLength(1)
})
const verifyPartialFlushOnInterruption = (kind: FragmentKind) =>
Effect.gen(function* () {
yield* setup
const session = yield* SessionV2.Service
const prompt = `Interrupt after ${kind}`
const fixture = fragmentFixture(kind, fragmentID(kind, "interrupted"), ["Partial"])
const streamed = yield* Deferred.make<void>()
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: prompt }), resume: false })
responseStream = Stream.concat(
Stream.fromIterable(fixture.partialEvents),
Stream.fromEffect(Deferred.succeed(streamed, undefined)).pipe(Stream.flatMap(() => Stream.never)),
)
const runner = yield* SessionRunner.Service
const fiber = yield* runner.drain({ sessionID, force: true }).pipe(Effect.forkChild)
yield* Deferred.await(streamed)
yield* Fiber.interrupt(fiber)
expect(yield* session.context(sessionID)).toMatchObject([
{ type: "user", text: prompt },
{
type: "assistant",
finish: "error",
error: { type: "aborted", message: "Step interrupted" },
content: [
kind === "tool input"
? { type: "tool", id: fragmentID(kind, "interrupted"), state: { status: "error" } }
: fixture.expectedContent,
],
},
])
})
describe("SessionRunnerLLM", () => { describe("SessionRunnerLLM", () => {
it.effect("advertises and executes a location registered tool", () => scenarioIt("advertises and executes a location registered tool", (scenario) =>
Effect.gen(function* () { Effect.gen(function* () {
yield* setup yield* setup
const registry = yield* ToolRegistry.Service const registry = yield* ToolRegistry.Service
@ -704,19 +576,12 @@ describe("SessionRunnerLLM", () => {
prompt: PromptInput.Prompt.make({ text: "Use application context" }), prompt: PromptInput.Prompt.make({ text: "Use application context" }),
resume: false, resume: false,
}) })
responses = [ yield* scenario.run(function* () {
[ const call = yield* scenario.llm.next()
LLMEvent.stepStart({ index: 0 }), expect(call.request.tools.map((tool) => tool.name)).toContain("location_context")
LLMEvent.toolCall({ id: "call-location", name: "location_context", input: { query: "hello" } }), yield* call.respond.toolCall("location_context", { query: "hello" }, { id: "call-location" })
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), yield* (yield* scenario.llm.next()).respond.events()
LLMEvent.finish({ reason: "tool-calls" }), })
],
[],
]
yield* session.resume(sessionID)
expect(requests[0]?.tools.map((tool) => tool.name)).toContain("location_context")
expect(contexts).toEqual([ expect(contexts).toEqual([
{ {
sessionID, sessionID,
@ -1157,10 +1022,7 @@ describe("SessionRunnerLLM", () => {
yield* scenario.run(function* () { yield* scenario.run(function* () {
const call = yield* scenario.llm.next() const call = yield* scenario.llm.next()
expect(call.request.system.map((part) => part.text)).toEqual([ expect(call.request.system.map((part) => part.text)).toEqual([defaultSystem, "Initial context\n\nBuild skills"])
defaultSystem,
"Initial context\n\nBuild skills",
])
yield* call.respond.events() yield* call.respond.events()
}) })
}), }),
@ -2114,10 +1976,7 @@ describe("SessionRunnerLLM", () => {
yield* scenario.run(function* () { yield* scenario.run(function* () {
const first = yield* scenario.llm.next() const first = yield* scenario.llm.next()
yield* first.respond.stream( yield* first.respond.stream(
Stream.concat( Stream.concat(initial, Stream.unwrap(Deferred.await(providerGate).pipe(Effect.as(final)))),
initial,
Stream.unwrap(Deferred.await(providerGate).pipe(Effect.as(final))),
),
) )
yield* Deferred.await(executionsStarted) yield* Deferred.await(executionsStarted)
@ -2405,8 +2264,16 @@ describe("SessionRunnerLLM", () => {
yield* scenario.run(function* () { yield* scenario.run(function* () {
const first = yield* scenario.llm.next() const first = yield* scenario.llm.next()
expect(userTexts(first.request)).toEqual(["Start working"]) expect(userTexts(first.request)).toEqual(["Start working"])
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Queue first" }), delivery: "queue" }) yield* session.prompt({
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Queue second" }), delivery: "queue" }) sessionID,
prompt: PromptInput.Prompt.make({ text: "Queue first" }),
delivery: "queue",
})
yield* session.prompt({
sessionID,
prompt: PromptInput.Prompt.make({ text: "Queue second" }),
delivery: "queue",
})
yield* first.respond.stop() yield* first.respond.stop()
const second = yield* scenario.llm.next() const second = yield* scenario.llm.next()
@ -2457,13 +2324,24 @@ describe("SessionRunnerLLM", () => {
yield* scenario.run(function* () { yield* scenario.run(function* () {
const first = yield* scenario.llm.next() const first = yield* scenario.llm.next()
expect(userTexts(first.request)).toEqual(["Start working"]) expect(userTexts(first.request)).toEqual(["Start working"])
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Queue first" }), delivery: "queue" }) yield* session.prompt({
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Queue second" }), delivery: "queue" }) sessionID,
prompt: PromptInput.Prompt.make({ text: "Queue first" }),
delivery: "queue",
})
yield* session.prompt({
sessionID,
prompt: PromptInput.Prompt.make({ text: "Queue second" }),
delivery: "queue",
})
yield* first.respond.stop() yield* first.respond.stop()
const second = yield* scenario.llm.next() const second = yield* scenario.llm.next()
expect(userTexts(second.request)).toEqual(["Start working", "Queue first"]) expect(userTexts(second.request)).toEqual(["Start working", "Queue first"])
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Steer before next queued input" }) }) yield* session.prompt({
sessionID,
prompt: PromptInput.Prompt.make({ text: "Steer before next queued input" }),
})
yield* session.prompt({ yield* session.prompt({
sessionID, sessionID,
prompt: PromptInput.Prompt.make({ text: "Also steer before next queued input" }), prompt: PromptInput.Prompt.make({ text: "Also steer before next queued input" }),
@ -2749,10 +2627,7 @@ describe("SessionRunnerLLM", () => {
const scenario = yield* RunnerScenario.make(() => const scenario = yield* RunnerScenario.make(() =>
Effect.gen(function* () { Effect.gen(function* () {
const session = yield* SessionV2.Service const session = yield* SessionV2.Service
yield* Deferred.succeed( yield* Deferred.succeed(rolledBack, yield* session.resume(sessionID).pipe(Effect.catchDefect(Effect.succeed)))
rolledBack,
yield* session.resume(sessionID).pipe(Effect.catchDefect(Effect.succeed)),
)
yield* Deferred.await(retry) yield* Deferred.await(retry)
const execution = yield* SessionExecution.Service const execution = yield* SessionExecution.Service
yield* execution.wake(sessionID) yield* execution.wake(sessionID)
@ -2808,40 +2683,39 @@ describe("SessionRunnerLLM", () => {
}), }),
) )
it.effect("runs different sessions concurrently", () => effectIt.effect("runs different sessions concurrently", () =>
Effect.gen(function* () { Effect.gen(function* () {
yield* setup const scenario = yield* RunnerScenario.make(() =>
yield* insertSession(otherSessionID) SessionV2.Service.use((session) =>
const session = yield* SessionV2.Service Effect.all([session.resume(sessionID), session.resume(otherSessionID)], {
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Run first" }), resume: false }) concurrency: "unbounded",
yield* session.prompt({ discard: true,
sessionID: otherSessionID, }),
prompt: PromptInput.Prompt.make({ text: "Run second" }), ),
resume: false, )
}) yield* Effect.gen(function* () {
yield* setup
yield* insertSession(otherSessionID)
const session = yield* SessionV2.Service
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Run first" }), resume: false })
yield* session.prompt({
sessionID: otherSessionID,
prompt: PromptInput.Prompt.make({ text: "Run second" }),
resume: false,
})
requests.length = 0 yield* scenario.run(function* () {
responses = undefined const first = yield* scenario.llm.next()
response = [] const second = yield* scenario.llm.next()
streamGate = yield* Deferred.make<void>() expect(
streamStarted = yield* Deferred.make<void>() [first, second].map((call) => call.request.providerOptions?.openai?.promptCacheKey).toSorted(),
).toEqual([sessionID, otherSessionID].toSorted())
yield* first.respond.events()
yield* second.respond.events()
})
const first = yield* session.resume(sessionID).pipe(Effect.forkChild) expect(yield* scenario.llm.requests).toHaveLength(2)
yield* Deferred.await(streamStarted) }).pipe(Effect.provide(testLayerWith(scenario.llm.layer)))
streamStarted = yield* Deferred.make<void>()
const second = yield* session.resume(otherSessionID).pipe(Effect.forkChild)
yield* Deferred.await(streamStarted)
expect(requests).toHaveLength(2)
expect(requests.map((request) => request.providerOptions?.openai?.promptCacheKey)).toEqual([
sessionID,
otherSessionID,
])
yield* Deferred.succeed(streamGate, undefined)
yield* Fiber.join(first)
yield* Fiber.join(second)
streamGate = undefined
streamStarted = undefined
}), }),
) )
@ -2875,9 +2749,7 @@ describe("SessionRunnerLLM", () => {
yield* (yield* scenario.llm.next()).respond.events() yield* (yield* scenario.llm.next()).respond.events()
}) })
const keys = (yield* scenario.llm.requests).map( const keys = (yield* scenario.llm.requests).map((request) => request.providerOptions?.openai?.promptCacheKey)
(request) => request.providerOptions?.openai?.promptCacheKey,
)
expect(keys).toEqual([longSessionID.slice(4), otherLongSessionID.slice(4)]) expect(keys).toEqual([longSessionID.slice(4), otherLongSessionID.slice(4)])
expect(keys.every((key) => typeof key === "string" && key.length === 64)).toBe(true) expect(keys.every((key) => typeof key === "string" && key.length === 64)).toBe(true)
expect(keys[0]).not.toBe(keys[1]) expect(keys[0]).not.toBe(keys[1])
@ -2888,12 +2760,13 @@ describe("SessionRunnerLLM", () => {
effectIt.effect("fans out one failed run and allows a later retry", () => effectIt.effect("fans out one failed run and allows a later retry", () =>
Effect.gen(function* () { Effect.gen(function* () {
const join = yield* Deferred.make<void>() const join = yield* Deferred.make<void>()
const joined = yield* Deferred.make< const joined =
readonly [ yield* Deferred.make<
Exit.Exit<void, SessionRunner.RunError | SessionV2.NotFoundError>, readonly [
Exit.Exit<void, SessionRunner.RunError | SessionV2.NotFoundError>, Exit.Exit<void, SessionRunner.RunError | SessionV2.NotFoundError>,
] Exit.Exit<void, SessionRunner.RunError | SessionV2.NotFoundError>,
>() ]
>()
const retry = yield* Deferred.make<void>() const retry = yield* Deferred.make<void>()
const scenario = yield* RunnerScenario.make(() => const scenario = yield* RunnerScenario.make(() =>
SessionV2.Service.use((session) => SessionV2.Service.use((session) =>
@ -3412,53 +3285,61 @@ describe("SessionRunnerLLM", () => {
}), }),
) )
it.effect("durably fails blocked local tools when interrupted while awaiting settlement", () => effectIt.effect("durably fails blocked local tools when interrupted while awaiting settlement", () =>
Effect.gen(function* () { Effect.gen(function* () {
yield* setup const scenario = yield* RunnerScenario.make(() =>
const session = yield* SessionV2.Service SessionRunner.Service.use((runner) => runner.drain({ sessionID, force: true })),
yield* session.prompt({ )
sessionID, yield* Effect.gen(function* () {
prompt: PromptInput.Prompt.make({ text: "Interrupt tool settlement" }), yield* setup
resume: false, const session = yield* SessionV2.Service
}) yield* session.prompt({
executions.length = 0 sessionID,
toolExecutionGate = yield* Deferred.make<void>() prompt: PromptInput.Prompt.make({ text: "Interrupt tool settlement" }),
toolExecutionsStarted = yield* Deferred.make<void>() resume: false,
toolExecutionsReady = 1 })
const executionsStarted = toolExecutionsStarted executions.length = 0
response = [ const executionGate = yield* Deferred.make<void>()
LLMEvent.stepStart({ index: 0 }), const executionsStarted = yield* Deferred.make<void>()
LLMEvent.toolCall({ id: "call-await-interrupt", name: "echo", input: { text: "blocked" } }), toolExecutionGate = executionGate
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), toolExecutionsStarted = executionsStarted
LLMEvent.finish({ reason: "tool-calls" }), toolExecutionsReady = 1
]
const runner = yield* SessionRunner.Service const run = yield* scenario
const run = yield* runner.drain({ sessionID, force: true }).pipe(Effect.forkChild) .run(function* () {
yield* Deferred.await(executionsStarted) yield* (yield* scenario.llm.next()).respond.toolCall(
yield* Fiber.interrupt(run) "echo",
toolExecutionGate = undefined { text: "blocked" },
toolExecutionsStarted = undefined { id: "call-await-interrupt" },
)
yield* Effect.never
})
.pipe(Effect.forkChild)
yield* Deferred.await(executionsStarted)
yield* Fiber.interrupt(run)
toolExecutionGate = undefined
toolExecutionsStarted = undefined
expect(yield* Fiber.await(run)).toMatchObject({ _tag: "Failure" }) expect(yield* Fiber.await(run)).toMatchObject({ _tag: "Failure" })
expect(yield* session.context(sessionID)).toMatchObject([ expect(yield* session.context(sessionID)).toMatchObject([
{ type: "user", text: "Interrupt tool settlement" }, { type: "user", text: "Interrupt tool settlement" },
{ {
type: "assistant", type: "assistant",
finish: "error", finish: "error",
error: { type: "aborted", message: "Step interrupted" }, error: { type: "aborted", message: "Step interrupted" },
content: [ content: [
{ {
type: "tool", type: "tool",
id: "call-await-interrupt", id: "call-await-interrupt",
state: { status: "error", error: { type: "aborted", message: "Tool execution interrupted" } }, state: { status: "error", error: { type: "aborted", message: "Tool execution interrupted" } },
}, },
], ],
}, },
]) ])
const eventTypes = yield* recordedEventTypes(sessionID) const eventTypes = yield* recordedEventTypes(sessionID)
expect(eventTypes).toContain("session.step.failed.1") expect(eventTypes).toContain("session.step.failed.1")
expect(eventTypes).not.toContain("session.step.ended.1") expect(eventTypes).not.toContain("session.step.ended.1")
}).pipe(Effect.provide(testLayerWith(scenario.llm.layer)))
}), }),
) )
@ -3544,16 +3425,14 @@ describe("SessionRunnerLLM", () => {
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Fail durably" }), resume: false }) yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Fail durably" }), resume: false })
expect( expect(
( (yield* scenario
yield* scenario .run(function* () {
.run(function* () { yield* (yield* scenario.llm.next()).respond.events(
yield* (yield* scenario.llm.next()).respond.events( LLMEvent.stepStart({ index: 0 }),
LLMEvent.stepStart({ index: 0 }), LLMEvent.providerError({ message: "Provider unavailable" }),
LLMEvent.providerError({ message: "Provider unavailable" }), )
) })
}) .pipe(Effect.flip)).message,
.pipe(Effect.flip)
).message,
).toBe("Provider unavailable") ).toBe("Provider unavailable")
expect(yield* scenario.llm.requests).toHaveLength(1) expect(yield* scenario.llm.requests).toHaveLength(1)
@ -3571,15 +3450,13 @@ describe("SessionRunnerLLM", () => {
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Fail before step" }), resume: false }) yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Fail before step" }), resume: false })
expect( expect(
( (yield* scenario
yield* scenario .run(function* () {
.run(function* () { yield* (yield* scenario.llm.next()).respond.events(
yield* (yield* scenario.llm.next()).respond.events( LLMEvent.providerError({ message: "Provider unavailable" }),
LLMEvent.providerError({ message: "Provider unavailable" }), )
) })
}) .pipe(Effect.flip)).message,
.pipe(Effect.flip)
).message,
).toBe("Provider unavailable") ).toBe("Provider unavailable")
expect(yield* scenario.llm.requests).toHaveLength(1) expect(yield* scenario.llm.requests).toHaveLength(1)
@ -3646,20 +3523,18 @@ describe("SessionRunnerLLM", () => {
const started = (toolExecutionsStarted = yield* Deferred.make<void>()) const started = (toolExecutionsStarted = yield* Deferred.make<void>())
toolExecutionsReady = 1 toolExecutionsReady = 1
expect( expect(
( (yield* scenario
yield* scenario .run(function* () {
.run(function* () { yield* (yield* scenario.llm.next()).respond.events(
yield* (yield* scenario.llm.next()).respond.events( LLMEvent.stepStart({ index: 0 }),
LLMEvent.stepStart({ index: 0 }), LLMEvent.toolCall({ id: "call-before-content-filter", name: "echo", input: { text: "settled" } }),
LLMEvent.toolCall({ id: "call-before-content-filter", name: "echo", input: { text: "settled" } }), LLMEvent.stepFinish({ index: 0, reason: "content-filter" }),
LLMEvent.stepFinish({ index: 0, reason: "content-filter" }), LLMEvent.finish({ reason: "content-filter" }),
LLMEvent.finish({ reason: "content-filter" }), )
) yield* Deferred.await(started)
yield* Deferred.await(started) yield* Deferred.succeed(gate, undefined)
yield* Deferred.succeed(gate, undefined) })
}) .pipe(Effect.flip)).message,
.pipe(Effect.flip)
).message,
).toBe("Provider blocked the response") ).toBe("Provider blocked the response")
toolExecutionGate = undefined toolExecutionGate = undefined
toolExecutionsStarted = undefined toolExecutionsStarted = undefined
@ -3689,19 +3564,17 @@ describe("SessionRunnerLLM", () => {
}) })
expect( expect(
( (yield* scenario
yield* scenario .run(function* () {
.run(function* () { yield* (yield* scenario.llm.next()).respond.events(
yield* (yield* scenario.llm.next()).respond.events( LLMEvent.stepStart({ index: 0 }),
LLMEvent.stepStart({ index: 0 }), LLMEvent.textStart({ id: "text-partial" }),
LLMEvent.textStart({ id: "text-partial" }), LLMEvent.textDelta({ id: "text-partial", text: "Partial" }),
LLMEvent.textDelta({ id: "text-partial", text: "Partial" }), LLMEvent.textEnd({ id: "text-partial" }),
LLMEvent.textEnd({ id: "text-partial" }), LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" }),
LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" }), )
) })
}) .pipe(Effect.flip)).message,
.pipe(Effect.flip)
).message,
).toBe("prompt too long") ).toBe("prompt too long")
expect(yield* scenario.llm.requests).toHaveLength(1) expect(yield* scenario.llm.requests).toHaveLength(1)
@ -3899,19 +3772,17 @@ describe("SessionRunnerLLM", () => {
const started = (toolExecutionsStarted = yield* Deferred.make<void>()) const started = (toolExecutionsStarted = yield* Deferred.make<void>())
toolExecutionsReady = 1 toolExecutionsReady = 1
expect( expect(
( (yield* scenario
yield* scenario .run(function* () {
.run(function* () { yield* (yield* scenario.llm.next()).respond.events(
yield* (yield* scenario.llm.next()).respond.events( LLMEvent.stepStart({ index: 0 }),
LLMEvent.stepStart({ index: 0 }), LLMEvent.toolCall({ id: "call-before-provider-error", name: "echo", input: { text: "settled" } }),
LLMEvent.toolCall({ id: "call-before-provider-error", name: "echo", input: { text: "settled" } }), LLMEvent.providerError({ message: "Provider unavailable" }),
LLMEvent.providerError({ message: "Provider unavailable" }), )
) yield* Deferred.await(started)
yield* Deferred.await(started) yield* Deferred.succeed(gate, undefined)
yield* Deferred.succeed(gate, undefined) })
}) .pipe(Effect.flip)).message,
.pipe(Effect.flip)
).message,
).toBe("Provider unavailable") ).toBe("Provider unavailable")
toolExecutionGate = undefined toolExecutionGate = undefined
toolExecutionsStarted = undefined toolExecutionsStarted = undefined
@ -3940,22 +3811,20 @@ describe("SessionRunnerLLM", () => {
}) })
expect( expect(
( (yield* scenario
yield* scenario .run(function* () {
.run(function* () { yield* (yield* scenario.llm.next()).respond.events(
yield* (yield* scenario.llm.next()).respond.events( LLMEvent.stepStart({ index: 0 }),
LLMEvent.stepStart({ index: 0 }), LLMEvent.toolCall({
LLMEvent.toolCall({ id: "call-hosted-provider-error",
id: "call-hosted-provider-error", name: "web_search",
name: "web_search", input: { query: "effect" },
input: { query: "effect" }, providerExecuted: true,
providerExecuted: true, }),
}), LLMEvent.providerError({ message: "Provider unavailable" }),
LLMEvent.providerError({ message: "Provider unavailable" }), )
) })
}) .pipe(Effect.flip)).message,
.pipe(Effect.flip)
).message,
).toBe("Provider unavailable") ).toBe("Provider unavailable")
expect(yield* scenario.llm.requests).toHaveLength(1) expect(yield* scenario.llm.requests).toHaveLength(1)
@ -3988,17 +3857,15 @@ describe("SessionRunnerLLM", () => {
}) })
expect( expect(
( (yield* scenario
yield* scenario .run(function* () {
.run(function* () { yield* (yield* scenario.llm.next()).respond.events(
yield* (yield* scenario.llm.next()).respond.events( LLMEvent.stepStart({ index: 0 }),
LLMEvent.stepStart({ index: 0 }), LLMEvent.toolCall({ id: "call-defect-provider-error", name: "defect", input: {} }),
LLMEvent.toolCall({ id: "call-defect-provider-error", name: "defect", input: {} }), LLMEvent.providerError({ message: "Provider unavailable" }),
LLMEvent.providerError({ message: "Provider unavailable" }), )
) })
}) .pipe(Effect.flip)).message,
.pipe(Effect.flip)
).message,
).toBe("Provider unavailable") ).toBe("Provider unavailable")
const context = yield* session.context(sessionID) const context = yield* session.context(sessionID)
@ -4025,21 +3892,19 @@ describe("SessionRunnerLLM", () => {
}) })
expect( expect(
( (yield* scenario
yield* scenario .run(function* () {
.run(function* () { yield* (yield* scenario.llm.next()).respond.events(
yield* (yield* scenario.llm.next()).respond.events( LLMEvent.stepStart({ index: 0 }),
LLMEvent.stepStart({ index: 0 }), LLMEvent.toolCall({
LLMEvent.toolCall({ id: "call-hosted-eof",
id: "call-hosted-eof", name: "web_search",
name: "web_search", input: { query: "effect" },
input: { query: "effect" }, providerExecuted: true,
providerExecuted: true, }),
}), )
) })
}) .pipe(Effect.flip)).message,
.pipe(Effect.flip)
).message,
).toBe("Provider did not return a tool result") ).toBe("Provider did not return a tool result")
const assistant = requireAssistant(yield* session.context(sessionID)) const assistant = requireAssistant(yield* session.context(sessionID))
const events = yield* recordedStepSettlementEvents(sessionID, assistant.id) const events = yield* recordedStepSettlementEvents(sessionID, assistant.id)
@ -4408,7 +4273,12 @@ describe("SessionRunnerLLM", () => {
LLMEvent.toolInputStart({ id: "call-parsed", name: "web_search" }), LLMEvent.toolInputStart({ id: "call-parsed", name: "web_search" }),
LLMEvent.toolInputDelta({ id: "call-parsed", name: "web_search", text: '{"query":"hello"}' }), LLMEvent.toolInputDelta({ id: "call-parsed", name: "web_search", text: '{"query":"hello"}' }),
LLMEvent.toolInputEnd({ id: "call-parsed", name: "web_search" }), LLMEvent.toolInputEnd({ id: "call-parsed", name: "web_search" }),
LLMEvent.toolCall({ id: "call-parsed", name: "web_search", input: { query: "hello" }, providerExecuted: true }), LLMEvent.toolCall({
id: "call-parsed",
name: "web_search",
input: { query: "hello" },
providerExecuted: true,
}),
LLMEvent.stepFinish({ index: 0, reason: "stop" }), LLMEvent.stepFinish({ index: 0, reason: "stop" }),
LLMEvent.finish({ reason: "stop" }), LLMEvent.finish({ reason: "stop" }),
) )