test(core): migrate runner tool scenarios
This commit is contained in:
parent
397ad07c68
commit
0125b73145
1 changed files with 245 additions and 228 deletions
|
|
@ -1743,17 +1743,16 @@ describe("SessionRunnerLLM", () => {
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|
||||||
it.effect("projects reasoning and tool events without executing or continuing tools", () =>
|
scenarioIt("projects reasoning and tool events without executing or continuing tools", (scenario) =>
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
yield* setup
|
yield* setup
|
||||||
const session = yield* SessionV2.Service
|
const session = yield* SessionV2.Service
|
||||||
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Use tools" }), resume: false })
|
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Use tools" }), resume: false })
|
||||||
|
|
||||||
requests.length = 0
|
yield* scenario.run(function* () {
|
||||||
responses = undefined
|
const call = yield* scenario.llm.next()
|
||||||
streamGate = undefined
|
expect(call.request.tools.map((tool) => tool.name)).toEqual(["echo", "defect", "storefail"])
|
||||||
streamStarted = undefined
|
yield* call.respond.events(
|
||||||
response = [
|
|
||||||
LLMEvent.stepStart({ index: 0 }),
|
LLMEvent.stepStart({ index: 0 }),
|
||||||
LLMEvent.reasoningStart({ id: "reasoning-1" }),
|
LLMEvent.reasoningStart({ id: "reasoning-1" }),
|
||||||
LLMEvent.reasoningDelta({ id: "reasoning-1", text: "Think" }),
|
LLMEvent.reasoningDelta({ id: "reasoning-1", text: "Think" }),
|
||||||
|
|
@ -1796,12 +1795,10 @@ describe("SessionRunnerLLM", () => {
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
LLMEvent.finish({ reason: "tool-calls" }),
|
LLMEvent.finish({ reason: "tool-calls" }),
|
||||||
]
|
)
|
||||||
|
})
|
||||||
|
|
||||||
yield* session.resume(sessionID)
|
expect(yield* scenario.llm.requests).toHaveLength(1)
|
||||||
|
|
||||||
expect(requests).toHaveLength(1)
|
|
||||||
expect(requests[0]?.tools.map((tool) => tool.name)).toEqual(["echo", "defect", "storefail"])
|
|
||||||
expect(yield* session.context(sessionID)).toMatchObject([
|
expect(yield* session.context(sessionID)).toMatchObject([
|
||||||
{ type: "user", text: "Use tools" },
|
{ type: "user", text: "Use tools" },
|
||||||
{
|
{
|
||||||
|
|
@ -1896,40 +1893,41 @@ describe("SessionRunnerLLM", () => {
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|
||||||
it.effect("reloads a model switch before a tool-driven continuation turn", () =>
|
scenarioIt("reloads a model switch before a tool-driven continuation turn", (scenario) =>
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
yield* setup
|
yield* setup
|
||||||
const session = yield* SessionV2.Service
|
const session = yield* SessionV2.Service
|
||||||
const events = yield* EventV2.Service
|
const events = yield* EventV2.Service
|
||||||
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Echo this" }), resume: false })
|
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Echo this" }), resume: false })
|
||||||
|
|
||||||
requests.length = 0
|
const executionGate = yield* Deferred.make<void>()
|
||||||
responses = [
|
const executionsStarted = yield* Deferred.make<void>()
|
||||||
[
|
toolExecutionGate = executionGate
|
||||||
LLMEvent.stepStart({ index: 0 }),
|
toolExecutionsStarted = executionsStarted
|
||||||
LLMEvent.toolCall({ id: "call-echo", name: "echo", input: { text: "hello" } }),
|
|
||||||
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
|
|
||||||
LLMEvent.finish({ reason: "tool-calls" }),
|
|
||||||
],
|
|
||||||
[
|
|
||||||
LLMEvent.stepStart({ index: 0 }),
|
|
||||||
LLMEvent.stepFinish({ index: 0, reason: "stop" }),
|
|
||||||
LLMEvent.finish({ reason: "stop" }),
|
|
||||||
],
|
|
||||||
]
|
|
||||||
toolExecutionGate = yield* Deferred.make<void>()
|
|
||||||
toolExecutionsStarted = yield* Deferred.make<void>()
|
|
||||||
toolExecutionsReady = 1
|
toolExecutionsReady = 1
|
||||||
const run = yield* Effect.forkChild(session.resume(sessionID))
|
yield* scenario.run(function* () {
|
||||||
yield* Deferred.await(toolExecutionsStarted)
|
const first = yield* scenario.llm.next()
|
||||||
|
yield* first.respond.toolCall("echo", { text: "hello" }, { id: "call-echo" })
|
||||||
|
yield* Deferred.await(executionsStarted)
|
||||||
yield* events.publish(SessionEvent.ModelSelected, {
|
yield* events.publish(SessionEvent.ModelSelected, {
|
||||||
sessionID,
|
sessionID,
|
||||||
model: { id: ModelV2.ID.make("replacement"), providerID: ProviderV2.ID.make("fake") },
|
model: { id: ModelV2.ID.make("replacement"), providerID: ProviderV2.ID.make("fake") },
|
||||||
})
|
})
|
||||||
systemBaseline = "Replacement context"
|
systemBaseline = "Replacement context"
|
||||||
yield* Deferred.succeed(toolExecutionGate, undefined)
|
yield* Deferred.succeed(executionGate, undefined)
|
||||||
yield* Fiber.join(run)
|
|
||||||
|
|
||||||
|
const second = yield* scenario.llm.next()
|
||||||
|
expect(second.request.model).toBe(replacementModel)
|
||||||
|
expect(second.request.system.map((part) => part.text)).toEqual([defaultSystem, "Initial context"])
|
||||||
|
expect(systemTexts(second.request)).toContain("Replacement context")
|
||||||
|
yield* second.respond.events(
|
||||||
|
LLMEvent.stepStart({ index: 0 }),
|
||||||
|
LLMEvent.stepFinish({ index: 0, reason: "stop" }),
|
||||||
|
LLMEvent.finish({ reason: "stop" }),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
const requests = yield* scenario.llm.requests
|
||||||
expect(requests.map((request) => request.model)).toEqual([model, replacementModel])
|
expect(requests.map((request) => request.model)).toEqual([model, replacementModel])
|
||||||
expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([
|
expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([
|
||||||
[defaultSystem, "Initial context"],
|
[defaultSystem, "Initial context"],
|
||||||
|
|
@ -1939,14 +1937,19 @@ describe("SessionRunnerLLM", () => {
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|
||||||
it.effect("restores durable reasoning provider metadata in a second-turn request", () =>
|
scenarioIt("restores durable reasoning provider metadata in a second-turn request", (scenario) =>
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
yield* setup
|
yield* setup
|
||||||
const session = yield* SessionV2.Service
|
const session = yield* SessionV2.Service
|
||||||
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Think first" }), resume: false })
|
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Think first" }), resume: false })
|
||||||
|
|
||||||
requests.length = 0
|
const streamed = yield* Deferred.make<void>()
|
||||||
response = [
|
const release = yield* Deferred.make<void>()
|
||||||
|
yield* scenario.run(function* () {
|
||||||
|
const first = yield* scenario.llm.next()
|
||||||
|
yield* first.respond.stream(
|
||||||
|
Stream.concat(
|
||||||
|
Stream.fromIterable([
|
||||||
LLMEvent.stepStart({ index: 0 }),
|
LLMEvent.stepStart({ index: 0 }),
|
||||||
LLMEvent.reasoningStart({ id: "reasoning-anthropic" }),
|
LLMEvent.reasoningStart({ id: "reasoning-anthropic" }),
|
||||||
LLMEvent.reasoningDelta({ id: "reasoning-anthropic", text: "Signed thought" }),
|
LLMEvent.reasoningDelta({ id: "reasoning-anthropic", text: "Signed thought" }),
|
||||||
|
|
@ -1971,8 +1974,16 @@ describe("SessionRunnerLLM", () => {
|
||||||
}),
|
}),
|
||||||
LLMEvent.stepFinish({ index: 0, reason: "stop" }),
|
LLMEvent.stepFinish({ index: 0, reason: "stop" }),
|
||||||
LLMEvent.finish({ reason: "stop" }),
|
LLMEvent.finish({ reason: "stop" }),
|
||||||
]
|
]),
|
||||||
yield* session.resume(sessionID)
|
Stream.unwrap(
|
||||||
|
Deferred.succeed(streamed, undefined).pipe(
|
||||||
|
Effect.andThen(Deferred.await(release)),
|
||||||
|
Effect.as(Stream.empty),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
yield* Deferred.await(streamed)
|
||||||
yield* replaySessionProjection(sessionID)
|
yield* replaySessionProjection(sessionID)
|
||||||
|
|
||||||
expect(yield* session.context(sessionID)).toMatchObject([
|
expect(yield* session.context(sessionID)).toMatchObject([
|
||||||
|
|
@ -1991,10 +2002,10 @@ describe("SessionRunnerLLM", () => {
|
||||||
])
|
])
|
||||||
|
|
||||||
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Continue" }), resume: false })
|
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Continue" }), resume: false })
|
||||||
response = []
|
yield* Deferred.succeed(release, undefined)
|
||||||
yield* session.resume(sessionID)
|
|
||||||
|
|
||||||
expect(requests[1]?.messages[1]?.content).toEqual([
|
const second = yield* scenario.llm.next()
|
||||||
|
expect(second.request.messages[1]?.content).toEqual([
|
||||||
{ type: "reasoning", text: "Signed thought", providerMetadata: { fake: { signature: "sig_1" } } },
|
{ type: "reasoning", text: "Signed thought", providerMetadata: { fake: { signature: "sig_1" } } },
|
||||||
{
|
{
|
||||||
type: "reasoning",
|
type: "reasoning",
|
||||||
|
|
@ -2002,17 +2013,24 @@ describe("SessionRunnerLLM", () => {
|
||||||
providerMetadata: { fake: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } },
|
providerMetadata: { fake: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } },
|
||||||
},
|
},
|
||||||
])
|
])
|
||||||
|
yield* second.respond.events()
|
||||||
|
})
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|
||||||
it.effect("replays durable provider-executed tool results inline in a second-turn request", () =>
|
scenarioIt("replays durable provider-executed tool results inline in a second-turn request", (scenario) =>
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
yield* setup
|
yield* setup
|
||||||
const session = yield* SessionV2.Service
|
const session = yield* SessionV2.Service
|
||||||
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Search first" }), resume: false })
|
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Search first" }), resume: false })
|
||||||
|
|
||||||
requests.length = 0
|
const streamed = yield* Deferred.make<void>()
|
||||||
response = [
|
const release = yield* Deferred.make<void>()
|
||||||
|
yield* scenario.run(function* () {
|
||||||
|
const first = yield* scenario.llm.next()
|
||||||
|
yield* first.respond.stream(
|
||||||
|
Stream.concat(
|
||||||
|
Stream.fromIterable([
|
||||||
LLMEvent.stepStart({ index: 0 }),
|
LLMEvent.stepStart({ index: 0 }),
|
||||||
LLMEvent.toolCall({
|
LLMEvent.toolCall({
|
||||||
id: "hosted-search",
|
id: "hosted-search",
|
||||||
|
|
@ -2030,16 +2048,24 @@ describe("SessionRunnerLLM", () => {
|
||||||
}),
|
}),
|
||||||
LLMEvent.stepFinish({ index: 0, reason: "stop" }),
|
LLMEvent.stepFinish({ index: 0, reason: "stop" }),
|
||||||
LLMEvent.finish({ reason: "stop" }),
|
LLMEvent.finish({ reason: "stop" }),
|
||||||
]
|
]),
|
||||||
yield* session.resume(sessionID)
|
Stream.unwrap(
|
||||||
|
Deferred.succeed(streamed, undefined).pipe(
|
||||||
|
Effect.andThen(Deferred.await(release)),
|
||||||
|
Effect.as(Stream.empty),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
yield* Deferred.await(streamed)
|
||||||
yield* replaySessionProjection(sessionID)
|
yield* replaySessionProjection(sessionID)
|
||||||
|
|
||||||
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Continue" }), resume: false })
|
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Continue" }), resume: false })
|
||||||
response = []
|
yield* Deferred.succeed(release, undefined)
|
||||||
yield* session.resume(sessionID)
|
|
||||||
|
|
||||||
expect(requests[1]?.messages.map((message) => message.role)).toEqual(["user", "assistant", "user"])
|
const second = yield* scenario.llm.next()
|
||||||
expect(requests[1]?.messages[1]?.content).toMatchObject([
|
expect(second.request.messages.map((message) => message.role)).toEqual(["user", "assistant", "user"])
|
||||||
|
expect(second.request.messages[1]?.content).toMatchObject([
|
||||||
{
|
{
|
||||||
type: "tool-call",
|
type: "tool-call",
|
||||||
id: "hosted-search",
|
id: "hosted-search",
|
||||||
|
|
@ -2057,22 +2083,23 @@ describe("SessionRunnerLLM", () => {
|
||||||
providerMetadata: { fake: { blockType: "web_search_tool_result" } },
|
providerMetadata: { fake: { blockType: "web_search_tool_result" } },
|
||||||
},
|
},
|
||||||
])
|
])
|
||||||
|
yield* second.respond.events()
|
||||||
|
})
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|
||||||
it.effect("starts recorded local tools eagerly and awaits settlement before continuing", () =>
|
scenarioIt("starts recorded local tools eagerly and awaits settlement before continuing", (scenario) =>
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
yield* setup
|
yield* setup
|
||||||
const session = yield* SessionV2.Service
|
const session = yield* SessionV2.Service
|
||||||
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Echo five times" }), resume: false })
|
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Echo five times" }), resume: false })
|
||||||
|
|
||||||
requests.length = 0
|
|
||||||
executions.length = 0
|
executions.length = 0
|
||||||
toolExecutionGate = yield* Deferred.make<void>()
|
const executionGate = yield* Deferred.make<void>()
|
||||||
toolExecutionsStarted = yield* Deferred.make<void>()
|
const executionsStarted = yield* Deferred.make<void>()
|
||||||
|
toolExecutionGate = executionGate
|
||||||
|
toolExecutionsStarted = executionsStarted
|
||||||
const providerGate = yield* Deferred.make<void>()
|
const providerGate = yield* Deferred.make<void>()
|
||||||
response = []
|
|
||||||
responses = undefined
|
|
||||||
const initial = Stream.fromIterable([
|
const initial = Stream.fromIterable([
|
||||||
LLMEvent.stepStart({ index: 0 }),
|
LLMEvent.stepStart({ index: 0 }),
|
||||||
...Array.from({ length: 5 }, (_, index) =>
|
...Array.from({ length: 5 }, (_, index) =>
|
||||||
|
|
@ -2083,14 +2110,16 @@ describe("SessionRunnerLLM", () => {
|
||||||
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
|
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
|
||||||
LLMEvent.finish({ reason: "tool-calls" }),
|
LLMEvent.finish({ reason: "tool-calls" }),
|
||||||
])
|
])
|
||||||
streamGate = undefined
|
|
||||||
responseStream = Stream.concat(
|
|
||||||
initial,
|
|
||||||
Stream.fromEffect(Deferred.await(providerGate)).pipe(Stream.flatMap(() => final)),
|
|
||||||
)
|
|
||||||
|
|
||||||
const run = yield* session.resume(sessionID).pipe(Effect.forkChild)
|
yield* scenario.run(function* () {
|
||||||
yield* Deferred.await(toolExecutionsStarted)
|
const first = yield* scenario.llm.next()
|
||||||
|
yield* first.respond.stream(
|
||||||
|
Stream.concat(
|
||||||
|
initial,
|
||||||
|
Stream.unwrap(Deferred.await(providerGate).pipe(Effect.as(final))),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
yield* Deferred.await(executionsStarted)
|
||||||
|
|
||||||
expect(executions).toHaveLength(5)
|
expect(executions).toHaveLength(5)
|
||||||
expect(maxActiveToolExecutions).toBe(5)
|
expect(maxActiveToolExecutions).toBe(5)
|
||||||
|
|
@ -2108,47 +2137,35 @@ describe("SessionRunnerLLM", () => {
|
||||||
|
|
||||||
yield* Deferred.succeed(providerGate, undefined)
|
yield* Deferred.succeed(providerGate, undefined)
|
||||||
yield* Effect.yieldNow
|
yield* Effect.yieldNow
|
||||||
expect(requests).toHaveLength(1)
|
expect(yield* scenario.llm.requests).toHaveLength(1)
|
||||||
|
|
||||||
yield* Deferred.succeed(toolExecutionGate, undefined)
|
yield* Deferred.succeed(executionGate, undefined)
|
||||||
yield* Fiber.join(run)
|
yield* (yield* scenario.llm.next()).respond.events()
|
||||||
|
})
|
||||||
toolExecutionGate = undefined
|
toolExecutionGate = undefined
|
||||||
toolExecutionsStarted = undefined
|
toolExecutionsStarted = undefined
|
||||||
|
|
||||||
expect(executions).toHaveLength(5)
|
expect(executions).toHaveLength(5)
|
||||||
expect(maxActiveToolExecutions).toBe(5)
|
expect(maxActiveToolExecutions).toBe(5)
|
||||||
expect(requests).toHaveLength(2)
|
expect(yield* scenario.llm.requests).toHaveLength(2)
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|
||||||
it.effect("settles repeated provider-local tool call IDs against their owning assistant messages", () =>
|
scenarioIt("settles repeated provider-local tool call IDs against their owning assistant messages", (scenario) =>
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
yield* setup
|
yield* setup
|
||||||
const session = yield* SessionV2.Service
|
const session = yield* SessionV2.Service
|
||||||
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Echo twice" }), resume: false })
|
yield* session.prompt({ sessionID, prompt: PromptInput.Prompt.make({ text: "Echo twice" }), resume: false })
|
||||||
|
|
||||||
requests.length = 0
|
|
||||||
executions.length = 0
|
executions.length = 0
|
||||||
responses = [
|
yield* scenario.run(function* () {
|
||||||
[
|
yield* (yield* scenario.llm.next()).respond.toolCall("echo", { text: "first" }, { id: "tool_0" })
|
||||||
LLMEvent.stepStart({ index: 0 }),
|
yield* (yield* scenario.llm.next()).respond.toolCall("echo", { text: "second" }, { id: "tool_0" })
|
||||||
LLMEvent.toolCall({ id: "tool_0", name: "echo", input: { text: "first" } }),
|
yield* (yield* scenario.llm.next()).respond.events()
|
||||||
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
|
})
|
||||||
LLMEvent.finish({ reason: "tool-calls" }),
|
|
||||||
],
|
|
||||||
[
|
|
||||||
LLMEvent.stepStart({ index: 0 }),
|
|
||||||
LLMEvent.toolCall({ id: "tool_0", name: "echo", input: { text: "second" } }),
|
|
||||||
LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }),
|
|
||||||
LLMEvent.finish({ reason: "tool-calls" }),
|
|
||||||
],
|
|
||||||
[],
|
|
||||||
]
|
|
||||||
|
|
||||||
yield* session.resume(sessionID)
|
|
||||||
|
|
||||||
expect(executions).toEqual(["first", "second"])
|
expect(executions).toEqual(["first", "second"])
|
||||||
expect(requests).toHaveLength(3)
|
expect(yield* scenario.llm.requests).toHaveLength(3)
|
||||||
expect(yield* session.context(sessionID)).toMatchObject([
|
expect(yield* session.context(sessionID)).toMatchObject([
|
||||||
{ type: "user", text: "Echo twice" },
|
{ type: "user", text: "Echo twice" },
|
||||||
{
|
{
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue