fix(llm): preserve native continuation metadata (#28678)
This commit is contained in:
Kit Langton
GitHub
parent
a58c3c53a9
commit
61390dbb49
12 changed files with 843 additions and 194 deletions
|
|
@ -1,10 +1,12 @@
|
|||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { HttpClientRequest } from "effect/unstable/http"
|
||||
import { CacheHint, LLM, LLMError, Message, ToolCallPart, Usage } from "../../src"
|
||||
import { Auth, LLMClient } from "../../src/route"
|
||||
import * as AnthropicMessages from "../../src/protocols/anthropic-messages"
|
||||
import { continuationRequest, nativeAnthropicMessagesContinuation } from "../continuation-scenarios"
|
||||
import { it } from "../lib/effect"
|
||||
import { fixedResponse } from "../lib/http"
|
||||
import { dynamicResponse, fixedResponse } from "../lib/http"
|
||||
import { sseEvents } from "../lib/sse"
|
||||
|
||||
const model = AnthropicMessages.route
|
||||
|
|
@ -40,7 +42,7 @@ describe("Anthropic Messages route", () => {
|
|||
|
||||
it.effect("prepares tool call and tool result messages", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
const prepared = yield* LLMClient.prepare<AnthropicMessages.AnthropicMessagesBody>(
|
||||
LLM.request({
|
||||
id: "req_tool_result",
|
||||
model,
|
||||
|
|
@ -69,6 +71,50 @@ describe("Anthropic Messages route", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("prepares the composed native continuation request", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare<AnthropicMessages.AnthropicMessagesBody>(
|
||||
continuationRequest({
|
||||
id: "req_native_continuation_anthropic",
|
||||
model,
|
||||
features: nativeAnthropicMessagesContinuation,
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toMatchObject({
|
||||
system: [{ type: "text", text: "You are concise. Continue from the provided history." }],
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "What is shown here?" },
|
||||
{ type: "image", source: { type: "base64", media_type: "image/png", data: "AAECAw==" } },
|
||||
],
|
||||
},
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "thinking", thinking: "I inspected the previous turn.", signature: "sig_continuation_1" },
|
||||
{ type: "text", text: "It shows a small test image." },
|
||||
],
|
||||
},
|
||||
{ role: "user", content: [{ type: "text", text: "Check the weather in Paris before continuing." }] },
|
||||
{
|
||||
role: "assistant",
|
||||
content: [{ type: "tool_use", id: "call_weather_1", name: "get_weather", input: { city: "Paris" } }],
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "tool_result", tool_use_id: "call_weather_1", content: '{"temperature":22}' }],
|
||||
},
|
||||
{ role: "assistant", content: [{ type: "text", text: "Paris is 22 degrees." }] },
|
||||
{ role: "user", content: [{ type: "text", text: "Continue from this conversation in one short sentence." }] },
|
||||
],
|
||||
})
|
||||
expect(prepared.body.tools).toEqual([expect.objectContaining({ name: "get_weather" })])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("lowers preserved Anthropic reasoning signature metadata", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
|
|
@ -392,17 +438,51 @@ describe("Anthropic Messages route", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects unsupported user media content", () =>
|
||||
it.effect("continues a conversation with user image content", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.prepare(
|
||||
const response = yield* LLMClient.generate(
|
||||
LLM.request({
|
||||
id: "req_media",
|
||||
model,
|
||||
messages: [Message.user({ type: "media", mediaType: "image/png", data: "AAECAw==" })],
|
||||
messages: [
|
||||
Message.user([
|
||||
{ type: "text", text: "What is in this image?" },
|
||||
{ type: "media", mediaType: "image/png", data: "AAECAw==" },
|
||||
]),
|
||||
],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(yield* Effect.promise(() => web.json())).toMatchObject({
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "What is in this image?" },
|
||||
{ type: "image", source: { type: "base64", media_type: "image/png", data: "AAECAw==" } },
|
||||
],
|
||||
},
|
||||
],
|
||||
})
|
||||
return input.respond(
|
||||
sseEvents(
|
||||
{ type: "content_block_start", index: 0, content_block: { type: "text", text: "" } },
|
||||
{ type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "An image." } },
|
||||
{ type: "content_block_stop", index: 0 },
|
||||
{ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 3 } },
|
||||
{ type: "message_stop" },
|
||||
),
|
||||
{ headers: { "content-type": "text/event-stream" } },
|
||||
)
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(error.message).toContain("Anthropic Messages user messages only support text content for now")
|
||||
expect(response.text).toBe("An image.")
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -84,6 +84,7 @@ describeRecordedGoldenScenarios([
|
|||
scenarios: [
|
||||
{ id: "text", temperature: false },
|
||||
{ id: "reasoning", temperature: false },
|
||||
{ id: "reasoning-continuation", temperature: false },
|
||||
{ id: "tool-call", temperature: false },
|
||||
{ id: "tool-loop", temperature: false },
|
||||
],
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import * as Azure from "../../src/providers/azure"
|
|||
import * as OpenAI from "../../src/providers/openai"
|
||||
import * as OpenAIResponses from "../../src/protocols/openai-responses"
|
||||
import * as ProviderShared from "../../src/protocols/shared"
|
||||
import { continuationRequest, nativeOpenAIResponsesContinuation } from "../continuation-scenarios"
|
||||
import { it } from "../lib/effect"
|
||||
import { dynamicResponse, fixedResponse } from "../lib/http"
|
||||
import { sseEvents } from "../lib/sse"
|
||||
|
|
@ -247,6 +248,49 @@ describe("OpenAI Responses route", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("prepares the composed native continuation request", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
|
||||
continuationRequest({
|
||||
id: "req_native_continuation_openai",
|
||||
model,
|
||||
features: nativeOpenAIResponsesContinuation,
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toMatchObject({
|
||||
input: [
|
||||
{ role: "system", content: "You are concise. Continue from the provided history." },
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "input_text", text: "What is shown here?" },
|
||||
{ type: "input_image", image_url: "data:image/png;base64,AAECAw==" },
|
||||
],
|
||||
},
|
||||
{
|
||||
type: "reasoning",
|
||||
id: "rs_continuation_1",
|
||||
encrypted_content: "encrypted-continuation-state",
|
||||
summary: [{ type: "summary_text", text: "I inspected the previous turn." }],
|
||||
},
|
||||
{ role: "assistant", content: [{ type: "output_text", text: "It shows a small test image." }] },
|
||||
{ role: "user", content: [{ type: "input_text", text: "Check the weather in Paris before continuing." }] },
|
||||
{ type: "function_call", call_id: "call_weather_1", name: "get_weather", arguments: '{"city":"Paris"}' },
|
||||
{ type: "function_call_output", call_id: "call_weather_1", output: '{"temperature":22}' },
|
||||
{ role: "assistant", content: [{ type: "output_text", text: "Paris is 22 degrees." }] },
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "input_text", text: "Continue from this conversation in one short sentence." }],
|
||||
},
|
||||
],
|
||||
include: ["reasoning.encrypted_content"],
|
||||
store: false,
|
||||
})
|
||||
expect(prepared.body.tools).toEqual([expect.objectContaining({ type: "function", name: "get_weather" })])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("maps OpenAI provider options to Responses options", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
|
||||
|
|
@ -380,6 +424,172 @@ describe("OpenAI Responses route", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("preserves encrypted reasoning metadata for continuation", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse(
|
||||
sseEvents(
|
||||
{ type: "response.reasoning_summary_text.delta", item_id: "rs_1", delta: "thinking" },
|
||||
{
|
||||
type: "response.output_item.done",
|
||||
item: {
|
||||
type: "reasoning",
|
||||
id: "rs_1",
|
||||
encrypted_content: "encrypted-state",
|
||||
summary: [{ type: "summary_text", text: "thinking" }],
|
||||
},
|
||||
},
|
||||
{ type: "response.completed", response: { id: "resp_1" } },
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(response.events).toContainEqual(
|
||||
expect.objectContaining({
|
||||
type: "reasoning-end",
|
||||
id: "rs_1",
|
||||
providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } },
|
||||
}),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("continues a stateless reasoning conversation", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* LLMClient.generate(
|
||||
LLM.request({
|
||||
id: "req_reasoning_continue",
|
||||
model,
|
||||
messages: [
|
||||
Message.user("What changed?"),
|
||||
Message.assistant([
|
||||
{
|
||||
type: "reasoning",
|
||||
text: "Checked the previous diff.",
|
||||
providerMetadata: {
|
||||
openai: {
|
||||
itemId: "rs_1",
|
||||
reasoningEncryptedContent: "encrypted-state",
|
||||
},
|
||||
},
|
||||
},
|
||||
{ type: "text", text: "The parser changed." },
|
||||
]),
|
||||
Message.user("Summarize it."),
|
||||
],
|
||||
}),
|
||||
).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(yield* Effect.promise(() => web.json())).toMatchObject({
|
||||
input: [
|
||||
{ role: "user", content: [{ type: "input_text", text: "What changed?" }] },
|
||||
{
|
||||
type: "reasoning",
|
||||
id: "rs_1",
|
||||
encrypted_content: "encrypted-state",
|
||||
summary: [{ type: "summary_text", text: "Checked the previous diff." }],
|
||||
},
|
||||
{ role: "assistant", content: [{ type: "output_text", text: "The parser changed." }] },
|
||||
{ role: "user", content: [{ type: "input_text", text: "Summarize it." }] },
|
||||
],
|
||||
})
|
||||
return input.respond(
|
||||
sseEvents(
|
||||
{ type: "response.output_text.delta", item_id: "msg_1", delta: "Parser now round-trips reasoning." },
|
||||
{ type: "response.completed", response: { id: "resp_1" } },
|
||||
),
|
||||
{ headers: { "content-type": "text/event-stream" } },
|
||||
)
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(response.text).toBe("Parser now round-trips reasoning.")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("preserves assistant content order around reasoning items", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
|
||||
LLM.request({
|
||||
id: "req_reasoning_order",
|
||||
model,
|
||||
messages: [
|
||||
Message.assistant([
|
||||
{ type: "text", text: "Before." },
|
||||
{
|
||||
type: "reasoning",
|
||||
text: "Checked order.",
|
||||
providerMetadata: {
|
||||
openai: {
|
||||
itemId: "rs_1",
|
||||
reasoningEncryptedContent: "encrypted-state",
|
||||
},
|
||||
},
|
||||
},
|
||||
{ type: "text", text: "After." },
|
||||
]),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body.input).toEqual([
|
||||
{ role: "assistant", content: [{ type: "output_text", text: "Before." }] },
|
||||
{
|
||||
type: "reasoning",
|
||||
id: "rs_1",
|
||||
encrypted_content: "encrypted-state",
|
||||
summary: [{ type: "summary_text", text: "Checked order." }],
|
||||
},
|
||||
{ role: "assistant", content: [{ type: "output_text", text: "After." }] },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("skips non-persisted reasoning ids without encrypted state", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_reasoning_without_encrypted_state",
|
||||
model,
|
||||
messages: [
|
||||
Message.user("What changed?"),
|
||||
Message.assistant([
|
||||
{
|
||||
type: "reasoning",
|
||||
text: "Checked the previous diff.",
|
||||
providerMetadata: {
|
||||
openai: {
|
||||
itemId: "rs_1",
|
||||
reasoningEncryptedContent: null,
|
||||
},
|
||||
},
|
||||
},
|
||||
{ type: "text", text: "The parser changed." },
|
||||
]),
|
||||
Message.user("Summarize it."),
|
||||
],
|
||||
providerOptions: { openai: { store: false } },
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toMatchObject({
|
||||
input: [
|
||||
{ role: "user", content: [{ type: "input_text", text: "What changed?" }] },
|
||||
{ role: "assistant", content: [{ type: "output_text", text: "The parser changed." }] },
|
||||
{ role: "user", content: [{ type: "input_text", text: "Summarize it." }] },
|
||||
],
|
||||
store: false,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("assembles streamed function call input", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue