fix(provider): preserve Mistral reasoning history (#38453)
This commit is contained in:
Aiden Cline
GitHub
parent
62e4641235
commit
20589d66d5
8 changed files with 1084 additions and 94 deletions
|
|
@ -72,7 +72,7 @@
|
|||
"@ai-sdk/google": "3.0.73",
|
||||
"@ai-sdk/google-vertex": "4.0.128",
|
||||
"@ai-sdk/groq": "3.0.31",
|
||||
"@ai-sdk/mistral": "3.0.34",
|
||||
"@ai-sdk/mistral": "3.0.51",
|
||||
"@ai-sdk/openai": "3.0.84",
|
||||
"@ai-sdk/openai-compatible": "2.0.41",
|
||||
"@ai-sdk/perplexity": "3.0.26",
|
||||
|
|
|
|||
|
|
@ -26,3 +26,257 @@ test("Mistral sends promptCacheKey as prompt_cache_key", async () => {
|
|||
|
||||
expect(body?.prompt_cache_key).toBe("session-123")
|
||||
})
|
||||
|
||||
test("Mistral round-trips native reasoning in assistant history", async () => {
|
||||
let body: { messages?: unknown[] } | undefined
|
||||
const mockFetch = Object.assign(
|
||||
async (_input: Parameters<typeof fetch>[0], init?: RequestInit) => {
|
||||
body = JSON.parse(String(init?.body))
|
||||
return Response.json({
|
||||
id: "response-1",
|
||||
created: 0,
|
||||
model: "mistral-small-latest",
|
||||
object: "chat.completion",
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: [
|
||||
{ type: "text", text: "The user is greeting me." },
|
||||
{
|
||||
type: "tool_reference",
|
||||
tool: "web_search",
|
||||
title: "Example result",
|
||||
url: "https://example.com/tool",
|
||||
favicon: "https://example.com/favicon.ico",
|
||||
description: "Example description",
|
||||
},
|
||||
{ type: "reference", reference_ids: [1, "source-2"] },
|
||||
],
|
||||
closed: true,
|
||||
signature: "sig-123",
|
||||
},
|
||||
{ type: "text", text: "Hi" },
|
||||
],
|
||||
},
|
||||
finish_reason: "stop",
|
||||
},
|
||||
],
|
||||
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
|
||||
})
|
||||
},
|
||||
{ preconnect: fetch.preconnect },
|
||||
)
|
||||
const model = createMistral({ apiKey: "test", fetch: mockFetch })("mistral-small-latest")
|
||||
|
||||
const first = await model.doGenerate({
|
||||
prompt: [{ role: "user", content: [{ type: "text", text: "Hello" }] }],
|
||||
})
|
||||
const reasoning = first.content.find((part) => part.type === "reasoning")
|
||||
const text = first.content.find((part) => part.type === "text")
|
||||
if (!reasoning || !text) throw new Error("expected reasoning and text")
|
||||
|
||||
await model.doGenerate({
|
||||
prompt: [
|
||||
{ role: "user", content: [{ type: "text", text: "Hello" }] },
|
||||
{
|
||||
role: "assistant",
|
||||
content: [{ ...reasoning, providerOptions: reasoning.providerMetadata }, text],
|
||||
},
|
||||
{ role: "user", content: [{ type: "text", text: "Hello again" }] },
|
||||
],
|
||||
})
|
||||
|
||||
expect(body?.messages?.[1]).toEqual({
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: [
|
||||
{ type: "text", text: "The user is greeting me." },
|
||||
{
|
||||
type: "tool_reference",
|
||||
tool: "web_search",
|
||||
title: "Example result",
|
||||
url: "https://example.com/tool",
|
||||
favicon: "https://example.com/favicon.ico",
|
||||
description: "Example description",
|
||||
},
|
||||
{ type: "reference", reference_ids: [1, "source-2"] },
|
||||
],
|
||||
closed: true,
|
||||
signature: "sig-123",
|
||||
},
|
||||
{ type: "text", text: "Hi" },
|
||||
],
|
||||
})
|
||||
|
||||
await model.doGenerate({
|
||||
prompt: [
|
||||
{ role: "user", content: [{ type: "text", text: "Hello" }] },
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "reasoning", text: "thinking" },
|
||||
{ type: "text", text: "Hi" },
|
||||
],
|
||||
},
|
||||
{ role: "user", content: [{ type: "text", text: "Hello again" }] },
|
||||
],
|
||||
})
|
||||
expect(body?.messages?.[1]).toEqual({ role: "assistant", content: "thinkingHi" })
|
||||
})
|
||||
|
||||
test("Mistral preserves native reasoning metadata while streaming", async () => {
|
||||
const chunks = [
|
||||
{
|
||||
id: "response-1",
|
||||
created: 0,
|
||||
model: "mistral-small-latest",
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
delta: {
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: [
|
||||
{ type: "text", text: "thinking" },
|
||||
{
|
||||
type: "tool_reference",
|
||||
tool: "web_search",
|
||||
title: "Example result",
|
||||
url: "https://example.com/tool",
|
||||
favicon: "https://example.com/favicon.ico",
|
||||
description: "Example description",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "response-1",
|
||||
created: 0,
|
||||
model: "mistral-small-latest",
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
delta: {
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: [{ type: "reference", reference_ids: [1, "source-2"] }],
|
||||
closed: true,
|
||||
signature: "sig-123",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "response-1",
|
||||
created: 0,
|
||||
model: "mistral-small-latest",
|
||||
choices: [{ index: 0, delta: { content: [{ type: "text", text: "answer" }] } }],
|
||||
},
|
||||
{
|
||||
id: "response-1",
|
||||
created: 0,
|
||||
model: "mistral-small-latest",
|
||||
choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
|
||||
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
|
||||
},
|
||||
]
|
||||
const mockFetch = Object.assign(
|
||||
async () =>
|
||||
new Response(chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join(""), {
|
||||
headers: { "Content-Type": "text/event-stream" },
|
||||
}),
|
||||
{ preconnect: fetch.preconnect },
|
||||
)
|
||||
const model = createMistral({ apiKey: "test", fetch: mockFetch })("mistral-small-latest")
|
||||
const result = await model.doStream({
|
||||
prompt: [{ role: "user", content: [{ type: "text", text: "Hello" }] }],
|
||||
})
|
||||
const events = []
|
||||
for await (const event of result.stream) events.push(event)
|
||||
|
||||
expect(events.find((event) => event.type === "reasoning-end")?.providerMetadata).toEqual({
|
||||
mistral: {
|
||||
thinking: {
|
||||
type: "thinking",
|
||||
thinking: [
|
||||
{ type: "text", text: "thinking" },
|
||||
{
|
||||
type: "tool_reference",
|
||||
tool: "web_search",
|
||||
title: "Example result",
|
||||
url: "https://example.com/tool",
|
||||
favicon: "https://example.com/favicon.ico",
|
||||
description: "Example description",
|
||||
},
|
||||
{ type: "reference", reference_ids: [1, "source-2"] },
|
||||
],
|
||||
closed: true,
|
||||
signature: "sig-123",
|
||||
},
|
||||
},
|
||||
})
|
||||
expect(
|
||||
events
|
||||
.filter((event) => event.type === "reasoning-start" || event.type === "reasoning-delta")
|
||||
.every((event) => event.providerMetadata === undefined),
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
test("Mistral preserves metadata-only thinking chunks", async () => {
|
||||
const thinking = {
|
||||
type: "thinking" as const,
|
||||
thinking: [
|
||||
{
|
||||
type: "tool_reference",
|
||||
tool: "web_search",
|
||||
title: "Example result",
|
||||
url: "https://example.com/tool",
|
||||
favicon: "https://example.com/favicon.ico",
|
||||
description: "Example description",
|
||||
},
|
||||
{ type: "reference", reference_ids: [1, "source-2"] },
|
||||
],
|
||||
closed: true,
|
||||
signature: "sig-123",
|
||||
}
|
||||
const mockFetch = Object.assign(
|
||||
async () =>
|
||||
Response.json({
|
||||
id: "response-1",
|
||||
created: 0,
|
||||
model: "mistral-small-latest",
|
||||
object: "chat.completion",
|
||||
choices: [{ index: 0, message: { role: "assistant", content: [thinking] }, finish_reason: "stop" }],
|
||||
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
|
||||
}),
|
||||
{ preconnect: fetch.preconnect },
|
||||
)
|
||||
const model = createMistral({ apiKey: "test", fetch: mockFetch })("mistral-small-latest")
|
||||
const result = await model.doGenerate({
|
||||
prompt: [{ role: "user", content: [{ type: "text", text: "Hello" }] }],
|
||||
})
|
||||
|
||||
expect(result.content).toEqual([
|
||||
{
|
||||
type: "reasoning",
|
||||
text: "",
|
||||
providerMetadata: { mistral: { thinking } },
|
||||
},
|
||||
])
|
||||
})
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@
|
|||
"@ai-sdk/google": "3.0.73",
|
||||
"@ai-sdk/google-vertex": "4.0.128",
|
||||
"@ai-sdk/groq": "3.0.31",
|
||||
"@ai-sdk/mistral": "3.0.34",
|
||||
"@ai-sdk/mistral": "3.0.51",
|
||||
"@ai-sdk/openai": "3.0.84",
|
||||
"@ai-sdk/openai-compatible": "2.0.41",
|
||||
"@ai-sdk/perplexity": "3.0.26",
|
||||
|
|
|
|||
|
|
@ -906,6 +906,117 @@ describe("session.llm.stream", () => {
|
|||
},
|
||||
)
|
||||
|
||||
const mistralFixture = { providerID: "mistral", modelID: "mistral-small-latest" }
|
||||
it.instance(
|
||||
"replays native Mistral reasoning from chat history",
|
||||
() =>
|
||||
Effect.gen(function* () {
|
||||
const fixture = loadFixture(mistralFixture.providerID, mistralFixture.modelID)
|
||||
const request = waitRequest(
|
||||
"/chat/completions",
|
||||
createEventResponse(
|
||||
[
|
||||
{
|
||||
id: "chatcmpl-mistral",
|
||||
object: "chat.completion.chunk",
|
||||
created: 0,
|
||||
model: fixture.model.id,
|
||||
choices: [{ index: 0, delta: { role: "assistant", content: "Hello" } }],
|
||||
},
|
||||
{
|
||||
id: "chatcmpl-mistral",
|
||||
object: "chat.completion.chunk",
|
||||
created: 0,
|
||||
model: fixture.model.id,
|
||||
choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
|
||||
},
|
||||
],
|
||||
true,
|
||||
),
|
||||
)
|
||||
|
||||
const resolved = yield* Provider.use.getModel(
|
||||
ProviderV2.ID.make(mistralFixture.providerID),
|
||||
ModelV2.ID.make(fixture.model.id),
|
||||
)
|
||||
const sessionID = SessionID.make("session-test-mistral-reasoning")
|
||||
const agent = {
|
||||
name: "test",
|
||||
mode: "primary",
|
||||
options: {},
|
||||
permission: [{ permission: "*", pattern: "*", action: "allow" }],
|
||||
} satisfies Agent.Info
|
||||
|
||||
const user = {
|
||||
id: MessageID.make("msg_user-mistral-reasoning"),
|
||||
sessionID,
|
||||
role: "user",
|
||||
time: { created: Date.now() },
|
||||
agent: agent.name,
|
||||
model: { providerID: ProviderV2.ID.make(mistralFixture.providerID), modelID: resolved.id },
|
||||
} satisfies SessionV1.User
|
||||
|
||||
const thinking = {
|
||||
type: "thinking",
|
||||
thinking: [
|
||||
{ type: "text", text: "thinking" },
|
||||
{
|
||||
type: "tool_reference",
|
||||
tool: "web_search",
|
||||
title: "Example result",
|
||||
url: "https://example.com/tool",
|
||||
favicon: "https://example.com/favicon.ico",
|
||||
description: "Example description",
|
||||
},
|
||||
{ type: "reference", reference_ids: [1, "source-2"] },
|
||||
],
|
||||
closed: true,
|
||||
signature: "sig-123",
|
||||
}
|
||||
|
||||
yield* drain({
|
||||
user,
|
||||
sessionID,
|
||||
model: resolved,
|
||||
agent,
|
||||
system: ["You are a helpful assistant."],
|
||||
messages: [
|
||||
{ role: "user", content: "Hello" },
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "reasoning",
|
||||
text: "thinking",
|
||||
providerOptions: { mistral: { thinking } },
|
||||
},
|
||||
{ type: "text", text: "Previous answer" },
|
||||
],
|
||||
},
|
||||
{ role: "user", content: "Continue" },
|
||||
] satisfies ModelMessage[],
|
||||
tools: {},
|
||||
})
|
||||
|
||||
const capture = yield* Effect.promise(() => request)
|
||||
const messages = capture.body.messages as Array<Record<string, unknown>>
|
||||
expect(messages.find((message) => message.role === "assistant")).toEqual({
|
||||
role: "assistant",
|
||||
content: [thinking, { type: "text", text: "Previous answer" }],
|
||||
})
|
||||
}),
|
||||
{
|
||||
config: () => ({
|
||||
enabled_providers: [mistralFixture.providerID],
|
||||
provider: {
|
||||
[mistralFixture.providerID]: {
|
||||
options: { apiKey: "test-key", baseURL: `${state.server!.url.origin}/v1` },
|
||||
},
|
||||
},
|
||||
}),
|
||||
},
|
||||
)
|
||||
|
||||
const alibabaQwenFixture = { providerID: "alibaba", modelID: "qwen-plus" }
|
||||
it.instance(
|
||||
"service stream cancellation cancels provider response body promptly",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue