fix(llm): restore OpenAI reasoning streams (#28552)
This commit is contained in:
parent
93131b6e4c
commit
16fb6dac8d
9 changed files with 172 additions and 15 deletions
32
packages/llm/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-reasoning.json
vendored
Normal file
32
packages/llm/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-reasoning.json
vendored
Normal file
File diff suppressed because one or more lines are too long
|
|
@ -83,6 +83,7 @@ describeRecordedGoldenScenarios([
|
|||
tags: ["flagship"],
|
||||
scenarios: [
|
||||
{ id: "text", temperature: false },
|
||||
{ id: "reasoning", temperature: false },
|
||||
{ id: "tool-call", temperature: false },
|
||||
{ id: "tool-loop", temperature: false },
|
||||
],
|
||||
|
|
|
|||
|
|
@ -260,6 +260,32 @@ describe("OpenAI Chat route", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("parses OpenAI-compatible reasoning content deltas", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
{ choices: [{ delta: { reasoning_content: "thinking" } }] },
|
||||
{ choices: [{ delta: { content: "Hello" } }] },
|
||||
{ choices: [{ delta: {}, finish_reason: "stop" }] },
|
||||
)
|
||||
|
||||
const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
expect(response.reasoning).toBe("thinking")
|
||||
expect(response.text).toBe("Hello")
|
||||
expect(response.events).toMatchObject([
|
||||
{ type: "step-start", index: 0 },
|
||||
{ type: "reasoning-start", id: "reasoning-0" },
|
||||
{ type: "reasoning-delta", id: "reasoning-0", text: "thinking" },
|
||||
{ type: "text-start", id: "text-0" },
|
||||
{ type: "text-delta", id: "text-0", text: "Hello" },
|
||||
{ type: "reasoning-end", id: "reasoning-0" },
|
||||
{ type: "text-end", id: "text-0" },
|
||||
{ type: "step-finish", index: 0, reason: "stop" },
|
||||
{ type: "finish", reason: "stop" },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("assembles streamed tool call input", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
|
|
|
|||
|
|
@ -118,6 +118,7 @@ describe("OpenAI Responses route", () => {
|
|||
it.effect("fails immediately when WebSocket is already closed", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* WebSocketExecutor.fromWebSocket(
|
||||
// oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion -- fromWebSocket reads readyState before touching WebSocket methods on this branch.
|
||||
{ readyState: globalThis.WebSocket.CLOSED } as globalThis.WebSocket,
|
||||
{ url: "wss://api.openai.test/v1/responses", headers: Headers.empty },
|
||||
).pipe(Effect.flip)
|
||||
|
|
@ -352,6 +353,33 @@ describe("OpenAI Responses route", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("parses reasoning summary stream fixtures", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
{ type: "response.reasoning_summary_text.delta", item_id: "rs_1", delta: "thinking" },
|
||||
{ type: "response.output_text.delta", item_id: "msg_1", delta: "Hello" },
|
||||
{ type: "response.reasoning_summary_text.done", item_id: "rs_1" },
|
||||
{ type: "response.completed", response: { id: "resp_1" } },
|
||||
)
|
||||
|
||||
const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
expect(response.reasoning).toBe("thinking")
|
||||
expect(response.text).toBe("Hello")
|
||||
expect(response.events).toMatchObject([
|
||||
{ type: "step-start", index: 0 },
|
||||
{ type: "reasoning-start", id: "rs_1" },
|
||||
{ type: "reasoning-delta", id: "rs_1", text: "thinking" },
|
||||
{ type: "text-start", id: "msg_1" },
|
||||
{ type: "text-delta", id: "msg_1", text: "Hello" },
|
||||
{ type: "reasoning-end", id: "rs_1" },
|
||||
{ type: "text-end", id: "msg_1" },
|
||||
{ type: "step-finish", index: 0, reason: "stop" },
|
||||
{ type: "finish", reason: "stop" },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("assembles streamed function call input", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import type { HttpRecorder } from "@opencode-ai/http-recorder"
|
||||
import { describe, type TestOptions } from "bun:test"
|
||||
import { describe } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import type { Model } from "../src"
|
||||
import { goldenScenarioTags, runGoldenScenario, type GoldenScenarioID } from "./recorded-scenarios"
|
||||
|
|
@ -17,7 +17,7 @@ type ScenarioInput =
|
|||
readonly tags?: ReadonlyArray<string>
|
||||
readonly maxTokens?: number
|
||||
readonly temperature?: number | false
|
||||
readonly timeout?: number | TestOptions
|
||||
readonly timeout?: number
|
||||
}
|
||||
|
||||
type TargetInput = {
|
||||
|
|
@ -38,6 +38,7 @@ const scenarioInput = (input: ScenarioInput) => (typeof input === "string" ? { i
|
|||
const scenarioTitle = (id: GoldenScenarioID) => {
|
||||
if (id === "text") return "streams text"
|
||||
if (id === "tool-call") return "streams tool call"
|
||||
if (id === "reasoning") return "uses reasoning"
|
||||
if (id === "image") return "reads image text"
|
||||
return "drives a tool loop"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -143,6 +143,25 @@ export const imageRequest = (input: {
|
|||
: { maxTokens: input.maxTokens ?? 20, temperature: input.temperature ?? 0 },
|
||||
})
|
||||
|
||||
export const reasoningRequest = (input: {
|
||||
readonly id: string
|
||||
readonly model: Model
|
||||
readonly maxTokens?: number
|
||||
readonly temperature?: number | false
|
||||
}) =>
|
||||
LLM.request({
|
||||
id: input.id,
|
||||
model: input.model,
|
||||
system: "Show concise reasoning when the provider supports visible reasoning summaries.",
|
||||
prompt: "Think briefly, then reply exactly with: Hello!",
|
||||
cache: "none",
|
||||
providerOptions: { openai: { reasoningEffort: "low", reasoningSummary: "auto" } },
|
||||
generation:
|
||||
input.temperature === false
|
||||
? { maxTokens: input.maxTokens ?? 120 }
|
||||
: { maxTokens: input.maxTokens ?? 120, temperature: input.temperature ?? 0 },
|
||||
})
|
||||
|
||||
export const runWeatherToolLoop = (request: LLMRequest) =>
|
||||
LLMClient.stream({
|
||||
request,
|
||||
|
|
@ -193,7 +212,7 @@ export const expectGoldenWeatherToolLoop = (events: ReadonlyArray<LLMEvent>) =>
|
|||
expect(LLMResponse.text({ events }).trim()).toMatch(/^Paris is sunny\.?$/)
|
||||
}
|
||||
|
||||
export type GoldenScenarioID = "text" | "tool-call" | "tool-loop" | "image"
|
||||
export type GoldenScenarioID = "text" | "tool-call" | "tool-loop" | "image" | "reasoning"
|
||||
|
||||
export interface GoldenScenarioContext {
|
||||
readonly id: string
|
||||
|
|
@ -215,6 +234,7 @@ export const goldenScenarioTags = (id: GoldenScenarioID) => {
|
|||
if (id === "text") return ["text", "golden"]
|
||||
if (id === "tool-call") return ["tool", "tool-call", "golden"]
|
||||
if (id === "image") return ["media", "image", "vision", "golden"]
|
||||
if (id === "reasoning") return ["reasoning", "golden"]
|
||||
return ["tool", "tool-loop", "golden"]
|
||||
}
|
||||
|
||||
|
|
@ -264,6 +284,21 @@ export const runGoldenScenario = (id: GoldenScenarioID, context: GoldenScenarioC
|
|||
return
|
||||
}
|
||||
|
||||
if (id === "reasoning") {
|
||||
const response = yield* generate(
|
||||
reasoningRequest({
|
||||
id: context.id,
|
||||
model: context.model,
|
||||
maxTokens: context.maxTokens ?? 120,
|
||||
temperature: context.temperature,
|
||||
}),
|
||||
)
|
||||
expect(response.text.trim()).toMatch(/^Hello!?$/)
|
||||
expect(response.usage?.reasoningTokens ?? 0).toBeGreaterThan(0)
|
||||
expectFinish(response.events, "stop")
|
||||
return
|
||||
}
|
||||
|
||||
expectGoldenWeatherToolLoop(
|
||||
yield* runWeatherToolLoop(
|
||||
goldenWeatherToolLoopRequest({
|
||||
|
|
@ -293,7 +328,7 @@ const usageSummary = (usage: LLMResponse["usage"] | undefined) => {
|
|||
const pushText = (summary: Array<Record<string, unknown>>, type: "text" | "reasoning", value: string) => {
|
||||
const last = summary.at(-1)
|
||||
if (last?.type === type) {
|
||||
last.value = `${last.value ?? ""}${value}`
|
||||
last.value = `${typeof last.value === "string" ? last.value : ""}${value}`
|
||||
return
|
||||
}
|
||||
summary.push({ type, value })
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue