diff --git a/packages/llm/src/protocols/anthropic-messages.ts b/packages/llm/src/protocols/anthropic-messages.ts index fba785373d..4f02248a9c 100644 --- a/packages/llm/src/protocols/anthropic-messages.ts +++ b/packages/llm/src/protocols/anthropic-messages.ts @@ -364,34 +364,56 @@ const mapFinishReason = (reason: string | null | undefined): FinishReason => { return "unknown" } +// Anthropic reports the non-overlapping breakdown natively — its +// `input_tokens` is the *non-cached* count per the Messages API docs, with +// cache reads and writes as separate fields. We sum them to derive the +// inclusive `inputTokens` the rest of the contract expects. Extended +// thinking tokens are *not* broken out by Anthropic — they're billed as +// part of `output_tokens`, so `reasoningTokens` stays `undefined` and +// `outputTokens` carries the combined total. const mapUsage = (usage: AnthropicUsage | undefined): Usage | undefined => { if (!usage) return undefined + const nonCached = usage.input_tokens + const cacheRead = usage.cache_read_input_tokens ?? undefined + const cacheWrite = usage.cache_creation_input_tokens ?? undefined + const inputTokens = ProviderShared.sumTokens(nonCached, cacheRead, cacheWrite) return new Usage({ - inputTokens: usage.input_tokens, + inputTokens, outputTokens: usage.output_tokens, - cacheReadInputTokens: usage.cache_read_input_tokens ?? undefined, - cacheWriteInputTokens: usage.cache_creation_input_tokens ?? undefined, - totalTokens: ProviderShared.totalTokens(usage.input_tokens, usage.output_tokens, undefined), - native: usage, + nonCachedInputTokens: nonCached, + cacheReadInputTokens: cacheRead, + cacheWriteInputTokens: cacheWrite, + totalTokens: ProviderShared.totalTokens(inputTokens, usage.output_tokens, undefined), + providerMetadata: { anthropic: usage }, }) } // Anthropic emits usage on `message_start` and again on `message_delta` — the // final delta carries the authoritative totals. Right-biased merge: each -// field prefers `right` when defined, falls back to `left`. `totalTokens` is -// recomputed from the merged input/output to stay consistent. +// field prefers `right` when defined, falls back to `left`. `inputTokens` is +// recomputed from the merged breakdown so the inclusive total stays +// consistent with `nonCached + cacheRead + cacheWrite`. const mergeUsage = (left: Usage | undefined, right: Usage | undefined) => { if (!left) return right if (!right) return left - const inputTokens = right.inputTokens ?? left.inputTokens + const nonCachedInputTokens = right.nonCachedInputTokens ?? left.nonCachedInputTokens + const cacheReadInputTokens = right.cacheReadInputTokens ?? left.cacheReadInputTokens + const cacheWriteInputTokens = right.cacheWriteInputTokens ?? left.cacheWriteInputTokens + const inputTokens = ProviderShared.sumTokens(nonCachedInputTokens, cacheReadInputTokens, cacheWriteInputTokens) const outputTokens = right.outputTokens ?? left.outputTokens return new Usage({ inputTokens, outputTokens, - cacheReadInputTokens: right.cacheReadInputTokens ?? left.cacheReadInputTokens, - cacheWriteInputTokens: right.cacheWriteInputTokens ?? left.cacheWriteInputTokens, + nonCachedInputTokens, + cacheReadInputTokens, + cacheWriteInputTokens, totalTokens: ProviderShared.totalTokens(inputTokens, outputTokens, undefined), - native: { ...left.native, ...right.native }, + providerMetadata: { + anthropic: { + ...(left.providerMetadata?.["anthropic"] ?? {}), + ...(right.providerMetadata?.["anthropic"] ?? {}), + }, + }, }) } diff --git a/packages/llm/src/protocols/bedrock-converse.ts b/packages/llm/src/protocols/bedrock-converse.ts index 260ee612cd..eadfda3a0b 100644 --- a/packages/llm/src/protocols/bedrock-converse.ts +++ b/packages/llm/src/protocols/bedrock-converse.ts @@ -363,15 +363,22 @@ const mapFinishReason = (reason: string): FinishReason => { return "unknown" } +// AWS Bedrock Converse reports `inputTokens` (inclusive total) with +// `cacheReadInputTokens` and `cacheWriteInputTokens` as subsets. Pass +// the total through and derive the non-cached breakdown. Bedrock does +// not break reasoning out of `outputTokens` for any current model. const mapUsage = (usage: BedrockUsageSchema | undefined): Usage | undefined => { if (!usage) return undefined + const cacheTotal = (usage.cacheReadInputTokens ?? 0) + (usage.cacheWriteInputTokens ?? 0) + const nonCached = ProviderShared.subtractTokens(usage.inputTokens, cacheTotal) return new Usage({ inputTokens: usage.inputTokens, outputTokens: usage.outputTokens, - totalTokens: ProviderShared.totalTokens(usage.inputTokens, usage.outputTokens, usage.totalTokens), + nonCachedInputTokens: nonCached, cacheReadInputTokens: usage.cacheReadInputTokens, cacheWriteInputTokens: usage.cacheWriteInputTokens, - native: usage, + totalTokens: ProviderShared.totalTokens(usage.inputTokens, usage.outputTokens, usage.totalTokens), + providerMetadata: { bedrock: usage }, }) } diff --git a/packages/llm/src/protocols/gemini.ts b/packages/llm/src/protocols/gemini.ts index 140da521a5..a59b0e5017 100644 --- a/packages/llm/src/protocols/gemini.ts +++ b/packages/llm/src/protocols/gemini.ts @@ -281,15 +281,28 @@ const fromRequest = Effect.fn("Gemini.fromRequest")(function* (request: LLMReque // ============================================================================= // Stream Parsing // ============================================================================= +// Gemini reports `promptTokenCount` (inclusive total) with a +// `cachedContentTokenCount` subset. `candidatesTokenCount` is *exclusive* +// of `thoughtsTokenCount` — visible-only, not a total — so we sum the two +// to produce the inclusive `outputTokens` the rest of the contract expects. +// Output is left undefined when the visible component is missing, so we +// don't fabricate an inclusive number from a partial breakdown. const mapUsage = (usage: GeminiUsage | undefined) => { if (!usage) return undefined + const cached = usage.cachedContentTokenCount + const nonCached = ProviderShared.subtractTokens(usage.promptTokenCount, cached) + const outputTokens = + usage.candidatesTokenCount !== undefined + ? usage.candidatesTokenCount + (usage.thoughtsTokenCount ?? 0) + : undefined return new Usage({ inputTokens: usage.promptTokenCount, - outputTokens: usage.candidatesTokenCount, + outputTokens, + nonCachedInputTokens: nonCached, + cacheReadInputTokens: cached, reasoningTokens: usage.thoughtsTokenCount, - cacheReadInputTokens: usage.cachedContentTokenCount, - totalTokens: ProviderShared.totalTokens(usage.promptTokenCount, usage.candidatesTokenCount, usage.totalTokenCount), - native: usage, + totalTokens: ProviderShared.totalTokens(usage.promptTokenCount, outputTokens, usage.totalTokenCount), + providerMetadata: { google: usage }, }) } diff --git a/packages/llm/src/protocols/openai-chat.ts b/packages/llm/src/protocols/openai-chat.ts index 5d42c0a4e9..133adb503b 100644 --- a/packages/llm/src/protocols/openai-chat.ts +++ b/packages/llm/src/protocols/openai-chat.ts @@ -290,15 +290,24 @@ const mapFinishReason = (reason: string | null | undefined): FinishReason => { return "unknown" } +// OpenAI Chat reports `prompt_tokens` (inclusive total) with a +// `cached_tokens` subset, and `completion_tokens` (inclusive total) with +// a `reasoning_tokens` subset. We pass the inclusive totals through and +// derive the non-cached breakdown so the `LLM.Usage` contract is +// satisfied on both sides. const mapUsage = (usage: OpenAIChatEvent["usage"]): Usage | undefined => { if (!usage) return undefined + const cached = usage.prompt_tokens_details?.cached_tokens + const reasoning = usage.completion_tokens_details?.reasoning_tokens + const nonCached = ProviderShared.subtractTokens(usage.prompt_tokens, cached) return new Usage({ inputTokens: usage.prompt_tokens, outputTokens: usage.completion_tokens, - reasoningTokens: usage.completion_tokens_details?.reasoning_tokens, - cacheReadInputTokens: usage.prompt_tokens_details?.cached_tokens, + nonCachedInputTokens: nonCached, + cacheReadInputTokens: cached, + reasoningTokens: reasoning, totalTokens: ProviderShared.totalTokens(usage.prompt_tokens, usage.completion_tokens, usage.total_tokens), - native: usage, + providerMetadata: { openai: usage }, }) } diff --git a/packages/llm/src/protocols/openai-responses.ts b/packages/llm/src/protocols/openai-responses.ts index 14dc32130c..035cc07713 100644 --- a/packages/llm/src/protocols/openai-responses.ts +++ b/packages/llm/src/protocols/openai-responses.ts @@ -276,15 +276,23 @@ const fromRequest = Effect.fn("OpenAIResponses.fromRequest")(function* (request: // ============================================================================= // Stream Parsing // ============================================================================= +// OpenAI Responses reports `input_tokens` (inclusive total) with a +// `cached_tokens` subset, and `output_tokens` (inclusive total) with a +// `reasoning_tokens` subset. Pass the totals through and derive the +// non-cached breakdown. const mapUsage = (usage: OpenAIResponsesUsage | null | undefined) => { if (!usage) return undefined + const cached = usage.input_tokens_details?.cached_tokens + const reasoning = usage.output_tokens_details?.reasoning_tokens + const nonCached = ProviderShared.subtractTokens(usage.input_tokens, cached) return new Usage({ inputTokens: usage.input_tokens, outputTokens: usage.output_tokens, - reasoningTokens: usage.output_tokens_details?.reasoning_tokens, - cacheReadInputTokens: usage.input_tokens_details?.cached_tokens, + nonCachedInputTokens: nonCached, + cacheReadInputTokens: cached, + reasoningTokens: reasoning, totalTokens: ProviderShared.totalTokens(usage.input_tokens, usage.output_tokens, usage.total_tokens), - native: usage, + providerMetadata: { openai: usage }, }) } diff --git a/packages/llm/src/protocols/shared.ts b/packages/llm/src/protocols/shared.ts index c931353998..a07d38bd19 100644 --- a/packages/llm/src/protocols/shared.ts +++ b/packages/llm/src/protocols/shared.ts @@ -42,6 +42,11 @@ export interface ToolAccumulator { * supplied total; otherwise falls back to `inputTokens + outputTokens` only * when at least one is defined. Returns `undefined` when neither input nor * output is known so routes don't publish a misleading `0`. + * + * Under the `LLM.Usage` contract, `inputTokens` and `outputTokens` are + * inclusive totals, so the computed fallback already covers cache reads / + * writes and reasoning — used mainly for Anthropic-style providers that + * don't surface a top-level total. */ export const totalTokens = ( inputTokens: number | undefined, @@ -53,6 +58,39 @@ export const totalTokens = ( return (inputTokens ?? 0) + (outputTokens ?? 0) } +/** + * Subtract `subtrahend` from `total`, clamping to zero if the provider + * reports a non-sensical breakdown (e.g. `cached_tokens > prompt_tokens`). + * Used by protocol mappers when deriving a non-overlapping breakdown field + * from a provider's inclusive total — `nonCachedInputTokens` from + * `inputTokens - cacheReadInputTokens - cacheWriteInputTokens`. + * + * If `total` is `undefined`, returns `undefined` (we don't fabricate + * counts). If `subtrahend` is `undefined`, returns `total` unchanged. The + * provider-native breakdown stays available on `Usage.providerMetadata` + * for debugging. + */ +export const subtractTokens = ( + total: number | undefined, + subtrahend: number | undefined, +): number | undefined => { + if (total === undefined) return undefined + if (subtrahend === undefined) return total + return Math.max(0, total - subtrahend) +} + +/** + * Sum a list of optional token counts, returning `undefined` only when + * every value is `undefined` (so we don't fabricate a `0`). Used by + * protocol mappers to derive the inclusive `inputTokens` total from a + * provider that natively reports a non-overlapping breakdown + * (e.g. Anthropic, whose `input_tokens` is already non-cached only). + */ +export const sumTokens = (...values: ReadonlyArray): number | undefined => { + if (values.every((value) => value === undefined)) return undefined + return values.reduce((acc, value) => acc + (value ?? 0), 0) +} + export const eventError = (route: string, message: string, raw?: string) => new LLMError({ module: "ProviderShared", diff --git a/packages/llm/src/schema/events.ts b/packages/llm/src/schema/events.ts index d0befe246e..6e6bb1541b 100644 --- a/packages/llm/src/schema/events.ts +++ b/packages/llm/src/schema/events.ts @@ -3,15 +3,70 @@ import { ContentBlockID, FinishReason, ProtocolID, ProviderMetadata, ResponseID, import { ModelRef } from "./options" import { ToolResultValue } from "./messages" +/** + * Token usage reported by an LLM provider. + * + * **Inclusive totals** (match AI SDK / OpenAI / LangChain convention — a + * reader from any of those ecosystems sees the number they expect): + * + * - `inputTokens` — total prompt tokens, *including* cached reads/writes. + * - `outputTokens` — total output tokens, *including* reasoning. + * - `totalTokens` — provider-supplied total, or `inputTokens + outputTokens`. + * + * **Non-overlapping breakdown** (every field is independently meaningful; + * consumers never have to subtract): + * + * - `nonCachedInputTokens` — the "fresh" portion of the prompt. + * - `cacheReadInputTokens` — input tokens served from cache. + * - `cacheWriteInputTokens` — input tokens written to cache. + * - `reasoningTokens` — subset of `outputTokens` spent on hidden reasoning. + * + * **Invariant**: `nonCachedInputTokens + cacheReadInputTokens + + * cacheWriteInputTokens = inputTokens`, and `reasoningTokens ≤ outputTokens`. + * Each protocol mapper computes whichever side it doesn't get natively, + * with `Math.max(0, …)` clamping for defense against provider bugs. Because + * every breakdown field is stored independently, downstream consumers can + * read whatever they need (cost-by-category, context-pressure, AI-SDK-style + * inclusive total) without ever subtracting — eliminating the underflow + * class of bug where a clamped difference would silently store the wrong + * value. + * + * **Semantics by provider**: + * + * - OpenAI Chat / Responses / Gemini / Bedrock: provider reports inclusive + * `inputTokens` and an inclusive `outputTokens`; mapper subtracts to + * derive the breakdown. + * - Anthropic: provider reports the breakdown natively (`input_tokens` is + * non-cached only); mapper sums to derive the inclusive `inputTokens`. + * Anthropic does *not* break extended-thinking out of `output_tokens`, so + * `reasoningTokens` is `undefined` and `outputTokens` carries the + * combined total — a documented limitation of the Anthropic API. + * + * `providerMetadata` always carries the provider's raw usage payload — + * keyed by provider name (`{ openai: ... }`, `{ anthropic: ... }`, etc.) + * — for fields we don't normalize and for billing-level audit trails. + * Matches the same escape-hatch field on `LLMEvent`. + */ export class Usage extends Schema.Class("LLM.Usage")({ inputTokens: Schema.optional(Schema.Number), outputTokens: Schema.optional(Schema.Number), - reasoningTokens: Schema.optional(Schema.Number), + nonCachedInputTokens: Schema.optional(Schema.Number), cacheReadInputTokens: Schema.optional(Schema.Number), cacheWriteInputTokens: Schema.optional(Schema.Number), + reasoningTokens: Schema.optional(Schema.Number), totalTokens: Schema.optional(Schema.Number), - native: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), -}) {} + providerMetadata: Schema.optional(ProviderMetadata), +}) { + /** + * Visible output tokens — `outputTokens` minus `reasoningTokens`, clamped + * to zero. The one place subtraction happens in this contract; the clamp + * means a provider reporting `reasoningTokens > outputTokens` produces a + * harmless zero rather than a negative that crashes downstream schemas. + */ + get visibleOutputTokens() { + return Math.max(0, (this.outputTokens ?? 0) - (this.reasoningTokens ?? 0)) + } +} export const RequestStart = Schema.Struct({ type: Schema.tag("request-start"), diff --git a/packages/llm/test/fixtures/recordings/anthropic-messages/anthropic-haiku-4-5-text.json b/packages/llm/test/fixtures/recordings/anthropic-messages/anthropic-haiku-4-5-text.json new file mode 100644 index 0000000000..dbb29a09fe --- /dev/null +++ b/packages/llm/test/fixtures/recordings/anthropic-messages/anthropic-haiku-4-5-text.json @@ -0,0 +1,38 @@ +{ + "version": 1, + "metadata": { + "name": "anthropic-messages/anthropic-haiku-4-5-text", + "recordedAt": "2026-05-11T02:02:03.804Z", + "provider": "anthropic", + "route": "anthropic-messages", + "transport": "http", + "model": "claude-haiku-4-5-20251001", + "tags": [ + "prefix:anthropic-messages", + "provider:anthropic", + "text", + "golden" + ] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.anthropic.com/v1/messages", + "headers": { + "anthropic-version": "2023-06-01", + "content-type": "application/json" + }, + "body": "{\"model\":\"claude-haiku-4-5-20251001\",\"system\":[{\"type\":\"text\",\"text\":\"You are concise.\"}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Reply exactly with: Hello!\"}]}],\"stream\":true,\"max_tokens\":40,\"temperature\":0}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_01SvRWwb75gDuhBpVMHjnFaf\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":18,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":2,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}} }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"} }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"Hello!\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 }\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":18,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":5} }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n" + } + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/anthropic-messages/anthropic-haiku-4-5-tool-call.json b/packages/llm/test/fixtures/recordings/anthropic-messages/anthropic-haiku-4-5-tool-call.json new file mode 100644 index 0000000000..1b18b62cfc --- /dev/null +++ b/packages/llm/test/fixtures/recordings/anthropic-messages/anthropic-haiku-4-5-tool-call.json @@ -0,0 +1,39 @@ +{ + "version": 1, + "metadata": { + "name": "anthropic-messages/anthropic-haiku-4-5-tool-call", + "recordedAt": "2026-05-11T02:02:04.363Z", + "provider": "anthropic", + "route": "anthropic-messages", + "transport": "http", + "model": "claude-haiku-4-5-20251001", + "tags": [ + "prefix:anthropic-messages", + "provider:anthropic", + "tool", + "tool-call", + "golden" + ] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.anthropic.com/v1/messages", + "headers": { + "anthropic-version": "2023-06-01", + "content-type": "application/json" + }, + "body": "{\"model\":\"claude-haiku-4-5-20251001\",\"system\":[{\"type\":\"text\",\"text\":\"Call tools exactly as requested.\"}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Call get_weather with city exactly Paris.\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}],\"tool_choice\":{\"type\":\"tool\",\"name\":\"get_weather\"},\"stream\":true,\"max_tokens\":80,\"temperature\":0}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_01Lu38yDM3WD8QBQTcg3dBaF\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":677,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":16,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_017Dqk9SAAsHHfiLKsUyitaQ\",\"name\":\"get_weather\",\"input\":{},\"caller\":{\"type\":\"direct\"}} }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"cit\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"y\\\": \\\"Paris\\\"}\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":677,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":33} }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n" + } + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/anthropic-messages/anthropic-opus-4-7-tool-loop.json b/packages/llm/test/fixtures/recordings/anthropic-messages/anthropic-opus-4-7-tool-loop.json new file mode 100644 index 0000000000..6b1cd5082b --- /dev/null +++ b/packages/llm/test/fixtures/recordings/anthropic-messages/anthropic-opus-4-7-tool-loop.json @@ -0,0 +1,59 @@ +{ + "version": 1, + "metadata": { + "name": "anthropic-messages/anthropic-opus-4-7-tool-loop", + "recordedAt": "2026-05-11T02:02:07.788Z", + "provider": "anthropic", + "route": "anthropic-messages", + "transport": "http", + "model": "claude-opus-4-7", + "tags": [ + "prefix:anthropic-messages", + "provider:anthropic", + "flagship", + "tool", + "tool-loop", + "golden" + ] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.anthropic.com/v1/messages", + "headers": { + "anthropic-version": "2023-06-01", + "content-type": "application/json" + }, + "body": "{\"model\":\"claude-opus-4-7\",\"system\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What is the weather in Paris?\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}],\"stream\":true,\"max_tokens\":80}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-opus-4-7\",\"id\":\"msg_01GK5kgi8AuVfRCnQFcXEfV8\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":812,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":0,\"service_tier\":\"standard\",\"inference_geo\":\"global\"}} }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_01BnVDAp13NU8ZdJ9JeJ7byF\",\"name\":\"get_weather\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"c\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"ity\\\": \\\"\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"Paris\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 }\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":812,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":66} }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n" + } + }, + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.anthropic.com/v1/messages", + "headers": { + "anthropic-version": "2023-06-01", + "content-type": "application/json" + }, + "body": "{\"model\":\"claude-opus-4-7\",\"system\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What is the weather in Paris?\"}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_01BnVDAp13NU8ZdJ9JeJ7byF\",\"name\":\"get_weather\",\"input\":{\"city\":\"Paris\"}}]},{\"role\":\"user\",\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_01BnVDAp13NU8ZdJ9JeJ7byF\",\"content\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}],\"stream\":true,\"max_tokens\":80}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-opus-4-7\",\"id\":\"msg_01237VTnjPeYSRh31UjXWaEa\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":909,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":8,\"service_tier\":\"standard\",\"inference_geo\":\"global\"}} }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"} }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"Paris is sunny.\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 }\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":909,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":12} }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n" + } + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/anthropic-messages/rejects-malformed-assistant-tool-order.json b/packages/llm/test/fixtures/recordings/anthropic-messages/rejects-malformed-assistant-tool-order.json new file mode 100644 index 0000000000..a391c4ce83 --- /dev/null +++ b/packages/llm/test/fixtures/recordings/anthropic-messages/rejects-malformed-assistant-tool-order.json @@ -0,0 +1,35 @@ +{ + "version": 1, + "metadata": { + "name": "anthropic-messages/rejects-malformed-assistant-tool-order", + "recordedAt": "2026-05-11T02:01:44.544Z", + "tags": [ + "prefix:anthropic-messages", + "provider:anthropic", + "protocol:anthropic-messages", + "tool", + "sad-path" + ] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.anthropic.com/v1/messages", + "headers": { + "anthropic-version": "2023-06-01", + "content-type": "application/json" + }, + "body": "{\"model\":\"claude-haiku-4-5-20251001\",\"messages\":[{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"call_1\",\"name\":\"get_weather\",\"input\":{\"city\":\"Paris\"}},{\"type\":\"text\",\"text\":\"I will check the weather.\"}]},{\"role\":\"user\",\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"call_1\",\"content\":\"{\\\"temperature\\\":\\\"72F\\\"}\"}]},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Use that result to answer briefly.\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get weather\",\"input_schema\":{\"type\":\"object\",\"properties\":{}}}],\"stream\":true,\"max_tokens\":4096}" + }, + "response": { + "status": 400, + "headers": { + "content-type": "application/json" + }, + "body": "{\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"messages.1: `tool_use` ids were found without `tool_result` blocks immediately after: call_1. Each `tool_use` block must have a corresponding `tool_result` block in the next message.\"},\"request_id\":\"req_011CauxVdQf3N2PPFJ5aH8Bh\"}" + } + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/gemini/gemini-2-5-flash-text.json b/packages/llm/test/fixtures/recordings/gemini/gemini-2-5-flash-text.json new file mode 100644 index 0000000000..6e0588cca5 --- /dev/null +++ b/packages/llm/test/fixtures/recordings/gemini/gemini-2-5-flash-text.json @@ -0,0 +1,37 @@ +{ + "version": 1, + "metadata": { + "name": "gemini/gemini-2-5-flash-text", + "recordedAt": "2026-05-11T02:02:08.410Z", + "provider": "google", + "route": "gemini", + "transport": "http", + "model": "gemini-2.5-flash", + "tags": [ + "prefix:gemini", + "provider:google", + "text", + "golden" + ] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:streamGenerateContent?alt=sse", + "headers": { + "content-type": "application/json" + }, + "body": "{\"contents\":[{\"role\":\"user\",\"parts\":[{\"text\":\"Reply exactly with: Hello!\"}]}],\"systemInstruction\":{\"parts\":[{\"text\":\"You are concise.\"}]},\"generationConfig\":{\"maxOutputTokens\":80,\"temperature\":0}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"Hello!\"}],\"role\": \"model\"},\"finishReason\": \"STOP\",\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 11,\"candidatesTokenCount\": 2,\"totalTokenCount\": 13,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 11}],\"serviceTier\": \"standard\"},\"modelVersion\": \"gemini-2.5-flash\",\"responseId\": \"nzgBatP3OZmW-8YP567bqQs\"}\r\n\r\n" + } + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/gemini/gemini-2-5-flash-tool-call.json b/packages/llm/test/fixtures/recordings/gemini/gemini-2-5-flash-tool-call.json new file mode 100644 index 0000000000..a5902e50d6 --- /dev/null +++ b/packages/llm/test/fixtures/recordings/gemini/gemini-2-5-flash-tool-call.json @@ -0,0 +1,38 @@ +{ + "version": 1, + "metadata": { + "name": "gemini/gemini-2-5-flash-tool-call", + "recordedAt": "2026-05-11T02:02:09.308Z", + "provider": "google", + "route": "gemini", + "transport": "http", + "model": "gemini-2.5-flash", + "tags": [ + "prefix:gemini", + "provider:google", + "tool", + "tool-call", + "golden" + ] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:streamGenerateContent?alt=sse", + "headers": { + "content-type": "application/json" + }, + "body": "{\"contents\":[{\"role\":\"user\",\"parts\":[{\"text\":\"Call get_weather with city exactly Paris.\"}]}],\"systemInstruction\":{\"parts\":[{\"text\":\"Call tools exactly as requested.\"}]},\"tools\":[{\"functionDeclarations\":[{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"required\":[\"city\"],\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}}}}]}],\"toolConfig\":{\"functionCallingConfig\":{\"mode\":\"ANY\",\"allowedFunctionNames\":[\"get_weather\"]}},\"generationConfig\":{\"maxOutputTokens\":80,\"temperature\":0}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": "data: {\"candidates\": [{\"content\": {\"parts\": [{\"functionCall\": {\"name\": \"get_weather\",\"args\": {\"city\": \"Paris\"}},\"thoughtSignature\": \"CiQBDDnWx/X6sWeX2joSugyWO3L/lt0AgIPCvhpqf3845fj+H70KXwEMOdbHB/cnaqYCro0pU+yLWoA55jhuwoLmTcnYm4Qzcm5DuW/v0NUyz8RDx6DFh61juENveUztly6yc6/XiWJHtsgncd9YgcZhuQKqtp5KZTkGYpT3g6v3yP9GK4AoCoUBAQw51sePh3WuWovHnwIotKLVZiU9pwh34k4FY7ugPOxyDAG9j69cy7BYYzSchI10LEjLoLlCMZuNIPootBgI02QWY/4h2PIv33BAADrFPM2T3aE4cAuMoa3GCu2nztJ/95junDhIhuXZSQ/Mh9EVxpx7ml99Z7Hxb7OtDsSZZLeCBuGmSw==\"}],\"role\": \"model\"},\"finishReason\": \"STOP\",\"index\": 0,\"finishMessage\": \"Model generated function call(s).\"}],\"usageMetadata\": {\"promptTokenCount\": 55,\"candidatesTokenCount\": 15,\"totalTokenCount\": 115,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 55}],\"thoughtsTokenCount\": 45,\"serviceTier\": \"standard\"},\"modelVersion\": \"gemini-2.5-flash\",\"responseId\": \"oDgBauj6HZ3B-8YPpaOc6QU\"}\r\n\r\n" + } + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/openai-chat/openai-chat-gpt-4o-mini-text.json b/packages/llm/test/fixtures/recordings/openai-chat/openai-chat-gpt-4o-mini-text.json new file mode 100644 index 0000000000..b908a120e8 --- /dev/null +++ b/packages/llm/test/fixtures/recordings/openai-chat/openai-chat-gpt-4o-mini-text.json @@ -0,0 +1,37 @@ +{ + "version": 1, + "metadata": { + "name": "openai-chat/openai-chat-gpt-4o-mini-text", + "recordedAt": "2026-05-11T02:01:46.536Z", + "provider": "openai", + "route": "openai-chat", + "transport": "http", + "model": "gpt-4o-mini", + "tags": [ + "prefix:openai-chat", + "provider:openai", + "text", + "golden" + ] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"system\",\"content\":\"You are concise.\"},{\"role\":\"user\",\"content\":\"Reply exactly with: Hello!\"}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"max_tokens\":40,\"temperature\":0}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "data: {\"id\":\"chatcmpl-DeAFmC9UiBnVu6dAjBvWVBVypm2gc\",\"object\":\"chat.completion.chunk\",\"created\":1778464906,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_3ef558a83f\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"wHajUz1js\"}\n\ndata: {\"id\":\"chatcmpl-DeAFmC9UiBnVu6dAjBvWVBVypm2gc\",\"object\":\"chat.completion.chunk\",\"created\":1778464906,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_3ef558a83f\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hello\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"lVHzGq\"}\n\ndata: {\"id\":\"chatcmpl-DeAFmC9UiBnVu6dAjBvWVBVypm2gc\",\"object\":\"chat.completion.chunk\",\"created\":1778464906,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_3ef558a83f\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"!\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"3gaQRWEjE7\"}\n\ndata: {\"id\":\"chatcmpl-DeAFmC9UiBnVu6dAjBvWVBVypm2gc\",\"object\":\"chat.completion.chunk\",\"created\":1778464906,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_3ef558a83f\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":null,\"obfuscation\":\"vuKPZ\"}\n\ndata: {\"id\":\"chatcmpl-DeAFmC9UiBnVu6dAjBvWVBVypm2gc\",\"object\":\"chat.completion.chunk\",\"created\":1778464906,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_3ef558a83f\",\"choices\":[],\"usage\":{\"prompt_tokens\":21,\"completion_tokens\":2,\"total_tokens\":23,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"kfFsssdujmM\"}\n\ndata: [DONE]\n\n" + } + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/openai-chat/openai-chat-gpt-4o-mini-tool-call.json b/packages/llm/test/fixtures/recordings/openai-chat/openai-chat-gpt-4o-mini-tool-call.json new file mode 100644 index 0000000000..1ed411fb96 --- /dev/null +++ b/packages/llm/test/fixtures/recordings/openai-chat/openai-chat-gpt-4o-mini-tool-call.json @@ -0,0 +1,38 @@ +{ + "version": 1, + "metadata": { + "name": "openai-chat/openai-chat-gpt-4o-mini-tool-call", + "recordedAt": "2026-05-11T02:01:47.484Z", + "provider": "openai", + "route": "openai-chat", + "transport": "http", + "model": "gpt-4o-mini", + "tags": [ + "prefix:openai-chat", + "provider:openai", + "tool", + "tool-call", + "golden" + ] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"system\",\"content\":\"Call tools exactly as requested.\"},{\"role\":\"user\",\"content\":\"Call get_weather with city exactly Paris.\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"tool_choice\":{\"type\":\"function\",\"function\":{\"name\":\"get_weather\"}},\"stream\":true,\"stream_options\":{\"include_usage\":true},\"max_tokens\":80,\"temperature\":0}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "data: {\"id\":\"chatcmpl-DeAFnVMdELyuymMRlTqN5dBdM7eGd\",\"object\":\"chat.completion.chunk\",\"created\":1778464907,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_d88f6e55bb\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"index\":0,\"id\":\"call_sH8T7MPdJXS5KJginKrzexL5\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}],\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"d0rJ\"}\n\ndata: {\"id\":\"chatcmpl-DeAFnVMdELyuymMRlTqN5dBdM7eGd\",\"object\":\"chat.completion.chunk\",\"created\":1778464907,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_d88f6e55bb\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"hL82erd6bBoy91\"}\n\ndata: {\"id\":\"chatcmpl-DeAFnVMdELyuymMRlTqN5dBdM7eGd\",\"object\":\"chat.completion.chunk\",\"created\":1778464907,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_d88f6e55bb\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"city\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"e3aoKH4tvJlXW\"}\n\ndata: {\"id\":\"chatcmpl-DeAFnVMdELyuymMRlTqN5dBdM7eGd\",\"object\":\"chat.completion.chunk\",\"created\":1778464907,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_d88f6e55bb\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\":\\\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"T8EzFVNaUCLE\"}\n\ndata: {\"id\":\"chatcmpl-DeAFnVMdELyuymMRlTqN5dBdM7eGd\",\"object\":\"chat.completion.chunk\",\"created\":1778464907,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_d88f6e55bb\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"Paris\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"lelRBxF08Zes\"}\n\ndata: {\"id\":\"chatcmpl-DeAFnVMdELyuymMRlTqN5dBdM7eGd\",\"object\":\"chat.completion.chunk\",\"created\":1778464907,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_d88f6e55bb\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"}\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"F1xO8sVMyZt4BU\"}\n\ndata: {\"id\":\"chatcmpl-DeAFnVMdELyuymMRlTqN5dBdM7eGd\",\"object\":\"chat.completion.chunk\",\"created\":1778464907,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_d88f6e55bb\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":null,\"obfuscation\":\"YL3pl\"}\n\ndata: {\"id\":\"chatcmpl-DeAFnVMdELyuymMRlTqN5dBdM7eGd\",\"object\":\"chat.completion.chunk\",\"created\":1778464907,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_d88f6e55bb\",\"choices\":[],\"usage\":{\"prompt_tokens\":67,\"completion_tokens\":5,\"total_tokens\":72,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"7cov9qkofwo\"}\n\ndata: [DONE]\n\n" + } + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/openai-chat/openai-chat-gpt-4o-mini-tool-loop.json b/packages/llm/test/fixtures/recordings/openai-chat/openai-chat-gpt-4o-mini-tool-loop.json new file mode 100644 index 0000000000..1cdc46f30c --- /dev/null +++ b/packages/llm/test/fixtures/recordings/openai-chat/openai-chat-gpt-4o-mini-tool-loop.json @@ -0,0 +1,56 @@ +{ + "version": 1, + "metadata": { + "name": "openai-chat/openai-chat-gpt-4o-mini-tool-loop", + "recordedAt": "2026-05-11T02:01:50.433Z", + "provider": "openai", + "route": "openai-chat", + "transport": "http", + "model": "gpt-4o-mini", + "tags": [ + "prefix:openai-chat", + "provider:openai", + "tool", + "tool-loop", + "golden" + ] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"max_tokens\":80,\"temperature\":0}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "data: {\"id\":\"chatcmpl-DeAFo2rbsuuMn0ftzxwfDpqnYANrT\",\"object\":\"chat.completion.chunk\",\"created\":1778464908,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c83829e916\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"index\":0,\"id\":\"call_HrDZhrMUauvVddKWzVQFJ69Q\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}],\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"aAqC\"}\n\ndata: {\"id\":\"chatcmpl-DeAFo2rbsuuMn0ftzxwfDpqnYANrT\",\"object\":\"chat.completion.chunk\",\"created\":1778464908,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c83829e916\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"haZ3pKk14oDay0\"}\n\ndata: {\"id\":\"chatcmpl-DeAFo2rbsuuMn0ftzxwfDpqnYANrT\",\"object\":\"chat.completion.chunk\",\"created\":1778464908,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c83829e916\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"city\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"ZUJHQytFyDezp\"}\n\ndata: {\"id\":\"chatcmpl-DeAFo2rbsuuMn0ftzxwfDpqnYANrT\",\"object\":\"chat.completion.chunk\",\"created\":1778464908,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c83829e916\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\":\\\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"3buqnStPteyG\"}\n\ndata: {\"id\":\"chatcmpl-DeAFo2rbsuuMn0ftzxwfDpqnYANrT\",\"object\":\"chat.completion.chunk\",\"created\":1778464908,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c83829e916\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"Paris\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"7epPqHEU3OeA\"}\n\ndata: {\"id\":\"chatcmpl-DeAFo2rbsuuMn0ftzxwfDpqnYANrT\",\"object\":\"chat.completion.chunk\",\"created\":1778464908,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c83829e916\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"}\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"gSU5gyl9K7sUCv\"}\n\ndata: {\"id\":\"chatcmpl-DeAFo2rbsuuMn0ftzxwfDpqnYANrT\",\"object\":\"chat.completion.chunk\",\"created\":1778464908,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c83829e916\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"tool_calls\"}],\"usage\":null,\"obfuscation\":\"P3oSvByfy60JCsz\"}\n\ndata: {\"id\":\"chatcmpl-DeAFo2rbsuuMn0ftzxwfDpqnYANrT\",\"object\":\"chat.completion.chunk\",\"created\":1778464908,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c83829e916\",\"choices\":[],\"usage\":{\"prompt_tokens\":71,\"completion_tokens\":14,\"total_tokens\":85,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"5AhQ5ToYra\"}\n\ndata: [DONE]\n\n" + } + }, + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_HrDZhrMUauvVddKWzVQFJ69Q\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_HrDZhrMUauvVddKWzVQFJ69Q\",\"content\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"max_tokens\":80,\"temperature\":0}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "data: {\"id\":\"chatcmpl-DeAFpaPsUwxt5hjwGgC4BBmlALxwR\",\"object\":\"chat.completion.chunk\",\"created\":1778464909,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c83829e916\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"2QWjdWPSe\"}\n\ndata: {\"id\":\"chatcmpl-DeAFpaPsUwxt5hjwGgC4BBmlALxwR\",\"object\":\"chat.completion.chunk\",\"created\":1778464909,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c83829e916\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Paris\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"sm5IpQ\"}\n\ndata: {\"id\":\"chatcmpl-DeAFpaPsUwxt5hjwGgC4BBmlALxwR\",\"object\":\"chat.completion.chunk\",\"created\":1778464909,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c83829e916\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" is\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"cWsGxqHw\"}\n\ndata: {\"id\":\"chatcmpl-DeAFpaPsUwxt5hjwGgC4BBmlALxwR\",\"object\":\"chat.completion.chunk\",\"created\":1778464909,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c83829e916\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" sunny\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"8hNCi\"}\n\ndata: {\"id\":\"chatcmpl-DeAFpaPsUwxt5hjwGgC4BBmlALxwR\",\"object\":\"chat.completion.chunk\",\"created\":1778464909,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c83829e916\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"4yKCNDRcwq\"}\n\ndata: {\"id\":\"chatcmpl-DeAFpaPsUwxt5hjwGgC4BBmlALxwR\",\"object\":\"chat.completion.chunk\",\"created\":1778464909,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c83829e916\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":null,\"obfuscation\":\"NvRgg\"}\n\ndata: {\"id\":\"chatcmpl-DeAFpaPsUwxt5hjwGgC4BBmlALxwR\",\"object\":\"chat.completion.chunk\",\"created\":1778464909,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c83829e916\",\"choices\":[],\"usage\":{\"prompt_tokens\":103,\"completion_tokens\":5,\"total_tokens\":108,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"4s7mLtNpZ\"}\n\ndata: [DONE]\n\n" + } + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/openai-compatible-chat/deepseek-chat-text.json b/packages/llm/test/fixtures/recordings/openai-compatible-chat/deepseek-chat-text.json new file mode 100644 index 0000000000..bc91ce6342 --- /dev/null +++ b/packages/llm/test/fixtures/recordings/openai-compatible-chat/deepseek-chat-text.json @@ -0,0 +1,37 @@ +{ + "version": 1, + "metadata": { + "name": "openai-compatible-chat/deepseek-chat-text", + "recordedAt": "2026-05-11T02:02:10.220Z", + "provider": "deepseek", + "route": "openai-compatible-chat", + "transport": "http", + "model": "deepseek-chat", + "tags": [ + "prefix:openai-compatible-chat", + "provider:deepseek", + "text", + "golden" + ] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.deepseek.com/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"deepseek-chat\",\"messages\":[{\"role\":\"system\",\"content\":\"You are concise.\"},{\"role\":\"user\",\"content\":\"Reply exactly with: Hello!\"}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"max_tokens\":40,\"temperature\":0}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "data: {\"id\":\"159019a5-f981-4103-8b3c-2be37cefc505\",\"object\":\"chat.completion.chunk\",\"created\":1778464929,\"model\":\"deepseek-v4-flash\",\"system_fingerprint\":\"fp_058df29938_prod0820_fp8_kvcache_20260402\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}\n\ndata: {\"id\":\"159019a5-f981-4103-8b3c-2be37cefc505\",\"object\":\"chat.completion.chunk\",\"created\":1778464929,\"model\":\"deepseek-v4-flash\",\"system_fingerprint\":\"fp_058df29938_prod0820_fp8_kvcache_20260402\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hello\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}\n\ndata: {\"id\":\"159019a5-f981-4103-8b3c-2be37cefc505\",\"object\":\"chat.completion.chunk\",\"created\":1778464929,\"model\":\"deepseek-v4-flash\",\"system_fingerprint\":\"fp_058df29938_prod0820_fp8_kvcache_20260402\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"!\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}\n\ndata: {\"id\":\"159019a5-f981-4103-8b3c-2be37cefc505\",\"object\":\"chat.completion.chunk\",\"created\":1778464929,\"model\":\"deepseek-v4-flash\",\"system_fingerprint\":\"fp_058df29938_prod0820_fp8_kvcache_20260402\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\"},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":14,\"completion_tokens\":2,\"total_tokens\":16,\"prompt_tokens_details\":{\"cached_tokens\":0},\"prompt_cache_hit_tokens\":0,\"prompt_cache_miss_tokens\":14}}\n\ndata: [DONE]\n\n" + } + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/openai-compatible-chat/togetherai-llama-3-3-70b-text.json b/packages/llm/test/fixtures/recordings/openai-compatible-chat/togetherai-llama-3-3-70b-text.json new file mode 100644 index 0000000000..585ae1bb19 --- /dev/null +++ b/packages/llm/test/fixtures/recordings/openai-compatible-chat/togetherai-llama-3-3-70b-text.json @@ -0,0 +1,37 @@ +{ + "version": 1, + "metadata": { + "name": "openai-compatible-chat/togetherai-llama-3-3-70b-text", + "recordedAt": "2026-05-11T02:02:13.341Z", + "provider": "togetherai", + "route": "openai-compatible-chat", + "transport": "http", + "model": "meta-llama/Llama-3.3-70B-Instruct-Turbo", + "tags": [ + "prefix:openai-compatible-chat", + "provider:togetherai", + "text", + "golden" + ] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.together.xyz/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"meta-llama/Llama-3.3-70B-Instruct-Turbo\",\"messages\":[{\"role\":\"system\",\"content\":\"You are concise.\"},{\"role\":\"user\",\"content\":\"Reply exactly with: Hello!\"}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"max_tokens\":40,\"temperature\":0}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream;charset=utf-8" + }, + "body": "data: {\"id\":\"oibreET-3pDw3Z-9f9d9996ce37066b\",\"object\":\"chat.completion.chunk\",\"created\":1778464931,\"choices\":[{\"index\":0,\"text\":\"Hello\",\"logprobs\":null,\"finish_reason\":null,\"seed\":null,\"delta\":{\"token_id\":9906,\"role\":\"assistant\",\"content\":\"Hello\"}}],\"model\":\"meta-llama/Llama-3.3-70B-Instruct-Turbo\",\"usage\":null}\n\ndata: {\"id\":\"oibreET-3pDw3Z-9f9d9996ce37066b\",\"object\":\"chat.completion.chunk\",\"created\":1778464931,\"choices\":[{\"index\":0,\"text\":\"!\",\"logprobs\":null,\"finish_reason\":null,\"seed\":null,\"delta\":{\"token_id\":null,\"role\":\"assistant\",\"content\":\"!\"}}],\"model\":\"meta-llama/Llama-3.3-70B-Instruct-Turbo\",\"usage\":null}\n\ndata: {\"id\":\"oibreET-3pDw3Z-9f9d9996ce37066b\",\"object\":\"chat.completion.chunk\",\"created\":1778464931,\"choices\":[{\"index\":0,\"text\":\"\",\"logprobs\":null,\"finish_reason\":\"stop\",\"seed\":12144769634208630000,\"delta\":{\"token_id\":128009,\"role\":\"assistant\",\"content\":\"\"}}],\"model\":\"meta-llama/Llama-3.3-70B-Instruct-Turbo\",\"usage\":{\"prompt_tokens\":45,\"completion_tokens\":3,\"total_tokens\":48,\"cached_tokens\":0}}\n\ndata: [DONE]\n\n" + } + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/openai-compatible-chat/togetherai-llama-3-3-70b-tool-call.json b/packages/llm/test/fixtures/recordings/openai-compatible-chat/togetherai-llama-3-3-70b-tool-call.json new file mode 100644 index 0000000000..8a19ef5fdc --- /dev/null +++ b/packages/llm/test/fixtures/recordings/openai-compatible-chat/togetherai-llama-3-3-70b-tool-call.json @@ -0,0 +1,38 @@ +{ + "version": 1, + "metadata": { + "name": "openai-compatible-chat/togetherai-llama-3-3-70b-tool-call", + "recordedAt": "2026-05-11T02:02:14.453Z", + "provider": "togetherai", + "route": "openai-compatible-chat", + "transport": "http", + "model": "meta-llama/Llama-3.3-70B-Instruct-Turbo", + "tags": [ + "prefix:openai-compatible-chat", + "provider:togetherai", + "tool", + "tool-call", + "golden" + ] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.together.xyz/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"meta-llama/Llama-3.3-70B-Instruct-Turbo\",\"messages\":[{\"role\":\"system\",\"content\":\"Call tools exactly as requested.\"},{\"role\":\"user\",\"content\":\"Call get_weather with city exactly Paris.\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"tool_choice\":{\"type\":\"function\",\"function\":{\"name\":\"get_weather\"}},\"stream\":true,\"stream_options\":{\"include_usage\":true},\"max_tokens\":80,\"temperature\":0}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream;charset=utf-8" + }, + "body": "data: {\"id\":\"oibreu3-6Ng1vN-9f9d99a9ba73066b\",\"object\":\"chat.completion.chunk\",\"created\":1778464933,\"choices\":[{\"index\":0,\"role\":\"assistant\",\"text\":\"\",\"logprobs\":null,\"finish_reason\":null,\"delta\":{\"token_id\":null,\"role\":\"assistant\",\"content\":\"\"}}],\"model\":\"meta-llama/Llama-3.3-70B-Instruct-Turbo\"}\n\ndata: {\"id\":\"oibreu3-6Ng1vN-9f9d99a9ba73066b\",\"object\":\"chat.completion.chunk\",\"created\":1778464933,\"choices\":[{\"index\":0,\"text\":\"\",\"logprobs\":null,\"finish_reason\":null,\"delta\":{\"token_id\":null,\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"index\":0,\"id\":\"call_jue52dtu6iozr0ny9rq357u5\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]}}],\"model\":\"meta-llama/Llama-3.3-70B-Instruct-Turbo\"}\n\ndata: {\"id\":\"oibreu3-6Ng1vN-9f9d99a9ba73066b\",\"object\":\"chat.completion.chunk\",\"created\":1778464933,\"choices\":[{\"index\":0,\"text\":\"\",\"logprobs\":null,\"finish_reason\":\"tool_calls\",\"delta\":{\"token_id\":null,\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}]}}],\"model\":\"meta-llama/Llama-3.3-70B-Instruct-Turbo\"}\n\ndata: {\"id\":\"oibreu3-6Ng1vN-9f9d99a9ba73066b\",\"object\":\"chat.completion.chunk\",\"created\":1778464933,\"choices\":[{\"index\":0,\"text\":\"\",\"logprobs\":null,\"finish_reason\":\"tool_calls\",\"seed\":17440360718047570000,\"delta\":{\"token_id\":128009,\"role\":\"assistant\",\"content\":\"\"}}],\"model\":\"meta-llama/Llama-3.3-70B-Instruct-Turbo\",\"usage\":{\"prompt_tokens\":194,\"completion_tokens\":19,\"total_tokens\":213,\"cached_tokens\":0}}\n\ndata: [DONE]\n\n" + } + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/openai-responses-websocket/openai-responses-websocket-gpt-4-1-mini-tool-loop.json b/packages/llm/test/fixtures/recordings/openai-responses-websocket/openai-responses-websocket-gpt-4-1-mini-tool-loop.json new file mode 100644 index 0000000000..88c7e6aa69 --- /dev/null +++ b/packages/llm/test/fixtures/recordings/openai-responses-websocket/openai-responses-websocket-gpt-4-1-mini-tool-loop.json @@ -0,0 +1,143 @@ +{ + "version": 1, + "metadata": { + "name": "openai-responses-websocket/openai-responses-websocket-gpt-4-1-mini-tool-loop", + "recordedAt": "2026-05-11T02:02:03.284Z", + "provider": "openai", + "route": "openai-responses-websocket", + "transport": "websocket", + "model": "gpt-4.1-mini", + "tags": [ + "prefix:openai-responses-websocket", + "provider:openai", + "transport:websocket", + "tool", + "tool-loop", + "golden" + ] + }, + "interactions": [ + { + "transport": "websocket", + "open": { + "url": "wss://api.openai.com/v1/responses", + "headers": {} + }, + "client": [ + { + "kind": "text", + "body": "{\"type\":\"response.create\",\"model\":\"gpt-4.1-mini\",\"input\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"},{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"What is the weather in Paris?\"}]}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}],\"store\":false,\"max_output_tokens\":80,\"temperature\":0}" + } + ], + "server": [ + { + "kind": "text", + "body": "{\"type\":\"response.created\",\"response\":{\"id\":\"resp_0f8d81b5d3287513016a013897415481a28585571d6710366f\",\"object\":\"response\",\"created_at\":1778464919,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":80,\"max_tool_calls\":null,\"model\":\"gpt-4.1-mini-2025-04-14\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0f8d81b5d3287513016a013897415481a28585571d6710366f\",\"object\":\"response\",\"created_at\":1778464919,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":80,\"max_tool_calls\":null,\"model\":\"gpt-4.1-mini-2025-04-14\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_0f8d81b5d3287513016a01389815f081a2a68583ed9b19c405\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_wRJoUdcb6w5SIpieBFOKxF6z\",\"name\":\"get_weather\"},\"output_index\":0,\"sequence_number\":2}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_0f8d81b5d3287513016a01389815f081a2a68583ed9b19c405\",\"obfuscation\":\"EAdvUfYaysnNd3\",\"output_index\":0,\"sequence_number\":3}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.function_call_arguments.delta\",\"delta\":\"city\",\"item_id\":\"fc_0f8d81b5d3287513016a01389815f081a2a68583ed9b19c405\",\"obfuscation\":\"b0QWJJlscLyl\",\"output_index\":0,\"sequence_number\":4}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_0f8d81b5d3287513016a01389815f081a2a68583ed9b19c405\",\"obfuscation\":\"m5EV1wzceCkjb\",\"output_index\":0,\"sequence_number\":5}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Paris\",\"item_id\":\"fc_0f8d81b5d3287513016a01389815f081a2a68583ed9b19c405\",\"obfuscation\":\"QM3FjPoN9Gi\",\"output_index\":0,\"sequence_number\":6}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"}\",\"item_id\":\"fc_0f8d81b5d3287513016a01389815f081a2a68583ed9b19c405\",\"obfuscation\":\"lug74orAYXtg0e\",\"output_index\":0,\"sequence_number\":7}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\",\"item_id\":\"fc_0f8d81b5d3287513016a01389815f081a2a68583ed9b19c405\",\"output_index\":0,\"sequence_number\":8}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_0f8d81b5d3287513016a01389815f081a2a68583ed9b19c405\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\",\"call_id\":\"call_wRJoUdcb6w5SIpieBFOKxF6z\",\"name\":\"get_weather\"},\"output_index\":0,\"sequence_number\":9}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0f8d81b5d3287513016a013897415481a28585571d6710366f\",\"object\":\"response\",\"created_at\":1778464919,\"status\":\"completed\",\"background\":false,\"completed_at\":1778464920,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":80,\"max_tool_calls\":null,\"model\":\"gpt-4.1-mini-2025-04-14\",\"moderation\":null,\"output\":[{\"id\":\"fc_0f8d81b5d3287513016a01389815f081a2a68583ed9b19c405\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\",\"call_id\":\"call_wRJoUdcb6w5SIpieBFOKxF6z\",\"name\":\"get_weather\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":69,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":15,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":84},\"user\":null,\"metadata\":{}},\"sequence_number\":10}" + } + ] + }, + { + "transport": "websocket", + "open": { + "url": "wss://api.openai.com/v1/responses", + "headers": {} + }, + "client": [ + { + "kind": "text", + "body": "{\"type\":\"response.create\",\"model\":\"gpt-4.1-mini\",\"input\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"},{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"What is the weather in Paris?\"}]},{\"type\":\"function_call\",\"call_id\":\"call_wRJoUdcb6w5SIpieBFOKxF6z\",\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"},{\"type\":\"function_call_output\",\"call_id\":\"call_wRJoUdcb6w5SIpieBFOKxF6z\",\"output\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}],\"store\":false,\"max_output_tokens\":80,\"temperature\":0}" + } + ], + "server": [ + { + "kind": "text", + "body": "{\"type\":\"response.created\",\"response\":{\"id\":\"resp_0767cfe3f5d98b2a016a0138994258819485d082e5c78849a4\",\"object\":\"response\",\"created_at\":1778464921,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":80,\"max_tool_calls\":null,\"model\":\"gpt-4.1-mini-2025-04-14\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0767cfe3f5d98b2a016a0138994258819485d082e5c78849a4\",\"object\":\"response\",\"created_at\":1778464921,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":80,\"max_tool_calls\":null,\"model\":\"gpt-4.1-mini-2025-04-14\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0767cfe3f5d98b2a016a01389b1434819493f3d33e0ec6973c\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_0767cfe3f5d98b2a016a01389b1434819493f3d33e0ec6973c\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"Paris\",\"item_id\":\"msg_0767cfe3f5d98b2a016a01389b1434819493f3d33e0ec6973c\",\"logprobs\":[],\"obfuscation\":\"fWbkBKTZ5oG\",\"output_index\":0,\"sequence_number\":4}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" is\",\"item_id\":\"msg_0767cfe3f5d98b2a016a01389b1434819493f3d33e0ec6973c\",\"logprobs\":[],\"obfuscation\":\"BGuPlDrPchXwG\",\"output_index\":0,\"sequence_number\":5}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" sunny\",\"item_id\":\"msg_0767cfe3f5d98b2a016a01389b1434819493f3d33e0ec6973c\",\"logprobs\":[],\"obfuscation\":\"7THzde2pni\",\"output_index\":0,\"sequence_number\":6}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\".\",\"item_id\":\"msg_0767cfe3f5d98b2a016a01389b1434819493f3d33e0ec6973c\",\"logprobs\":[],\"obfuscation\":\"Eo38bSElfyNmljj\",\"output_index\":0,\"sequence_number\":7}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0767cfe3f5d98b2a016a01389b1434819493f3d33e0ec6973c\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":8,\"text\":\"Paris is sunny.\"}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_0767cfe3f5d98b2a016a01389b1434819493f3d33e0ec6973c\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"Paris is sunny.\"},\"sequence_number\":9}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_0767cfe3f5d98b2a016a01389b1434819493f3d33e0ec6973c\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"Paris is sunny.\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":10}" + }, + { + "kind": "text", + "body": "{\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0767cfe3f5d98b2a016a0138994258819485d082e5c78849a4\",\"object\":\"response\",\"created_at\":1778464921,\"status\":\"completed\",\"background\":false,\"completed_at\":1778464923,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":80,\"max_tool_calls\":null,\"model\":\"gpt-4.1-mini-2025-04-14\",\"moderation\":null,\"output\":[{\"id\":\"msg_0767cfe3f5d98b2a016a01389b1434819493f3d33e0ec6973c\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"Paris is sunny.\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":99,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":6,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":105},\"user\":null,\"metadata\":{}},\"sequence_number\":11}" + } + ] + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-text.json b/packages/llm/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-text.json new file mode 100644 index 0000000000..befd07c850 --- /dev/null +++ b/packages/llm/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-text.json @@ -0,0 +1,38 @@ +{ + "version": 1, + "metadata": { + "name": "openai-responses/openai-responses-gpt-5-5-text", + "recordedAt": "2026-05-11T02:01:52.043Z", + "provider": "openai", + "route": "openai-responses", + "transport": "http", + "model": "gpt-5.5", + "tags": [ + "prefix:openai-responses", + "provider:openai", + "flagship", + "text", + "golden" + ] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/responses", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gpt-5.5\",\"input\":[{\"role\":\"system\",\"content\":\"You are concise.\"},{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"Reply exactly with: Hello!\"}]}],\"max_output_tokens\":40,\"stream\":true}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_03db31943129228b006a01388ea6a481a081e6e6b17947a5ee\",\"object\":\"response\",\"created_at\":1778464910,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":40,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"effort\":\"medium\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}\n\nevent: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_03db31943129228b006a01388ea6a481a081e6e6b17947a5ee\",\"object\":\"response\",\"created_at\":1778464910,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":40,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"effort\":\"medium\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}\n\nevent: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_03db31943129228b006a01388fb4a881a0aa678375e614d2c4\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}\n\nevent: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_03db31943129228b006a01388fb4a881a0aa678375e614d2c4\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"Hello\",\"item_id\":\"msg_03db31943129228b006a01388fb4a881a0aa678375e614d2c4\",\"logprobs\":[],\"obfuscation\":\"gpNAzOcV2UP\",\"output_index\":0,\"sequence_number\":4}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"!\",\"item_id\":\"msg_03db31943129228b006a01388fb4a881a0aa678375e614d2c4\",\"logprobs\":[],\"obfuscation\":\"r4ktr666Pt0BiSJ\",\"output_index\":0,\"sequence_number\":5}\n\nevent: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_03db31943129228b006a01388fb4a881a0aa678375e614d2c4\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":6,\"text\":\"Hello!\"}\n\nevent: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_03db31943129228b006a01388fb4a881a0aa678375e614d2c4\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"Hello!\"},\"sequence_number\":7}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_03db31943129228b006a01388fb4a881a0aa678375e614d2c4\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"Hello!\"}],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":8}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_03db31943129228b006a01388ea6a481a081e6e6b17947a5ee\",\"object\":\"response\",\"created_at\":1778464910,\"status\":\"completed\",\"background\":false,\"completed_at\":1778464911,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":40,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\":null,\"output\":[{\"id\":\"msg_03db31943129228b006a01388fb4a881a0aa678375e614d2c4\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"Hello!\"}],\"phase\":\"final_answer\",\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"effort\":\"medium\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":20,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":6,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":26},\"user\":null,\"metadata\":{}},\"sequence_number\":9}\n\n" + } + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-tool-call.json b/packages/llm/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-tool-call.json new file mode 100644 index 0000000000..544c0ba53b --- /dev/null +++ b/packages/llm/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-tool-call.json @@ -0,0 +1,39 @@ +{ + "version": 1, + "metadata": { + "name": "openai-responses/openai-responses-gpt-5-5-tool-call", + "recordedAt": "2026-05-11T02:01:54.829Z", + "provider": "openai", + "route": "openai-responses", + "transport": "http", + "model": "gpt-5.5", + "tags": [ + "prefix:openai-responses", + "provider:openai", + "flagship", + "tool", + "tool-call", + "golden" + ] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/responses", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gpt-5.5\",\"input\":[{\"role\":\"system\",\"content\":\"Call tools exactly as requested.\"},{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"Call get_weather with city exactly Paris.\"}]}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}],\"tool_choice\":{\"type\":\"function\",\"name\":\"get_weather\"},\"max_output_tokens\":80,\"stream\":true}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_03cd5bf3fbd90b99006a01389022a48193b770ab6d062b3ff5\",\"object\":\"response\",\"created_at\":1778464912,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":80,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"effort\":\"medium\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":{\"type\":\"function\",\"name\":\"get_weather\"},\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}\n\nevent: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_03cd5bf3fbd90b99006a01389022a48193b770ab6d062b3ff5\",\"object\":\"response\",\"created_at\":1778464912,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":80,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"effort\":\"medium\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":{\"type\":\"function\",\"name\":\"get_weather\"},\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}\n\nevent: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_03cd5bf3fbd90b99006a0138923f8c8193bca949d9ecdfcc03\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_eEA5zbL9Seb7SbNCXifopygi\",\"name\":\"get_weather\"},\"output_index\":0,\"sequence_number\":2}\n\nevent: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_03cd5bf3fbd90b99006a0138923f8c8193bca949d9ecdfcc03\",\"obfuscation\":\"3jhfcjwHwEFUsm\",\"output_index\":0,\"sequence_number\":3}\n\nevent: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"city\",\"item_id\":\"fc_03cd5bf3fbd90b99006a0138923f8c8193bca949d9ecdfcc03\",\"obfuscation\":\"xcS1ivWglNNX\",\"output_index\":0,\"sequence_number\":4}\n\nevent: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_03cd5bf3fbd90b99006a0138923f8c8193bca949d9ecdfcc03\",\"obfuscation\":\"bM7tal2PhDGQ3\",\"output_index\":0,\"sequence_number\":5}\n\nevent: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Paris\",\"item_id\":\"fc_03cd5bf3fbd90b99006a0138923f8c8193bca949d9ecdfcc03\",\"obfuscation\":\"4cQtJsBXtgn\",\"output_index\":0,\"sequence_number\":6}\n\nevent: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"}\",\"item_id\":\"fc_03cd5bf3fbd90b99006a0138923f8c8193bca949d9ecdfcc03\",\"obfuscation\":\"7ng8mgc0SqelhX\",\"output_index\":0,\"sequence_number\":7}\n\nevent: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\",\"item_id\":\"fc_03cd5bf3fbd90b99006a0138923f8c8193bca949d9ecdfcc03\",\"output_index\":0,\"sequence_number\":8}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_03cd5bf3fbd90b99006a0138923f8c8193bca949d9ecdfcc03\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\",\"call_id\":\"call_eEA5zbL9Seb7SbNCXifopygi\",\"name\":\"get_weather\"},\"output_index\":0,\"sequence_number\":9}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_03cd5bf3fbd90b99006a01389022a48193b770ab6d062b3ff5\",\"object\":\"response\",\"created_at\":1778464912,\"status\":\"completed\",\"background\":false,\"completed_at\":1778464914,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":80,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\":null,\"output\":[{\"id\":\"fc_03cd5bf3fbd90b99006a0138923f8c8193bca949d9ecdfcc03\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\",\"call_id\":\"call_eEA5zbL9Seb7SbNCXifopygi\",\"name\":\"get_weather\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"effort\":\"medium\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":{\"type\":\"function\",\"name\":\"get_weather\"},\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":61,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":18,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":79},\"user\":null,\"metadata\":{}},\"sequence_number\":10}\n\n" + } + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-tool-loop.json b/packages/llm/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-tool-loop.json new file mode 100644 index 0000000000..f0c519d09d --- /dev/null +++ b/packages/llm/test/fixtures/recordings/openai-responses/openai-responses-gpt-5-5-tool-loop.json @@ -0,0 +1,57 @@ +{ + "version": 1, + "metadata": { + "name": "openai-responses/openai-responses-gpt-5-5-tool-loop", + "recordedAt": "2026-05-11T02:01:58.688Z", + "provider": "openai", + "route": "openai-responses", + "transport": "http", + "model": "gpt-5.5", + "tags": [ + "prefix:openai-responses", + "provider:openai", + "flagship", + "tool", + "tool-loop", + "golden" + ] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/responses", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gpt-5.5\",\"input\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"},{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"What is the weather in Paris?\"}]}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}],\"max_output_tokens\":80,\"stream\":true}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_002938b5a91b8a90006a013892e648819f8a1c887d6a80ae92\",\"object\":\"response\",\"created_at\":1778464914,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":80,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"effort\":\"medium\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}\n\nevent: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_002938b5a91b8a90006a013892e648819f8a1c887d6a80ae92\",\"object\":\"response\",\"created_at\":1778464914,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":80,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"effort\":\"medium\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}\n\nevent: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"rs_002938b5a91b8a90006a013894af54819fa387f5a066f14730\",\"type\":\"reasoning\",\"summary\":[]},\"output_index\":0,\"sequence_number\":2}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"rs_002938b5a91b8a90006a013894af54819fa387f5a066f14730\",\"type\":\"reasoning\",\"summary\":[]},\"output_index\":0,\"sequence_number\":3}\n\nevent: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_002938b5a91b8a90006a0138951da4819fa867ae1cf299de51\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_Zz0OL9xt7R1z0eTMx3p36BmX\",\"name\":\"get_weather\"},\"output_index\":1,\"sequence_number\":4}\n\nevent: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_002938b5a91b8a90006a0138951da4819fa867ae1cf299de51\",\"obfuscation\":\"WooL7EJol03OR0\",\"output_index\":1,\"sequence_number\":5}\n\nevent: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"city\",\"item_id\":\"fc_002938b5a91b8a90006a0138951da4819fa867ae1cf299de51\",\"obfuscation\":\"K35a7Ps4q34B\",\"output_index\":1,\"sequence_number\":6}\n\nevent: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_002938b5a91b8a90006a0138951da4819fa867ae1cf299de51\",\"obfuscation\":\"U0lxUOBbnEWi6\",\"output_index\":1,\"sequence_number\":7}\n\nevent: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Paris\",\"item_id\":\"fc_002938b5a91b8a90006a0138951da4819fa867ae1cf299de51\",\"obfuscation\":\"eYJ8S5XzMMk\",\"output_index\":1,\"sequence_number\":8}\n\nevent: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"}\",\"item_id\":\"fc_002938b5a91b8a90006a0138951da4819fa867ae1cf299de51\",\"obfuscation\":\"3tPQtIQZLbEtvL\",\"output_index\":1,\"sequence_number\":9}\n\nevent: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\",\"item_id\":\"fc_002938b5a91b8a90006a0138951da4819fa867ae1cf299de51\",\"output_index\":1,\"sequence_number\":10}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_002938b5a91b8a90006a0138951da4819fa867ae1cf299de51\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\",\"call_id\":\"call_Zz0OL9xt7R1z0eTMx3p36BmX\",\"name\":\"get_weather\"},\"output_index\":1,\"sequence_number\":11}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_002938b5a91b8a90006a013892e648819f8a1c887d6a80ae92\",\"object\":\"response\",\"created_at\":1778464914,\"status\":\"completed\",\"background\":false,\"completed_at\":1778464917,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":80,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\":null,\"output\":[{\"id\":\"rs_002938b5a91b8a90006a013894af54819fa387f5a066f14730\",\"type\":\"reasoning\",\"summary\":[]},{\"id\":\"fc_002938b5a91b8a90006a0138951da4819fa867ae1cf299de51\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\",\"call_id\":\"call_Zz0OL9xt7R1z0eTMx3p36BmX\",\"name\":\"get_weather\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"effort\":\"medium\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":74,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":44,\"output_tokens_details\":{\"reasoning_tokens\":24},\"total_tokens\":118},\"user\":null,\"metadata\":{}},\"sequence_number\":12}\n\n" + } + }, + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/responses", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gpt-5.5\",\"input\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"},{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"What is the weather in Paris?\"}]},{\"type\":\"function_call\",\"call_id\":\"call_Zz0OL9xt7R1z0eTMx3p36BmX\",\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"},{\"type\":\"function_call_output\",\"call_id\":\"call_Zz0OL9xt7R1z0eTMx3p36BmX\",\"output\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}],\"max_output_tokens\":80,\"stream\":true}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0851ba0c41fc4434006a013895976081a38f006157c1fd43db\",\"object\":\"response\",\"created_at\":1778464917,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":80,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"effort\":\"medium\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}\n\nevent: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0851ba0c41fc4434006a013895976081a38f006157c1fd43db\",\"object\":\"response\",\"created_at\":1778464917,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":80,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"effort\":\"medium\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}\n\nevent: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0851ba0c41fc4434006a0138964d2481a3997fbea69c3b7378\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}\n\nevent: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_0851ba0c41fc4434006a0138964d2481a3997fbea69c3b7378\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"Paris\",\"item_id\":\"msg_0851ba0c41fc4434006a0138964d2481a3997fbea69c3b7378\",\"logprobs\":[],\"obfuscation\":\"S6CIU3RQIod\",\"output_index\":0,\"sequence_number\":4}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" is\",\"item_id\":\"msg_0851ba0c41fc4434006a0138964d2481a3997fbea69c3b7378\",\"logprobs\":[],\"obfuscation\":\"Eht80PaSjVwPy\",\"output_index\":0,\"sequence_number\":5}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" sunny\",\"item_id\":\"msg_0851ba0c41fc4434006a0138964d2481a3997fbea69c3b7378\",\"logprobs\":[],\"obfuscation\":\"qOBv4MMHpy\",\"output_index\":0,\"sequence_number\":6}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\".\",\"item_id\":\"msg_0851ba0c41fc4434006a0138964d2481a3997fbea69c3b7378\",\"logprobs\":[],\"obfuscation\":\"CU7ay4cq29qD1OT\",\"output_index\":0,\"sequence_number\":7}\n\nevent: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0851ba0c41fc4434006a0138964d2481a3997fbea69c3b7378\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":8,\"text\":\"Paris is sunny.\"}\n\nevent: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_0851ba0c41fc4434006a0138964d2481a3997fbea69c3b7378\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"Paris is sunny.\"},\"sequence_number\":9}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_0851ba0c41fc4434006a0138964d2481a3997fbea69c3b7378\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"Paris is sunny.\"}],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":10}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0851ba0c41fc4434006a013895976081a38f006157c1fd43db\",\"object\":\"response\",\"created_at\":1778464917,\"status\":\"completed\",\"background\":false,\"completed_at\":1778464918,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":80,\"max_tool_calls\":null,\"model\":\"gpt-5.5-2026-04-23\",\"moderation\":null,\"output\":[{\"id\":\"msg_0851ba0c41fc4434006a0138964d2481a3997fbea69c3b7378\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"Paris is sunny.\"}],\"phase\":\"final_answer\",\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"effort\":\"medium\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":113,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":8,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":121},\"user\":null,\"metadata\":{}},\"sequence_number\":11}\n\n" + } + } + ] +} diff --git a/packages/llm/test/provider/anthropic-messages.test.ts b/packages/llm/test/provider/anthropic-messages.test.ts index 85900a1143..0005ae7dfe 100644 --- a/packages/llm/test/provider/anthropic-messages.test.ts +++ b/packages/llm/test/provider/anthropic-messages.test.ts @@ -1,6 +1,6 @@ import { describe, expect } from "bun:test" import { Effect } from "effect" -import { CacheHint, LLM, LLMError } from "../../src" +import { CacheHint, LLM, LLMError, Usage } from "../../src" import { LLMClient } from "../../src/route" import * as AnthropicMessages from "../../src/protocols/anthropic-messages" import { it } from "../lib/effect" @@ -110,10 +110,11 @@ describe("Anthropic Messages route", () => { expect(response.text).toBe("Hello!") expect(response.reasoning).toBe("thinking") expect(response.usage).toMatchObject({ - inputTokens: 5, + inputTokens: 6, outputTokens: 2, + nonCachedInputTokens: 5, cacheReadInputTokens: 1, - totalTokens: 7, + totalTokens: 8, }) expect(response.events.find((event) => event.type === "reasoning-end")).toMatchObject({ providerMetadata: { anthropic: { signature: "sig_1" } }, @@ -152,7 +153,13 @@ describe("Anthropic Messages route", () => { { type: "request-finish", reason: "tool-calls", - usage: { inputTokens: 5, outputTokens: 1, totalTokens: 6, native: { input_tokens: 5, output_tokens: 1 } }, + usage: new Usage({ + inputTokens: 5, + outputTokens: 1, + nonCachedInputTokens: 5, + totalTokens: 6, + providerMetadata: { anthropic: { input_tokens: 5, output_tokens: 1 } }, + }), }, ]) }), diff --git a/packages/llm/test/provider/gemini.test.ts b/packages/llm/test/provider/gemini.test.ts index 9de4e0dc25..e0b3864a26 100644 --- a/packages/llm/test/provider/gemini.test.ts +++ b/packages/llm/test/provider/gemini.test.ts @@ -1,6 +1,6 @@ import { describe, expect } from "bun:test" import { Effect } from "effect" -import { LLM, LLMError } from "../../src" +import { LLM, LLMError, Usage } from "../../src" import { LLMClient } from "../../src/route" import * as Gemini from "../../src/protocols/gemini" import { it } from "../lib/effect" @@ -198,9 +198,10 @@ describe("Gemini route", () => { expect(response.reasoning).toBe("thinking") expect(response.usage).toMatchObject({ inputTokens: 5, - outputTokens: 2, - reasoningTokens: 1, + outputTokens: 3, + nonCachedInputTokens: 4, cacheReadInputTokens: 1, + reasoningTokens: 1, totalTokens: 7, }) expect(response.events).toEqual([ @@ -210,20 +211,23 @@ describe("Gemini route", () => { { type: "request-finish", reason: "stop", - usage: { + usage: new Usage({ inputTokens: 5, - outputTokens: 2, - reasoningTokens: 1, + outputTokens: 3, + nonCachedInputTokens: 4, cacheReadInputTokens: 1, + reasoningTokens: 1, totalTokens: 7, - native: { - promptTokenCount: 5, - candidatesTokenCount: 2, - totalTokenCount: 7, - thoughtsTokenCount: 1, - cachedContentTokenCount: 1, + providerMetadata: { + google: { + promptTokenCount: 5, + candidatesTokenCount: 2, + totalTokenCount: 7, + thoughtsTokenCount: 1, + cachedContentTokenCount: 1, + }, }, - }, + }), }, ]) }), @@ -257,12 +261,13 @@ describe("Gemini route", () => { { type: "request-finish", reason: "tool-calls", - usage: { + usage: new Usage({ inputTokens: 5, outputTokens: 1, + nonCachedInputTokens: 5, totalTokens: 6, - native: { promptTokenCount: 5, candidatesTokenCount: 1 }, - }, + providerMetadata: { google: { promptTokenCount: 5, candidatesTokenCount: 1 } }, + }), }, ]) }), diff --git a/packages/llm/test/provider/openai-chat.test.ts b/packages/llm/test/provider/openai-chat.test.ts index 8b0dfc2894..2c692dcd7d 100644 --- a/packages/llm/test/provider/openai-chat.test.ts +++ b/packages/llm/test/provider/openai-chat.test.ts @@ -1,7 +1,7 @@ import { describe, expect } from "bun:test" import { Effect, Schema, Stream } from "effect" import { HttpClientRequest } from "effect/unstable/http" -import { LLM, LLMError } from "../../src" +import { LLM, LLMError, Usage } from "../../src" import * as Azure from "../../src/providers/azure" import * as OpenAI from "../../src/providers/openai" import * as OpenAIChat from "../../src/protocols/openai-chat" @@ -230,20 +230,23 @@ describe("OpenAI Chat route", () => { { type: "request-finish", reason: "stop", - usage: { + usage: new Usage({ inputTokens: 5, outputTokens: 2, - reasoningTokens: 0, + nonCachedInputTokens: 4, cacheReadInputTokens: 1, + reasoningTokens: 0, totalTokens: 7, - native: { - prompt_tokens: 5, - completion_tokens: 2, - total_tokens: 7, - prompt_tokens_details: { cached_tokens: 1 }, - completion_tokens_details: { reasoning_tokens: 0 }, + providerMetadata: { + openai: { + prompt_tokens: 5, + completion_tokens: 2, + total_tokens: 7, + prompt_tokens_details: { cached_tokens: 1 }, + completion_tokens_details: { reasoning_tokens: 0 }, + }, }, - }, + }), }, ]) }), diff --git a/packages/llm/test/provider/openai-responses.test.ts b/packages/llm/test/provider/openai-responses.test.ts index 5141b44cc2..2319857ed1 100644 --- a/packages/llm/test/provider/openai-responses.test.ts +++ b/packages/llm/test/provider/openai-responses.test.ts @@ -1,7 +1,7 @@ import { describe, expect } from "bun:test" import { ConfigProvider, Effect, Layer, Stream } from "effect" import { Headers, HttpClientRequest } from "effect/unstable/http" -import { LLM, LLMError } from "../../src" +import { LLM, LLMError, Usage } from "../../src" import { Auth, LLMClient, RequestExecutor, WebSocketExecutor } from "../../src/route" import * as Azure from "../../src/providers/azure" import * as OpenAI from "../../src/providers/openai" @@ -342,20 +342,23 @@ describe("OpenAI Responses route", () => { type: "request-finish", reason: "stop", providerMetadata: { openai: { responseId: "resp_1", serviceTier: "default" } }, - usage: { + usage: new Usage({ inputTokens: 5, outputTokens: 2, - reasoningTokens: 0, + nonCachedInputTokens: 4, cacheReadInputTokens: 1, + reasoningTokens: 0, totalTokens: 7, - native: { - input_tokens: 5, - output_tokens: 2, - total_tokens: 7, - input_tokens_details: { cached_tokens: 1 }, - output_tokens_details: { reasoning_tokens: 0 }, + providerMetadata: { + openai: { + input_tokens: 5, + output_tokens: 2, + total_tokens: 7, + input_tokens_details: { cached_tokens: 1 }, + output_tokens_details: { reasoning_tokens: 0 }, + }, }, - }, + }), }, ]) }), @@ -411,7 +414,13 @@ describe("OpenAI Responses route", () => { { type: "request-finish", reason: "tool-calls", - usage: { inputTokens: 5, outputTokens: 1, totalTokens: 6, native: { input_tokens: 5, output_tokens: 1 } }, + usage: new Usage({ + inputTokens: 5, + outputTokens: 1, + nonCachedInputTokens: 5, + totalTokens: 6, + providerMetadata: { openai: { input_tokens: 5, output_tokens: 1 } }, + }), }, ]) }), diff --git a/packages/llm/test/schema.test.ts b/packages/llm/test/schema.test.ts index 46eb85b075..23bd9fd9bb 100644 --- a/packages/llm/test/schema.test.ts +++ b/packages/llm/test/schema.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test" import { Schema } from "effect" -import { ContentPart, LLMEvent, LLMRequest, ModelID, ModelLimits, ModelRef, ProviderID } from "../src/schema" +import { ContentPart, LLMEvent, LLMRequest, ModelID, ModelLimits, ModelRef, ProviderID, Usage } from "../src/schema" +import { ProviderShared } from "../src/protocols/shared" const model = new ModelRef({ id: ModelID.make("fake-model"), @@ -48,3 +49,30 @@ describe("llm schema", () => { expect(ContentPart.guards.media({ type: "text", text: "hi" })).toBe(false) }) }) + +describe("LLM.Usage", () => { + test("subtractTokens clamps non-sensical breakdowns to zero", () => { + // Defense against a provider reporting cached_tokens > prompt_tokens or + // reasoning_tokens > completion_tokens — the negative would otherwise + // round-trip through the pipeline and crash strict downstream schemas. + expect(ProviderShared.subtractTokens(5, 3)).toBe(2) + expect(ProviderShared.subtractTokens(5, 10)).toBe(0) + expect(ProviderShared.subtractTokens(5, undefined)).toBe(5) + expect(ProviderShared.subtractTokens(undefined, 3)).toBeUndefined() + expect(ProviderShared.subtractTokens(undefined, undefined)).toBeUndefined() + }) + + test("sumTokens returns undefined only when every input is undefined", () => { + expect(ProviderShared.sumTokens(1, 2, 3)).toBe(6) + expect(ProviderShared.sumTokens(1, undefined, 3)).toBe(4) + expect(ProviderShared.sumTokens(undefined, undefined, undefined)).toBeUndefined() + expect(ProviderShared.sumTokens()).toBeUndefined() + }) + + test("visibleOutputTokens clamps reasoning > output to zero", () => { + expect(new Usage({ outputTokens: 10, reasoningTokens: 4 }).visibleOutputTokens).toBe(6) + expect(new Usage({ outputTokens: 10 }).visibleOutputTokens).toBe(10) + expect(new Usage({ outputTokens: 4, reasoningTokens: 10 }).visibleOutputTokens).toBe(0) + expect(new Usage({}).visibleOutputTokens).toBe(0) + }) +})