fix(core): recover v2 context overflow (#31005)

packages/llm/src/index.ts

@@ -1,6 +1,7 @@
 export { LLMClient } from "./route/client"
 export { Auth } from "./route/auth"
 export { Provider } from "./provider"
+export { isContextOverflow, isContextOverflowFailure } from "./provider-error"
 export type {
   RouteModelInput,
   RouteRoutedModelInput,

packages/llm/src/protocols/anthropic-messages.ts

@@ -18,6 +18,7 @@ import {
   type ToolResultPart,
 } from "../schema"
 import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared"
+import { isContextOverflow } from "../provider-error"
 import * as Cache from "./utils/cache"
 import { Lifecycle } from "./utils/lifecycle"
 import { ToolStream } from "./utils/tool-stream"
@@ -786,7 +787,12 @@ const providerErrorMessage = (event: AnthropicEvent): string => {
 
 const onError = (state: ParserState, event: AnthropicEvent): StepResult => [
   state,
-  [LLMEvent.providerError({ message: providerErrorMessage(event) })],
+  [
+    LLMEvent.providerError({
+      message: providerErrorMessage(event),
+      classification: isContextOverflow(event.error?.message ?? "") ? "context-overflow" : undefined,
+    }),
+  ],
 ]
 
 const step = (state: ParserState, event: AnthropicEvent) => {

packages/llm/src/protocols/bedrock-converse.ts

@@ -15,6 +15,7 @@ import {
   type ToolResultPart,
 } from "../schema"
 import { BedrockEventStream } from "./bedrock-event-stream"
+import { isContextOverflow } from "../provider-error"
 import { JsonObject, optionalArray, ProviderShared } from "./shared"
 import { BedrockAuth } from "./utils/bedrock-auth"
 import { BedrockCache } from "./utils/bedrock-cache"
@@ -582,7 +583,16 @@ const step = (state: ParserState, event: BedrockEvent) =>
     if (event.validationException || event.throttlingException) {
       const message =
         event.validationException?.message ?? event.throttlingException?.message ?? "Bedrock Converse error"
-      return [state, [LLMEvent.providerError({ message, retryable: event.throttlingException !== undefined })]] as const
+      return [
+        state,
+        [
+          LLMEvent.providerError({
+            message,
+            classification: event.validationException && isContextOverflow(message) ? "context-overflow" : undefined,
+            retryable: event.throttlingException !== undefined,
+          }),
+        ],
+      ] as const
     }
 
     return [state, []] as const

packages/llm/src/protocols/openai-responses.ts

@@ -18,6 +18,7 @@ import {
   type ToolResultPart,
 } from "../schema"
 import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared"
+import { isContextOverflow } from "../provider-error"
 import { OpenAIOptions } from "./utils/openai-options"
 import { Lifecycle } from "./utils/lifecycle"
 import { ToolStream } from "./utils/tool-stream"
@@ -880,14 +881,23 @@ const providerErrorMessage = (event: OpenAIResponsesEvent, fallback: string): st
   return message || code || fallback
 }
 
+const providerError = (event: OpenAIResponsesEvent, fallback: string) => {
+  const code = event.code || event.response?.error?.code || undefined
+  const message = providerErrorMessage(event, fallback)
+  return LLMEvent.providerError({
+    message,
+    classification: code === "context_length_exceeded" || isContextOverflow(message) ? "context-overflow" : undefined,
+  })
+}
+
 const onResponseFailed = (state: ParserState, event: OpenAIResponsesEvent): StepResult => [
   state,
-  [LLMEvent.providerError({ message: providerErrorMessage(event, "OpenAI Responses response failed") })],
+  [providerError(event, "OpenAI Responses response failed")],
 ]
 
 const onError = (state: ParserState, event: OpenAIResponsesEvent): StepResult => [
   state,
-  [LLMEvent.providerError({ message: providerErrorMessage(event, "OpenAI Responses stream error") })],
+  [providerError(event, "OpenAI Responses stream error")],
 ]
 
 const step = (state: ParserState, event: OpenAIResponsesEvent) => {

packages/llm/src/provider-error.ts

@@ -0,0 +1,32 @@
+import { Schema } from "effect"
+import { LLMError, ProviderErrorEvent } from "./schema"
+
+const patterns = [
+  /prompt is too long/i,
+  /input is too long for requested model/i,
+  /exceeds the context window/i,
+  /input token count.*exceeds the maximum/i,
+  /maximum prompt length is \d+/i,
+  /reduce the length of the messages/i,
+  /maximum context length is \d+ tokens/i,
+  /exceeds the limit of \d+/i,
+  /exceeds the available context size/i,
+  /greater than the context length/i,
+  /context window exceeds limit/i,
+  /exceeded model token limit/i,
+  /context[_ ]length[_ ]exceeded/i,
+  /request entity too large/i,
+  /context length is only \d+ tokens/i,
+  /input length.*exceeds.*context length/i,
+  /prompt too long; exceeded (?:max )?context length/i,
+  /too large for model with \d+ maximum context length/i,
+  /model_context_window_exceeded/i,
+]
+
+export const isContextOverflow = (message: string) =>
+  patterns.some((pattern) => pattern.test(message)) || /^4(00|13)\s*(status code)?\s*\(no body\)/i.test(message)
+
+export const isContextOverflowFailure = (failure: unknown) =>
+  failure instanceof LLMError
+    ? failure.reason._tag === "InvalidRequest" && failure.reason.classification === "context-overflow"
+    : Schema.is(ProviderErrorEvent)(failure) && failure.classification === "context-overflow"

packages/llm/src/route/executor.ts

@@ -22,6 +22,7 @@ import {
   TransportReason,
   UnknownProviderReason,
 } from "../schema"
+import { isContextOverflow } from "../provider-error"
 
 export interface Interface {
   readonly execute: (
@@ -249,8 +250,18 @@ const statusReason = (input: {
       http: input.http,
     })
   }
-  if (input.status === 400 || input.status === 404 || input.status === 409 || input.status === 422) {
-    return new InvalidRequestReason({ message: input.message, http: input.http })
+  if (
+    input.status === 400 ||
+    input.status === 404 ||
+    input.status === 409 ||
+    input.status === 413 ||
+    input.status === 422
+  ) {
+    return new InvalidRequestReason({
+      message: input.message,
+      classification: isContextOverflow(body) ? "context-overflow" : undefined,
+      http: input.http,
+    })
   }
   if (input.status >= 500 || retryableStatus(input.status)) {
     return new ProviderInternalReason({

packages/llm/src/schema/errors.ts

@@ -1,6 +1,9 @@
 import { Schema } from "effect"
 import { ModelID, ProviderID, ProviderMetadata, RouteID } from "./ids"
 
+export const ProviderFailureClassification = Schema.Literal("context-overflow")
+export type ProviderFailureClassification = typeof ProviderFailureClassification.Type
+
 export class HttpRequestDetails extends Schema.Class<HttpRequestDetails>("LLM.HttpRequestDetails")({
   method: Schema.String,
   url: Schema.String,
@@ -32,6 +35,7 @@ export class InvalidRequestReason extends Schema.Class<InvalidRequestReason>("LL
   _tag: Schema.tag("InvalidRequest"),
   message: Schema.String,
   parameter: Schema.optional(Schema.String),
+  classification: Schema.optional(ProviderFailureClassification),
   providerMetadata: Schema.optional(ProviderMetadata),
   http: Schema.optional(HttpContext),
 }) {

packages/llm/src/schema/events.ts

@@ -2,6 +2,7 @@ import { Schema } from "effect"
 import { ContentBlockID, FinishReason, ProtocolID, ProviderMetadata, RouteID, ToolCallID } from "./ids"
 import { ModelSchema } from "./options"
 import { ToolOutput, ToolResultValue } from "./messages"
+import { ProviderFailureClassification } from "./errors"
 
 /**
  * Token usage reported by an LLM provider.
@@ -199,6 +200,7 @@ export type Finish = Schema.Schema.Type<typeof Finish>
 export const ProviderErrorEvent = Schema.Struct({
   type: Schema.tag("provider-error"),
   message: Schema.String,
+  classification: Schema.optional(ProviderFailureClassification),
   retryable: Schema.optional(Schema.Boolean),
   providerMetadata: Schema.optional(ProviderMetadata),
 }).annotate({ identifier: "LLM.Event.ProviderError" })

packages/llm/test/executor.test.ts

@@ -73,6 +73,46 @@ const expectLLMError = (error: unknown) => {
 const errorHttp = (error: LLMError) => ("http" in error.reason ? error.reason.http : undefined)
 
 describe("RequestExecutor", () => {
+  it.effect("classifies context overflow responses", () =>
+    Effect.gen(function* () {
+      const executor = yield* RequestExecutor.Service
+      const error = yield* executor.execute(request).pipe(Effect.flip)
+
+      expectLLMError(error)
+      expect(error.reason).toMatchObject({ _tag: "InvalidRequest", classification: "context-overflow" })
+    }).pipe(
+      Effect.provide(
+        responsesLayer([
+          new Response('{"error":{"code":"context_length_exceeded","message":"prompt too long"}}', {
+            status: 400,
+          }),
+        ]),
+      ),
+    ),
+  )
+
+  it.effect("does not classify generic HTTP 413 payload errors as context overflow", () =>
+    Effect.gen(function* () {
+      const executor = yield* RequestExecutor.Service
+      const error = yield* executor.execute(request).pipe(Effect.flip)
+
+      expectLLMError(error)
+      expect(error.reason).toMatchObject({ _tag: "InvalidRequest" })
+      expect("classification" in error.reason ? error.reason.classification : undefined).toBeUndefined()
+    }).pipe(Effect.provide(responsesLayer([new Response("request too large", { status: 413 })]))),
+  )
+
+  it.effect("does not classify ordinary invalid requests as context overflow", () =>
+    Effect.gen(function* () {
+      const executor = yield* RequestExecutor.Service
+      const error = yield* executor.execute(request).pipe(Effect.flip)
+
+      expectLLMError(error)
+      expect(error.reason).toMatchObject({ _tag: "InvalidRequest" })
+      expect("classification" in error.reason ? error.reason.classification : undefined).toBeUndefined()
+    }).pipe(Effect.provide(responsesLayer([new Response("invalid parameter", { status: 400 })]))),
+  )
+
   it.effect("returns redacted diagnostics for retryable rate limits", () =>
     Effect.gen(function* () {
       const executor = yield* RequestExecutor.Service

packages/llm/test/provider/anthropic-messages.test.ts

@@ -477,6 +477,29 @@ describe("Anthropic Messages route", () => {
     }),
   )
 
+  it.effect("classifies prompt-too-long provider errors", () =>
+    Effect.gen(function* () {
+      const response = yield* LLMClient.generate(request).pipe(
+        Effect.provide(
+          fixedResponse(
+            sseEvents({
+              type: "error",
+              error: { type: "invalid_request_error", message: "prompt is too long: 210000 tokens" },
+            }),
+          ),
+        ),
+      )
+
+      expect(response.events).toEqual([
+        {
+          type: "provider-error",
+          message: "invalid_request_error: prompt is too long: 210000 tokens",
+          classification: "context-overflow",
+        },
+      ])
+    }),
+  )
+
   it.effect("falls back to error type when no message is present", () =>
     Effect.gen(function* () {
       const response = yield* LLMClient.generate(request).pipe(

packages/llm/test/provider/bedrock-converse.test.ts

@@ -351,6 +351,23 @@ describe("Bedrock Converse route", () => {
     }),
   )
 
+  it.effect("classifies input-too-long validation exceptions", () =>
+    Effect.gen(function* () {
+      const response = yield* LLMClient.generate(baseRequest).pipe(
+        Effect.provide(
+          fixedBytes(eventStreamBody(["validationException", { message: "Input is too long for requested model" }])),
+        ),
+      )
+
+      expect(response.events.find((event) => event.type === "provider-error")).toEqual({
+        type: "provider-error",
+        message: "Input is too long for requested model",
+        classification: "context-overflow",
+        retryable: false,
+      })
+    }),
+  )
+
   it.effect("rejects requests with no auth path", () =>
     Effect.gen(function* () {
       const unsignedModel = AmazonBedrock.configure({

packages/llm/test/provider/openai-responses.test.ts

@@ -1351,7 +1351,13 @@ describe("OpenAI Responses route", () => {
         ),
       )
 
-      expect(response.events).toEqual([{ type: "provider-error", message: "context_length_exceeded: prompt too long" }])
+      expect(response.events).toEqual([
+        {
+          type: "provider-error",
+          message: "context_length_exceeded: prompt too long",
+          classification: "context-overflow",
+        },
+      ])
     }),
   )