fix(ai): classify provider failures consistently
This commit is contained in:
Aiden Cline
parent
87d5b27668
commit
0960ef48fd
9 changed files with 398 additions and 46 deletions
|
|
@ -15,7 +15,13 @@ import type {
|
|||
SharedV3ProviderOptions,
|
||||
} from "@ai-sdk/provider"
|
||||
import {
|
||||
AuthenticationReason,
|
||||
classifyProviderFailure,
|
||||
FinishReason,
|
||||
HttpContext,
|
||||
HttpRequestDetails,
|
||||
HttpResponseDetails,
|
||||
InvalidRequestReason,
|
||||
InvalidProviderOutputReason,
|
||||
LLMEvent,
|
||||
LLMError,
|
||||
|
|
@ -23,13 +29,28 @@ import {
|
|||
ProviderID,
|
||||
ProviderMetadata,
|
||||
ToolResultValue,
|
||||
TransportReason,
|
||||
UnknownProviderReason,
|
||||
type ContentPart,
|
||||
type LLMRequest,
|
||||
type ToolDefinition,
|
||||
type UsageInput,
|
||||
} from "@opencode-ai/ai"
|
||||
import { Auth, Endpoint, type AnyRoute } from "@opencode-ai/ai/route"
|
||||
import {
|
||||
APICallError,
|
||||
EmptyResponseBodyError,
|
||||
InvalidArgumentError,
|
||||
InvalidPromptError,
|
||||
InvalidResponseDataError,
|
||||
JSONParseError,
|
||||
LoadAPIKeyError,
|
||||
LoadSettingError,
|
||||
NoContentGeneratedError,
|
||||
NoSuchModelError,
|
||||
TypeValidationError,
|
||||
UnsupportedFunctionalityError,
|
||||
} from "@ai-sdk/provider"
|
||||
import { Auth, Endpoint, RequestExecutor, type AnyRoute } from "@opencode-ai/ai/route"
|
||||
import { Cause, Context, Effect, Layer, Option, Schema, Scope, Stream } from "effect"
|
||||
import { ModelV2 } from "./model"
|
||||
import { ProviderV2 } from "./provider"
|
||||
|
|
@ -40,6 +61,8 @@ type UserContent = Extract<LanguageModelV3Message, { role: "user" }>["content"]
|
|||
type AssistantContent = Extract<LanguageModelV3Message, { role: "assistant" }>["content"]
|
||||
type ToolResultContent = Extract<AssistantContent[number], { type: "tool-result" }>
|
||||
|
||||
class ChunkTimeoutError extends Error {}
|
||||
|
||||
export interface SDKEvent {
|
||||
readonly model: ModelV2.Info
|
||||
readonly package: string
|
||||
|
|
@ -64,7 +87,7 @@ function wrapSSE(res: Response, ms: number, ctl: AbortController) {
|
|||
async pull(ctrl) {
|
||||
const part = await new Promise<Awaited<ReturnType<typeof reader.read>>>((resolve, reject) => {
|
||||
const id = setTimeout(() => {
|
||||
const err = new Error("SSE read timed out")
|
||||
const err = new ChunkTimeoutError("SSE read timed out")
|
||||
ctl.abort(err)
|
||||
void reader.cancel(err)
|
||||
reject(err)
|
||||
|
|
@ -716,10 +739,37 @@ function messageValue(input: unknown) {
|
|||
}
|
||||
|
||||
function llmError(method: string, error: unknown) {
|
||||
const reason =
|
||||
error instanceof LLMError
|
||||
? new InvalidProviderOutputReason({ message: error.message })
|
||||
: new UnknownProviderReason({ message: error instanceof Error ? error.message : String(error) })
|
||||
if (error instanceof LLMError) return error
|
||||
const cause = error instanceof Error ? error.cause : undefined
|
||||
const failures = [error, cause]
|
||||
const code = failures.map(machineCode).find((value) => value !== undefined)
|
||||
const reason = (() => {
|
||||
if (
|
||||
error instanceof ChunkTimeoutError ||
|
||||
failures.some((failure) => failure instanceof Error && failure.name === "TimeoutError") ||
|
||||
(code !== undefined && TRANSPORT_TIMEOUT_CODES.has(code))
|
||||
)
|
||||
return new TransportReason({ message: errorMessage(error), kind: code ?? "Timeout" })
|
||||
if (APICallError.isInstance(error)) return apiCallReason(error)
|
||||
if (code !== undefined && TRANSPORT_CONNECTION_CODES.has(code))
|
||||
return new TransportReason({ message: errorMessage(error), kind: code })
|
||||
const malformed = failures.find(isMalformedError)
|
||||
if (malformed) return new InvalidProviderOutputReason({ message: malformed.message })
|
||||
if (LoadAPIKeyError.isInstance(error))
|
||||
return new AuthenticationReason({ message: error.message, kind: "missing" })
|
||||
if (NoSuchModelError.isInstance(error)) return new InvalidRequestReason({ message: error.message })
|
||||
if (
|
||||
LoadSettingError.isInstance(error) ||
|
||||
InvalidPromptError.isInstance(error) ||
|
||||
InvalidArgumentError.isInstance(error) ||
|
||||
UnsupportedFunctionalityError.isInstance(error)
|
||||
)
|
||||
return new InvalidRequestReason({ message: error.message })
|
||||
const providerCode = apiFailureCode(error)
|
||||
if (providerCode)
|
||||
return classifyProviderFailure({ message: apiFailureMessage(error), code: providerCode })
|
||||
return new UnknownProviderReason({ message: errorMessage(error) })
|
||||
})()
|
||||
return new LLMError({
|
||||
module: "AISDK",
|
||||
method,
|
||||
|
|
@ -727,4 +777,107 @@ function llmError(method: string, error: unknown) {
|
|||
})
|
||||
}
|
||||
|
||||
function apiCallReason(error: APICallError) {
|
||||
const malformed = isMalformedError(error.cause) ? error.cause : undefined
|
||||
if (error.statusCode !== undefined && error.statusCode < 400 && malformed)
|
||||
return new InvalidProviderOutputReason({ message: malformed.message })
|
||||
const code = apiFailureCode(error.data) ?? apiFailureCode(error.responseBody)
|
||||
if (error.statusCode === undefined) {
|
||||
if (code) return classifyProviderFailure({ message: error.message, code })
|
||||
if (error.isRetryable)
|
||||
return new TransportReason({ message: error.message, url: RequestExecutor.redactUrl(error.url) })
|
||||
return new UnknownProviderReason({ message: error.message })
|
||||
}
|
||||
const body = RequestExecutor.redactResponseBody(error.responseBody, {
|
||||
url: error.url,
|
||||
headers: error.responseHeaders ?? {},
|
||||
})
|
||||
return classifyProviderFailure({
|
||||
message: error.message,
|
||||
status: error.statusCode,
|
||||
code,
|
||||
retryAfterMs: retryAfterMs(error.responseHeaders),
|
||||
http: new HttpContext({
|
||||
request: new HttpRequestDetails({ method: "POST", url: RequestExecutor.redactUrl(error.url), headers: {} }),
|
||||
response: new HttpResponseDetails({
|
||||
status: error.statusCode,
|
||||
headers: RequestExecutor.redactHeaders(error.responseHeaders ?? {}),
|
||||
}),
|
||||
...body,
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
const TRANSPORT_TIMEOUT_CODES = new Set([
|
||||
"ETIMEDOUT",
|
||||
"UND_ERR_BODY_TIMEOUT",
|
||||
"UND_ERR_CONNECT_TIMEOUT",
|
||||
"UND_ERR_HEADERS_TIMEOUT",
|
||||
])
|
||||
const TRANSPORT_CONNECTION_CODES = new Set([
|
||||
"EAI_AGAIN",
|
||||
"ECONNREFUSED",
|
||||
"ECONNRESET",
|
||||
"EHOSTUNREACH",
|
||||
"ENETUNREACH",
|
||||
"ENOTFOUND",
|
||||
"EPIPE",
|
||||
"UND_ERR_SOCKET",
|
||||
])
|
||||
|
||||
function retryAfterMs(headers: Record<string, string> | undefined) {
|
||||
if (!headers) return undefined
|
||||
const millis = Number(headers["retry-after-ms"])
|
||||
if (Number.isFinite(millis)) return Math.max(0, millis)
|
||||
const value = headers["retry-after"]
|
||||
if (!value) return undefined
|
||||
const seconds = Number(value)
|
||||
if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000)
|
||||
const date = Date.parse(value)
|
||||
return Number.isNaN(date) ? undefined : Math.max(0, date - Date.now())
|
||||
}
|
||||
|
||||
function field(error: unknown, name: string) {
|
||||
return typeof error === "object" && error !== null ? Reflect.get(error, name) : undefined
|
||||
}
|
||||
|
||||
function machineCode(error: unknown) {
|
||||
const code = field(error, "code")
|
||||
return typeof code === "string" ? code.toUpperCase() : undefined
|
||||
}
|
||||
|
||||
function apiFailureCode(error: unknown): string | undefined {
|
||||
if (typeof error === "string") {
|
||||
const decoded = Option.getOrUndefined(Schema.decodeUnknownOption(Schema.UnknownFromJsonString)(error))
|
||||
return apiFailureCode(decoded)
|
||||
}
|
||||
const code = field(error, "code")
|
||||
if (typeof code === "string") return code
|
||||
const type = field(error, "type")
|
||||
if (typeof type === "string" && type !== "error") return type
|
||||
const nested = field(error, "error")
|
||||
return nested === undefined || nested === error ? undefined : apiFailureCode(nested)
|
||||
}
|
||||
|
||||
function apiFailureMessage(error: unknown) {
|
||||
const message = field(error, "message")
|
||||
if (typeof message === "string") return message
|
||||
const nested = field(field(error, "error"), "message")
|
||||
return typeof nested === "string" ? nested : String(error)
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown) {
|
||||
return error instanceof Error ? error.message : String(error)
|
||||
}
|
||||
|
||||
function isMalformedError(error: unknown): error is Error {
|
||||
return (
|
||||
InvalidResponseDataError.isInstance(error) ||
|
||||
JSONParseError.isInstance(error) ||
|
||||
TypeValidationError.isInstance(error) ||
|
||||
EmptyResponseBodyError.isInstance(error) ||
|
||||
NoContentGeneratedError.isInstance(error)
|
||||
)
|
||||
}
|
||||
|
||||
export const node = makeLocationNode({ service: Service, layer: locationLayer, deps: [] })
|
||||
|
|
|
|||
|
|
@ -1,11 +1,17 @@
|
|||
import type { LanguageModelV3CallOptions } from "@ai-sdk/provider"
|
||||
import {
|
||||
APICallError,
|
||||
InvalidResponseDataError,
|
||||
type LanguageModelV3,
|
||||
type LanguageModelV3CallOptions,
|
||||
type LanguageModelV3StreamPart,
|
||||
} from "@ai-sdk/provider"
|
||||
import { AISDK } from "@opencode-ai/core/aisdk"
|
||||
import { ModelV2 } from "@opencode-ai/core/model"
|
||||
import { ProviderV2 } from "@opencode-ai/core/provider"
|
||||
import { LLM, Message } from "@opencode-ai/ai"
|
||||
import { LLMClient } from "@opencode-ai/ai/route"
|
||||
import { expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { Effect, Stream } from "effect"
|
||||
import { testEffect } from "./lib/effect"
|
||||
|
||||
const it = testEffect(AISDK.locationLayer)
|
||||
|
|
@ -19,6 +25,49 @@ const model = (packageName: string, settings: Record<string, unknown> = {}) =>
|
|||
limit: { context: 100, output: 20 },
|
||||
})
|
||||
|
||||
const failingLanguage = (error: unknown): LanguageModelV3 => ({
|
||||
specificationVersion: "v3",
|
||||
provider: "test-provider",
|
||||
modelId: "api-model",
|
||||
supportedUrls: {},
|
||||
doGenerate: async () => {
|
||||
throw error
|
||||
},
|
||||
doStream: async () => {
|
||||
throw error
|
||||
},
|
||||
})
|
||||
|
||||
const streamingLanguage = (...events: LanguageModelV3StreamPart[]): LanguageModelV3 => ({
|
||||
...failingLanguage(new Error("unused")),
|
||||
doStream: async () => ({
|
||||
stream: new ReadableStream<LanguageModelV3StreamPart>({
|
||||
start(controller) {
|
||||
events.forEach((event) => controller.enqueue(event))
|
||||
controller.close()
|
||||
},
|
||||
}),
|
||||
request: { body: {} },
|
||||
}),
|
||||
})
|
||||
|
||||
const streamFailure = (language: LanguageModelV3) =>
|
||||
Effect.gen(function* () {
|
||||
const aisdk = yield* AISDK.Service
|
||||
yield* aisdk.hook.sdk((event) => {
|
||||
event.sdk = {}
|
||||
})
|
||||
yield* aisdk.hook.language((event) => {
|
||||
event.language = language
|
||||
})
|
||||
const resolved = yield* aisdk.model(model("@ai-sdk/openai"))
|
||||
const request = LLM.request({ model: resolved, prompt: "Hello" })
|
||||
const prepared = yield* LLMClient.prepare<LanguageModelV3CallOptions>(request)
|
||||
return yield* resolved.route
|
||||
.streamPrepared(prepared.body, request, { http: { execute: () => Effect.die("unused") } })
|
||||
.pipe(Stream.runDrain, Effect.flip)
|
||||
})
|
||||
|
||||
it.effect("keys language models by package and flattened overlays", () =>
|
||||
Effect.gen(function* () {
|
||||
const aisdk = yield* AISDK.Service
|
||||
|
|
@ -238,3 +287,77 @@ it.effect("projects replay metadata onto AI SDK prompt parts", () =>
|
|||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("classifies AI SDK API failures with redacted diagnostics", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* streamFailure(
|
||||
failingLanguage(
|
||||
new APICallError({
|
||||
message: "Quota exceeded",
|
||||
url: "https://provider.test/v1?key=url-secret",
|
||||
requestBodyValues: {},
|
||||
statusCode: 429,
|
||||
responseHeaders: { "retry-after-ms": "250", "x-api-key": "header-secret" },
|
||||
responseBody: '{"error":{"code":"insufficient_quota","api_key":"body-secret"}}',
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
||||
expect(error).toMatchObject({
|
||||
reason: {
|
||||
_tag: "QuotaExceeded",
|
||||
http: {
|
||||
request: { url: "https://provider.test/v1?key=%3Credacted%3E" },
|
||||
response: { headers: { "x-api-key": "<redacted>" } },
|
||||
body: '{"error":{"code":"insufficient_quota","api_key":"<redacted>"}}',
|
||||
},
|
||||
},
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("classifies AI SDK timeouts", () =>
|
||||
Effect.gen(function* () {
|
||||
const timeout = yield* streamFailure(failingLanguage(new DOMException("timed out", "TimeoutError")))
|
||||
|
||||
expect(timeout).toMatchObject({ reason: { _tag: "Transport", kind: "Timeout" } })
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("classifies AI SDK connection failures", () =>
|
||||
Effect.gen(function* () {
|
||||
const reset = yield* streamFailure(
|
||||
failingLanguage(Object.assign(new Error("connection reset"), { code: "ECONNRESET" })),
|
||||
)
|
||||
|
||||
expect(reset).toMatchObject({ reason: { _tag: "Transport", kind: "ECONNRESET" } })
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("classifies structured AI SDK stream errors", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* streamFailure(
|
||||
streamingLanguage({ type: "error", error: { type: "overloaded_error", message: "Overloaded" } }),
|
||||
)
|
||||
|
||||
expect(error).toMatchObject({ reason: { _tag: "ProviderInternal", message: "Overloaded" } })
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("classifies malformed AI SDK response causes", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* streamFailure(
|
||||
failingLanguage(
|
||||
new APICallError({
|
||||
message: "Failed to process response",
|
||||
url: "https://provider.test/v1",
|
||||
requestBodyValues: {},
|
||||
statusCode: 200,
|
||||
cause: new InvalidResponseDataError({ data: { invalid: true } }),
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
||||
expect(error).toMatchObject({ reason: { _tag: "InvalidProviderOutput" } })
|
||||
}),
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue