Refactor LLM route-first provider API (#28523)

packages/llm/src/protocols/anthropic-messages.ts

@@ -386,7 +386,7 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques
     tools,
     tool_choice: toolChoice,
     stream: true as const,
-    max_tokens: generation?.maxTokens ?? request.model.limits.output ?? 4096,
+    max_tokens: generation?.maxTokens ?? request.model.route.defaults.limits?.output ?? 4096,
     temperature: generation?.temperature,
     top_p: generation?.topP,
     top_k: generation?.topK,
@@ -452,8 +452,8 @@ const mergeUsage = (left: Usage | undefined, right: Usage | undefined) => {
     totalTokens: ProviderShared.totalTokens(inputTokens, outputTokens, undefined),
     providerMetadata: {
       anthropic: {
-        ...(left.providerMetadata?.["anthropic"] ?? {}),
-        ...(right.providerMetadata?.["anthropic"] ?? {}),
+        ...left.providerMetadata?.["anthropic"],
+        ...right.providerMetadata?.["anthropic"],
       },
     },
   })
@@ -673,19 +673,12 @@ export const protocol = Protocol.make({
 
 export const route = Route.make({
   id: ADAPTER,
+  provider: "anthropic",
   protocol,
-  endpoint: Endpoint.path(PATH),
-  auth: Auth.apiKeyHeader("x-api-key"),
+  endpoint: Endpoint.path(PATH, { baseURL: DEFAULT_BASE_URL }),
+  auth: Auth.none,
   framing: Framing.sse,
   headers: () => ({ "anthropic-version": "2023-06-01" }),
 })
 
-// =============================================================================
-// Model Helper
-// =============================================================================
-export const model = Route.model(route, {
-  provider: "anthropic",
-  baseURL: DEFAULT_BASE_URL,
-})
-
 export * as AnthropicMessages from "./anthropic-messages"

packages/llm/src/protocols/bedrock-converse.ts

@@ -1,5 +1,5 @@
 import { Effect, Schema } from "effect"
-import { Route, type RouteModelInput } from "../route/client"
+import { Route } from "../route/client"
 import { Endpoint } from "../route/endpoint"
 import { Protocol } from "../route/protocol"
 import {
@@ -14,7 +14,7 @@ import {
 } from "../schema"
 import { BedrockEventStream } from "./bedrock-event-stream"
 import { JsonObject, optionalArray, ProviderShared } from "./shared"
-import { BedrockAuth, type Credentials as BedrockCredentials } from "./utils/bedrock-auth"
+import { BedrockAuth } from "./utils/bedrock-auth"
 import { BedrockCache } from "./utils/bedrock-cache"
 import { BedrockMedia } from "./utils/bedrock-media"
 import { Lifecycle } from "./utils/lifecycle"
@@ -24,23 +24,6 @@ const ADAPTER = "bedrock-converse"
 
 export type { Credentials as BedrockCredentials } from "./utils/bedrock-auth"
 
-// =============================================================================
-// Public Model Input
-// =============================================================================
-export type BedrockConverseModelInput = RouteModelInput & {
-  /**
-   * Bearer API key (Bedrock's newer API key auth). Sets the `Authorization`
-   * header and bypasses SigV4 signing. Mutually exclusive with `credentials`.
-   */
-  readonly apiKey?: string
-  /**
-   * AWS credentials for SigV4 signing. The route signs each request at
-   * `toHttp` time using `aws4fetch`. Mutually exclusive with `apiKey`.
-   */
-  readonly credentials?: BedrockCredentials
-  readonly headers?: Record<string, string>
-}
-
 // =============================================================================
 // Request Body Schema
 // =============================================================================
@@ -61,6 +44,7 @@ type BedrockToolUseBlock = Schema.Schema.Type<typeof BedrockToolUseBlock>
 const BedrockToolResultContentItem = Schema.Union([
   Schema.Struct({ text: Schema.String }),
   Schema.Struct({ json: Schema.Unknown }),
+  BedrockMedia.ImageBlock,
 ])
 
 const BedrockToolResultBlock = Schema.Struct({
@@ -261,15 +245,33 @@ const lowerToolCall = (part: ToolCallPart): BedrockToolUseBlock => ({
   },
 })
 
-const lowerToolResult = (part: ToolResultPart): BedrockToolResultBlock => ({
-  toolResult: {
-    toolUseId: part.id,
-    content:
-      part.result.type === "text" || part.result.type === "error"
-        ? [{ text: ProviderShared.toolResultText(part) }]
-        : [{ json: part.result.value }],
-    status: part.result.type === "error" ? "error" : "success",
-  },
+const lowerToolResultContent = Effect.fn("BedrockConverse.lowerToolResultContent")(function* (part: ToolResultPart) {
+  if (part.result.type === "text" || part.result.type === "error")
+    return [{ text: ProviderShared.toolResultText(part) }]
+  if (part.result.type === "json") return [{ json: part.result.value }]
+
+  const content: Array<Schema.Schema.Type<typeof BedrockToolResultContentItem>> = []
+  for (const item of part.result.value) {
+    if (item.type === "text") {
+      content.push({ text: item.text })
+      continue
+    }
+    const media = yield* BedrockMedia.lower(item)
+    if (!("image" in media))
+      return yield* ProviderShared.invalidRequest("Bedrock Converse only supports image media in tool results")
+    content.push(media)
+  }
+  return content
+})
+
+const lowerToolResult = Effect.fn("BedrockConverse.lowerToolResult")(function* (part: ToolResultPart) {
+  return {
+    toolResult: {
+      toolUseId: part.id,
+      content: yield* lowerToolResultContent(part),
+      status: part.result.type === "error" ? "error" : "success",
+    },
+  } satisfies BedrockToolResultBlock
 })
 
 const lowerMessages = Effect.fn("BedrockConverse.lowerMessages")(function* (
@@ -331,7 +333,7 @@ const lowerMessages = Effect.fn("BedrockConverse.lowerMessages")(function* (
     for (const part of message.content) {
       if (!ProviderShared.supportsContent(part, ["tool-result"]))
         return yield* ProviderShared.unsupportedContent("Bedrock Converse", "tool", ["tool-result"])
-      content.push(lowerToolResult(part))
+      content.push(yield* lowerToolResult(part))
       const cachePoint = BedrockCache.block(breakpoints, part.cache)
       if (cachePoint) content.push(cachePoint)
     }
@@ -597,11 +599,11 @@ export const protocol = Protocol.make({
 
 export const route = Route.make({
   id: ADAPTER,
+  provider: "bedrock",
   protocol,
-  // Bedrock's URL embeds the region in the host (set on `model.baseURL` by
-  // the provider helper from credentials) and the validated modelId in the
-  // path. We read the validated body so the URL matches the body that gets
-  // signed.
+  // Bedrock's URL embeds the region in the route endpoint host and the
+  // validated modelId in the path. We read the validated body so the URL
+  // matches the body that gets signed.
   endpoint: Endpoint.path<BedrockConverseBody>(
     ({ body }) => `/model/${encodeURIComponent(body.modelId)}/converse-stream`,
   ),
@@ -609,26 +611,6 @@ export const route = Route.make({
   framing,
 })
 
-export const nativeCredentials = BedrockAuth.nativeCredentials
-
-const bedrockModel = Route.model(
-  route,
-  {
-    provider: "bedrock",
-  },
-  {
-    mapInput: (input: BedrockConverseModelInput) => {
-      const { credentials, ...rest } = input
-      const region = credentials?.region ?? "us-east-1"
-      return {
-        ...rest,
-        baseURL: rest.baseURL ?? `https://bedrock-runtime.${region}.amazonaws.com`,
-        native: nativeCredentials(input.native, credentials),
-      }
-    },
-  },
-)
-
-export const model = bedrockModel
+export const sigV4Auth = BedrockAuth.sigV4
 
 export * as BedrockConverse from "./bedrock-converse"

packages/llm/src/protocols/gemini.ts

@@ -404,19 +404,14 @@ export const protocol = Protocol.make({
 
 export const route = Route.make({
   id: ADAPTER,
+  provider: "google",
   protocol,
   // Gemini's path embeds the model id and pins SSE framing at the URL level.
-  endpoint: Endpoint.path(({ request }) => `/models/${request.model.id}:streamGenerateContent?alt=sse`),
-  auth: Auth.apiKeyHeader("x-goog-api-key"),
+  endpoint: Endpoint.path(({ request }) => `/models/${request.model.id}:streamGenerateContent?alt=sse`, {
+    baseURL: DEFAULT_BASE_URL,
+  }),
+  auth: Auth.none,
   framing: Framing.sse,
 })
 
-// =============================================================================
-// Model Helper
-// =============================================================================
-export const model = Route.model(route, {
-  provider: "google",
-  baseURL: DEFAULT_BASE_URL,
-})
-
 export * as Gemini from "./gemini"

packages/llm/src/protocols/openai-chat.ts

@@ -2,7 +2,6 @@ import { Array as Arr, Effect, Schema } from "effect"
 import { Route } from "../route/client"
 import { Auth } from "../route/auth"
 import { Endpoint } from "../route/endpoint"
-import { Framing } from "../route/framing"
 import { HttpTransport } from "../route/transport"
 import { Protocol } from "../route/protocol"
 import {
@@ -393,28 +392,15 @@ export const protocol = Protocol.make({
   },
 })
 
-const encodeBody = Schema.encodeSync(Schema.fromJsonString(OpenAIChatBody))
-
-export const httpTransport = HttpTransport.httpJson({
-  endpoint: Endpoint.path(PATH),
-  auth: Auth.bearer(),
-  framing: Framing.sse,
-  encodeBody,
-})
+export const httpTransport = HttpTransport.sseJson.with<OpenAIChatBody>()
 
 export const route = Route.make({
   id: ADAPTER,
   provider: "openai",
   protocol,
+  endpoint: Endpoint.path(PATH, { baseURL: DEFAULT_BASE_URL }),
+  auth: Auth.none,
   transport: httpTransport,
-  defaults: {
-    baseURL: DEFAULT_BASE_URL,
-  },
 })
 
-// =============================================================================
-// Model Helper
-// =============================================================================
-export const model = route.model
-
 export * as OpenAIChat from "./openai-chat"

packages/llm/src/protocols/openai-compatible-chat.ts

@@ -5,16 +5,14 @@ import * as OpenAIChat from "./openai-chat"
 
 const ADAPTER = "openai-compatible-chat"
 
-export type OpenAICompatibleChatModelInput = Omit<RouteRoutedModelInput, "baseURL"> & {
-  readonly baseURL: string
-}
+export type OpenAICompatibleChatModelInput = RouteRoutedModelInput
 
 /**
  * Route for non-OpenAI providers that expose an OpenAI Chat-compatible
  * `/chat/completions` endpoint. Reuses `OpenAIChat.protocol` end-to-end and
  * overrides only the route id so providers can be resolved per-family without
- * colliding with native OpenAI. The model carries the host on `baseURL`,
- * supplied by whichever profile/provider helper builds it.
+ * colliding with native OpenAI. Provider helpers configure the route endpoint
+ * before model selection.
  */
 export const route = Route.make({
   id: ADAPTER,
@@ -23,6 +21,4 @@ export const route = Route.make({
   framing: Framing.sse,
 })
 
-export const model = Route.model<OpenAICompatibleChatModelInput>(route)
-
 export * as OpenAICompatibleChat from "./openai-compatible-chat"

packages/llm/src/protocols/openai-responses.ts

@@ -2,11 +2,11 @@ import { Effect, Schema } from "effect"
 import { Route } from "../route/client"
 import { Auth } from "../route/auth"
 import { Endpoint } from "../route/endpoint"
-import { Framing } from "../route/framing"
 import { HttpTransport, WebSocketTransport } from "../route/transport"
 import { Protocol } from "../route/protocol"
 import {
   LLMEvent,
+  type MediaPart,
   Usage,
   type FinishReason,
   type LLMRequest,
@@ -31,6 +31,12 @@ const OpenAIResponsesInputText = Schema.Struct({
   type: Schema.tag("input_text"),
   text: Schema.String,
 })
+const OpenAIResponsesInputImage = Schema.Struct({
+  type: Schema.tag("input_image"),
+  image_url: Schema.String,
+})
+const OpenAIResponsesInputContent = Schema.Union([OpenAIResponsesInputText, OpenAIResponsesInputImage])
+type OpenAIResponsesInputContent = Schema.Schema.Type<typeof OpenAIResponsesInputContent>
 
 const OpenAIResponsesOutputText = Schema.Struct({
   type: Schema.tag("output_text"),
@@ -39,7 +45,7 @@ const OpenAIResponsesOutputText = Schema.Struct({
 
 const OpenAIResponsesInputItem = Schema.Union([
   Schema.Struct({ role: Schema.tag("system"), content: Schema.String }),
-  Schema.Struct({ role: Schema.tag("user"), content: Schema.Array(OpenAIResponsesInputText) }),
+  Schema.Struct({ role: Schema.tag("user"), content: Schema.Array(OpenAIResponsesInputContent) }),
   Schema.Struct({ role: Schema.tag("assistant"), content: Schema.Array(OpenAIResponsesOutputText) }),
   Schema.Struct({
     type: Schema.tag("function_call"),
@@ -151,12 +157,15 @@ const OpenAIResponsesEvent = Schema.Struct({
   item_id: Schema.optional(Schema.String),
   item: Schema.optional(OpenAIResponsesStreamItem),
   response: Schema.optional(
-    Schema.Struct({
-      id: Schema.optional(Schema.String),
-      service_tier: Schema.optional(Schema.String),
-      incomplete_details: optionalNull(Schema.Struct({ reason: Schema.String })),
-      usage: optionalNull(OpenAIResponsesUsage),
-    }),
+    Schema.StructWithRest(
+      Schema.Struct({
+        id: Schema.optional(Schema.String),
+        service_tier: optionalNull(Schema.String),
+        incomplete_details: optionalNull(Schema.Struct({ reason: Schema.String })),
+        usage: optionalNull(OpenAIResponsesUsage),
+      }),
+      [Schema.Record(Schema.String, Schema.Unknown)],
+    ),
   ),
   code: Schema.optional(Schema.String),
   message: Schema.optional(Schema.String),
@@ -196,6 +205,22 @@ const lowerToolCall = (part: ToolCallPart): OpenAIResponsesInputItem => ({
   arguments: ProviderShared.encodeJson(part.input),
 })
 
+const imageUrl = (part: MediaPart) =>
+  typeof part.data === "string" && part.data.startsWith("data:")
+    ? part.data
+    : `data:${part.mediaType};base64,${ProviderShared.mediaBytes(part)}`
+
+const lowerUserContent = Effect.fn("OpenAIResponses.lowerUserContent")(function* (
+  part: LLMRequest["messages"][number]["content"][number],
+) {
+  if (part.type === "text") return { type: "input_text" as const, text: part.text }
+  if (part.type === "media" && part.mediaType.startsWith("image/")) {
+    return { type: "input_image" as const, image_url: imageUrl(part) }
+  }
+  if (part.type === "media") return yield* invalid("OpenAI Responses user media content only supports images")
+  return yield* ProviderShared.unsupportedContent("OpenAI Responses", "user", ["text", "media"])
+})
+
 const lowerMessages = Effect.fn("OpenAIResponses.lowerMessages")(function* (request: LLMRequest) {
   const system: OpenAIResponsesInputItem[] =
     request.system.length === 0 ? [] : [{ role: "system", content: ProviderShared.joinText(request.system) }]
@@ -203,13 +228,7 @@ const lowerMessages = Effect.fn("OpenAIResponses.lowerMessages")(function* (requ
 
   for (const message of request.messages) {
     if (message.role === "user") {
-      const content: TextPart[] = []
-      for (const part of message.content) {
-        if (!ProviderShared.supportsContent(part, ["text"]))
-          return yield* ProviderShared.unsupportedContent("OpenAI Responses", "user", ["text"])
-        content.push(part)
-      }
-      input.push({ role: "user", content: content.map((part) => ({ type: "input_text", text: part.text })) })
+      input.push({ role: "user", content: yield* Effect.forEach(message.content, lowerUserContent) })
       continue
     }
 
@@ -536,27 +555,18 @@ export const protocol = Protocol.make({
   },
 })
 
-const encodeBody = Schema.encodeSync(Schema.fromJsonString(OpenAIResponsesBody))
-const transportBase = {
-  endpoint: Endpoint.path<OpenAIResponsesBody>(PATH),
-  auth: Auth.bearer(),
-  encodeBody,
-}
-const routeDefaults = {
-  baseURL: DEFAULT_BASE_URL,
-}
+const endpoint = Endpoint.path<OpenAIResponsesBody>(PATH, { baseURL: DEFAULT_BASE_URL })
+const auth = Auth.none
 
-export const httpTransport = HttpTransport.httpJson({
-  ...transportBase,
-  framing: Framing.sse,
-})
+export const httpTransport = HttpTransport.sseJson.with<OpenAIResponsesBody>()
 
 export const route = Route.make({
   id: ADAPTER,
   provider: "openai",
   protocol,
+  endpoint,
+  auth,
   transport: httpTransport,
-  defaults: routeDefaults,
 })
 
 const decodeWebSocketMessage = ProviderShared.validateWith(Schema.decodeUnknownEffect(OpenAIResponsesWebSocketMessage))
@@ -569,8 +579,10 @@ const webSocketMessage = (body: OpenAIResponsesBody | Record<string, unknown>) =
     return yield* decodeWebSocketMessage({ ...message, type: "response.create" })
   })
 
-export const webSocketTransport = WebSocketTransport.json({
-  ...transportBase,
+export const webSocketTransport = WebSocketTransport.jsonTransport.with<
+  OpenAIResponsesBody,
+  OpenAIResponsesWebSocketMessage
+>({
   toMessage: webSocketMessage,
   encodeMessage: encodeWebSocketMessage,
 })
@@ -579,15 +591,9 @@ export const webSocketRoute = Route.make({
   id: `${ADAPTER}-websocket`,
   provider: "openai",
   protocol,
+  endpoint,
+  auth,
   transport: webSocketTransport,
-  defaults: routeDefaults,
 })
 
-// =============================================================================
-// Model Helper
-// =============================================================================
-export const model = route.model
-
-export const webSocketModel = webSocketRoute.model
-
 export * as OpenAIResponses from "./openai-responses"

packages/llm/src/protocols/shared.ts

@@ -11,6 +11,7 @@ import {
   type MediaPart,
   type ToolResultPart,
 } from "../schema"
+export { isRecord } from "../utils/record"
 
 export const Json = Schema.fromJsonString(Schema.Unknown)
 export const decodeJson = Schema.decodeUnknownSync(Json)
@@ -19,13 +20,6 @@ export const JsonObject = Schema.Record(Schema.String, Schema.Unknown)
 export const optionalArray = <const S extends Schema.Top>(schema: S) => Schema.optional(Schema.Array(schema))
 export const optionalNull = <const S extends Schema.Top>(schema: S) => Schema.optional(Schema.NullOr(schema))
 
-/**
- * Plain-record narrowing. Excludes arrays so routes checking nested JSON
- * Schema fragments don't accidentally treat a tuple as a key/value bag.
- */
-export const isRecord = (value: unknown): value is Record<string, unknown> =>
-  typeof value === "object" && value !== null && !Array.isArray(value)
-
 /**
  * Streaming tool-call accumulator. Adapters that build a tool call across
  * multiple `tool-input-delta` chunks store the partial JSON input string here
@@ -132,6 +126,7 @@ export const trimBaseUrl = (value: string) => value.replace(/\/+$/, "")
 
 export const toolResultText = (part: ToolResultPart) => {
   if (part.result.type === "text" || part.result.type === "error") return String(part.result.value)
+  if (part.result.type === "content") return encodeJson(part.result.value)
   return encodeJson(part.result.value)
 }
 

packages/llm/src/protocols/utils/bedrock-auth.ts

@@ -1,15 +1,14 @@
 import { AwsV4Signer } from "aws4fetch"
-import { Effect, Option, Schema } from "effect"
+import { Effect } from "effect"
 import { Headers } from "effect/unstable/http"
 import { Auth, type AuthInput } from "../../route/auth"
-import type { LLMRequest } from "../../schema"
 import { ProviderShared } from "../shared"
 
 /**
- * AWS credentials for SigV4 signing. Bedrock also supports Bearer API key auth
- * via `model.apiKey`, which bypasses SigV4 signing. STS-vended credentials
- * should be refreshed by the consumer (rebuild the model) before they expire;
- * the route does not refresh.
+ * AWS credentials for SigV4 signing. Bedrock also supports Bearer API key auth,
+ * which provider facades configure as route auth instead of SigV4. STS-vended
+ * credentials should be refreshed by the consumer (rebuild the model) before
+ * they expire; the route does not refresh.
  */
 export interface Credentials {
   readonly region: string
@@ -18,32 +17,6 @@ export interface Credentials {
   readonly sessionToken?: string
 }
 
-const NativeCredentials = Schema.Struct({
-  accessKeyId: Schema.String,
-  secretAccessKey: Schema.String,
-  region: Schema.optional(Schema.String),
-  sessionToken: Schema.optional(Schema.String),
-})
-
-const decodeNativeCredentials = Schema.decodeUnknownOption(NativeCredentials)
-
-export const region = (request: LLMRequest) => {
-  const fromNative = request.model.native?.aws_region
-  if (typeof fromNative === "string" && fromNative !== "") return fromNative
-  return (
-    decodeNativeCredentials(request.model.native?.aws_credentials).pipe(
-      Option.map((credentials) => credentials.region),
-      Option.getOrUndefined,
-    ) ?? "us-east-1"
-  )
-}
-
-const credentialsFromInput = (request: LLMRequest): Credentials | undefined =>
-  decodeNativeCredentials(request.model.native?.aws_credentials).pipe(
-    Option.map((creds) => ({ ...creds, region: creds.region ?? region(request) })),
-    Option.getOrUndefined,
-  )
-
 const signRequest = (input: {
   readonly url: string
   readonly body: string
@@ -71,33 +44,27 @@ const signRequest = (input: {
       ),
   })
 
-/**
- * Bedrock auth. `model.apiKey` (Bedrock's newer Bearer API key auth) wins if
- * set; otherwise sign the exact JSON bytes with SigV4 using credentials from
- * `model.native.aws_credentials`.
- */
-export const auth = Auth.custom((input: AuthInput) => {
-  if (input.request.model.apiKey) return Auth.toEffect(Auth.bearer())(input)
-  return Effect.gen(function* () {
-    const credentials = credentialsFromInput(input.request)
-    if (!credentials) {
-      return yield* ProviderShared.invalidRequest(
-        "Bedrock Converse requires either model.apiKey or AWS credentials in model.native.aws_credentials",
-      )
-    }
-    const headersForSigning = Headers.set(input.headers, "content-type", "application/json")
-    const signed = yield* signRequest({ url: input.url, body: input.body, headers: headersForSigning, credentials })
-    return Headers.setAll(headersForSigning, signed)
-  })
-})
-
-export const nativeCredentials = (native: Record<string, unknown> | undefined, credentials: Credentials | undefined) =>
-  credentials
-    ? {
-        ...native,
-        aws_credentials: credentials,
-        aws_region: credentials.region,
+/** Sign the exact JSON bytes with SigV4 using credentials configured on the route. */
+export const sigV4 = (credentials: Credentials | undefined) =>
+  Auth.custom((input: AuthInput) => {
+    return Effect.gen(function* () {
+      if (!credentials) {
+        return yield* ProviderShared.invalidRequest(
+          "Bedrock Converse requires either route bearer auth or AWS credentials configured on the route",
+        )
       }
-    : native
+      const headersForSigning = Headers.set(input.headers, "content-type", "application/json")
+      const signed = yield* signRequest({
+        url: input.url,
+        body: input.body,
+        headers: headersForSigning,
+        credentials,
+      })
+      return Headers.setAll(headersForSigning, signed)
+    })
+  })
+
+/** Bedrock route auth defaults to SigV4 and expects credentials from route configuration. */
+export const auth = sigV4(undefined)
 
 export * as BedrockAuth from "./bedrock-auth"