Add native LLM core foundation (#24712)

packages/llm/src/protocols/utils/bedrock-auth.ts

@@ -0,0 +1,103 @@
+import { AwsV4Signer } from "aws4fetch"
+import { Effect, Option, Schema } from "effect"
+import { Headers } from "effect/unstable/http"
+import { Auth, type AuthInput } from "../../route/auth"
+import type { LLMRequest } from "../../schema"
+import { ProviderShared } from "../shared"
+
+/**
+ * AWS credentials for SigV4 signing. Bedrock also supports Bearer API key auth
+ * via `model.apiKey`, which bypasses SigV4 signing. STS-vended credentials
+ * should be refreshed by the consumer (rebuild the model) before they expire;
+ * the route does not refresh.
+ */
+export interface Credentials {
+  readonly region: string
+  readonly accessKeyId: string
+  readonly secretAccessKey: string
+  readonly sessionToken?: string
+}
+
+const NativeCredentials = Schema.Struct({
+  accessKeyId: Schema.String,
+  secretAccessKey: Schema.String,
+  region: Schema.optional(Schema.String),
+  sessionToken: Schema.optional(Schema.String),
+})
+
+const decodeNativeCredentials = Schema.decodeUnknownOption(NativeCredentials)
+
+export const region = (request: LLMRequest) => {
+  const fromNative = request.model.native?.aws_region
+  if (typeof fromNative === "string" && fromNative !== "") return fromNative
+  return (
+    decodeNativeCredentials(request.model.native?.aws_credentials).pipe(
+      Option.map((credentials) => credentials.region),
+      Option.getOrUndefined,
+    ) ?? "us-east-1"
+  )
+}
+
+const credentialsFromInput = (request: LLMRequest): Credentials | undefined =>
+  decodeNativeCredentials(request.model.native?.aws_credentials).pipe(
+    Option.map((creds) => ({ ...creds, region: creds.region ?? region(request) })),
+    Option.getOrUndefined,
+  )
+
+const signRequest = (input: {
+  readonly url: string
+  readonly body: string
+  readonly headers: Headers.Headers
+  readonly credentials: Credentials
+}) =>
+  Effect.tryPromise({
+    try: async () => {
+      const signed = await new AwsV4Signer({
+        url: input.url,
+        method: "POST",
+        headers: Object.entries(input.headers),
+        body: input.body,
+        region: input.credentials.region,
+        accessKeyId: input.credentials.accessKeyId,
+        secretAccessKey: input.credentials.secretAccessKey,
+        sessionToken: input.credentials.sessionToken,
+        service: "bedrock",
+      }).sign()
+      return Object.fromEntries(signed.headers.entries())
+    },
+    catch: (error) =>
+      ProviderShared.invalidRequest(
+        `Bedrock Converse SigV4 signing failed: ${error instanceof Error ? error.message : String(error)}`,
+      ),
+  })
+
+/**
+ * Bedrock auth. `model.apiKey` (Bedrock's newer Bearer API key auth) wins if
+ * set; otherwise sign the exact JSON bytes with SigV4 using credentials from
+ * `model.native.aws_credentials`.
+ */
+export const auth = Auth.custom((input: AuthInput) => {
+  if (input.request.model.apiKey) return Auth.toEffect(Auth.bearer())(input)
+  return Effect.gen(function* () {
+    const credentials = credentialsFromInput(input.request)
+    if (!credentials) {
+      return yield* ProviderShared.invalidRequest(
+        "Bedrock Converse requires either model.apiKey or AWS credentials in model.native.aws_credentials",
+      )
+    }
+    const headersForSigning = Headers.set(input.headers, "content-type", "application/json")
+    const signed = yield* signRequest({ url: input.url, body: input.body, headers: headersForSigning, credentials })
+    return Headers.setAll(headersForSigning, signed)
+  })
+})
+
+export const nativeCredentials = (native: Record<string, unknown> | undefined, credentials: Credentials | undefined) =>
+  credentials
+    ? {
+        ...native,
+        aws_credentials: credentials,
+        aws_region: credentials.region,
+      }
+    : native
+
+export * as BedrockAuth from "./bedrock-auth"

packages/llm/src/protocols/utils/bedrock-cache.ts

@@ -0,0 +1,20 @@
+import { Schema } from "effect"
+import type { CacheHint } from "../../schema"
+
+// Bedrock cache markers are positional: emit a `cachePoint` block immediately
+// after the content the caller wants treated as a cacheable prefix.
+export const CachePointBlock = Schema.Struct({
+  cachePoint: Schema.Struct({ type: Schema.tag("default") }),
+})
+export type CachePointBlock = Schema.Schema.Type<typeof CachePointBlock>
+
+// Bedrock recently added optional `ttl: "5m" | "1h"` on cachePoint. Map
+// `CacheHint.ttlSeconds` here once a recorded cassette validates the wire shape.
+const DEFAULT: CachePointBlock = { cachePoint: { type: "default" } }
+
+export const block = (cache: CacheHint | undefined): CachePointBlock | undefined => {
+  if (cache?.type !== "ephemeral" && cache?.type !== "persistent") return undefined
+  return DEFAULT
+}
+
+export * as BedrockCache from "./bedrock-cache"

packages/llm/src/protocols/utils/bedrock-media.ts

@@ -0,0 +1,80 @@
+import { Effect, Schema } from "effect"
+import type { MediaPart } from "../../schema"
+import { ProviderShared } from "../shared"
+
+// Bedrock Converse accepts image `format` as the file extension and
+// `source.bytes` as base64 in the JSON wire format.
+export const ImageFormat = Schema.Literals(["png", "jpeg", "gif", "webp"])
+export type ImageFormat = Schema.Schema.Type<typeof ImageFormat>
+
+export const ImageBlock = Schema.Struct({
+  image: Schema.Struct({
+    format: ImageFormat,
+    source: Schema.Struct({ bytes: Schema.String }),
+  }),
+})
+export type ImageBlock = Schema.Schema.Type<typeof ImageBlock>
+
+// Bedrock document blocks require a user-facing name so the model can refer to
+// the uploaded document.
+export const DocumentFormat = Schema.Literals(["pdf", "csv", "doc", "docx", "xls", "xlsx", "html", "txt", "md"])
+export type DocumentFormat = Schema.Schema.Type<typeof DocumentFormat>
+
+export const DocumentBlock = Schema.Struct({
+  document: Schema.Struct({
+    format: DocumentFormat,
+    name: Schema.String,
+    source: Schema.Struct({ bytes: Schema.String }),
+  }),
+})
+export type DocumentBlock = Schema.Schema.Type<typeof DocumentBlock>
+
+const IMAGE_FORMATS = {
+  "image/png": "png",
+  "image/jpeg": "jpeg",
+  "image/jpg": "jpeg",
+  "image/gif": "gif",
+  "image/webp": "webp",
+} as const satisfies Record<string, ImageFormat>
+
+const DOCUMENT_FORMATS = {
+  "application/pdf": "pdf",
+  "text/csv": "csv",
+  "application/msword": "doc",
+  "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
+  "application/vnd.ms-excel": "xls",
+  "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",
+  "text/html": "html",
+  "text/plain": "txt",
+  "text/markdown": "md",
+} as const satisfies Record<string, DocumentFormat>
+
+const imageBlock = (part: MediaPart, format: ImageFormat): ImageBlock => ({
+  image: { format, source: { bytes: ProviderShared.mediaBytes(part) } },
+})
+
+const documentBlock = (part: MediaPart, format: DocumentFormat): DocumentBlock => ({
+  document: {
+    format,
+    name: part.filename ?? `document.${format}`,
+    source: { bytes: ProviderShared.mediaBytes(part) },
+  },
+})
+
+// Route by MIME. Known image/document formats lower into a typed block; anything
+// else fails with a clear error instead of silently degrading to a malformed
+// document block. Image MIME types not in `IMAGE_FORMATS` (e.g. `image/svg+xml`)
+// get an image-specific error so the caller knows it's a format-support issue,
+// not a kind-detection issue.
+export const lower = (part: MediaPart) => {
+  const mime = part.mediaType.toLowerCase()
+  const imageFormat = IMAGE_FORMATS[mime as keyof typeof IMAGE_FORMATS]
+  if (imageFormat) return Effect.succeed(imageBlock(part, imageFormat))
+  if (mime.startsWith("image/"))
+    return ProviderShared.invalidRequest(`Bedrock Converse does not support image media type ${part.mediaType}`)
+  const documentFormat = DOCUMENT_FORMATS[mime as keyof typeof DOCUMENT_FORMATS]
+  if (documentFormat) return Effect.succeed(documentBlock(part, documentFormat))
+  return ProviderShared.invalidRequest(`Bedrock Converse does not support media type ${part.mediaType}`)
+}
+
+export * as BedrockMedia from "./bedrock-media"

packages/llm/src/protocols/utils/gemini-tool-schema.ts

@@ -0,0 +1,101 @@
+import { ProviderShared } from "../shared"
+
+// Gemini accepts a JSON Schema-like dialect for tool parameters, but rejects a
+// handful of common JSON Schema shapes. Keep this projection isolated so the
+// Gemini protocol file still reads like the other protocol modules.
+const SCHEMA_INTENT_KEYS = [
+  "type",
+  "properties",
+  "items",
+  "prefixItems",
+  "enum",
+  "const",
+  "$ref",
+  "additionalProperties",
+  "patternProperties",
+  "required",
+  "not",
+  "if",
+  "then",
+  "else",
+]
+
+const isRecord = ProviderShared.isRecord
+
+const hasCombiner = (schema: unknown) =>
+  isRecord(schema) && (Array.isArray(schema.anyOf) || Array.isArray(schema.oneOf) || Array.isArray(schema.allOf))
+
+const hasSchemaIntent = (schema: unknown) =>
+  isRecord(schema) && (hasCombiner(schema) || SCHEMA_INTENT_KEYS.some((key) => key in schema))
+
+const sanitizeNode = (schema: unknown): unknown => {
+  if (!isRecord(schema)) return Array.isArray(schema) ? schema.map(sanitizeNode) : schema
+
+  const result: Record<string, unknown> = Object.fromEntries(
+    Object.entries(schema).map(([key, value]) => [
+      key,
+      key === "enum" && Array.isArray(value) ? value.map(String) : sanitizeNode(value),
+    ]),
+  )
+
+  if (Array.isArray(result.enum) && (result.type === "integer" || result.type === "number")) result.type = "string"
+
+  const properties = result.properties
+  if (result.type === "object" && isRecord(properties) && Array.isArray(result.required)) {
+    result.required = result.required.filter((field) => typeof field === "string" && field in properties)
+  }
+
+  if (result.type === "array" && !hasCombiner(result)) {
+    result.items = result.items ?? {}
+    if (isRecord(result.items) && !hasSchemaIntent(result.items)) result.items = { ...result.items, type: "string" }
+  }
+
+  if (typeof result.type === "string" && result.type !== "object" && !hasCombiner(result)) {
+    delete result.properties
+    delete result.required
+  }
+
+  return result
+}
+
+const emptyObjectSchema = (schema: Record<string, unknown>) =>
+  schema.type === "object" &&
+  (!isRecord(schema.properties) || Object.keys(schema.properties).length === 0) &&
+  !schema.additionalProperties
+
+const projectNode = (schema: unknown): Record<string, unknown> | undefined => {
+  if (!isRecord(schema)) return undefined
+  if (emptyObjectSchema(schema)) return undefined
+  return Object.fromEntries(
+    [
+      ["description", schema.description],
+      ["required", schema.required],
+      ["format", schema.format],
+      ["type", Array.isArray(schema.type) ? schema.type.filter((type) => type !== "null")[0] : schema.type],
+      ["nullable", Array.isArray(schema.type) && schema.type.includes("null") ? true : undefined],
+      ["enum", schema.const !== undefined ? [schema.const] : schema.enum],
+      [
+        "properties",
+        isRecord(schema.properties)
+          ? Object.fromEntries(Object.entries(schema.properties).map(([key, value]) => [key, projectNode(value)]))
+          : undefined,
+      ],
+      [
+        "items",
+        Array.isArray(schema.items)
+          ? schema.items.map(projectNode)
+          : schema.items === undefined
+            ? undefined
+            : projectNode(schema.items),
+      ],
+      ["allOf", Array.isArray(schema.allOf) ? schema.allOf.map(projectNode) : undefined],
+      ["anyOf", Array.isArray(schema.anyOf) ? schema.anyOf.map(projectNode) : undefined],
+      ["oneOf", Array.isArray(schema.oneOf) ? schema.oneOf.map(projectNode) : undefined],
+      ["minLength", schema.minLength],
+    ].filter((entry) => entry[1] !== undefined),
+  )
+}
+
+export const convert = (schema: unknown) => projectNode(sanitizeNode(schema))
+
+export * as GeminiToolSchema from "./gemini-tool-schema"

packages/llm/src/protocols/utils/openai-options.ts

@@ -0,0 +1,55 @@
+import { Schema } from "effect"
+import type { LLMRequest, ReasoningEffort, TextVerbosity as TextVerbosityValue } from "../../schema"
+import { ReasoningEfforts, TextVerbosity } from "../../schema"
+
+export const OpenAIReasoningEfforts = ReasoningEfforts.filter(
+  (effort): effort is Exclude<ReasoningEffort, "max"> => effort !== "max",
+)
+export type OpenAIReasoningEffort = (typeof OpenAIReasoningEfforts)[number]
+
+const REASONING_EFFORTS = new Set<string>(ReasoningEfforts)
+const OPENAI_REASONING_EFFORTS = new Set<string>(OpenAIReasoningEfforts)
+const TEXT_VERBOSITY = new Set<string>(["low", "medium", "high"])
+
+export const OpenAIReasoningEffort = Schema.Literals(OpenAIReasoningEfforts)
+export const OpenAITextVerbosity = TextVerbosity
+
+const isAnyReasoningEffort = (effort: unknown): effort is ReasoningEffort =>
+  typeof effort === "string" && REASONING_EFFORTS.has(effort)
+
+export const isReasoningEffort = (effort: unknown): effort is OpenAIReasoningEffort =>
+  typeof effort === "string" && OPENAI_REASONING_EFFORTS.has(effort)
+
+const isTextVerbosity = (value: unknown): value is TextVerbosityValue =>
+  typeof value === "string" && TEXT_VERBOSITY.has(value)
+
+const options = (request: LLMRequest) => request.providerOptions?.openai
+
+export const store = (request: LLMRequest): boolean | undefined => {
+  const value = options(request)?.store
+  return typeof value === "boolean" ? value : undefined
+}
+
+export const reasoningEffort = (request: LLMRequest): ReasoningEffort | undefined => {
+  const value = options(request)?.reasoningEffort
+  return isAnyReasoningEffort(value) ? value : undefined
+}
+
+export const reasoningSummary = (request: LLMRequest): "auto" | undefined => {
+  return options(request)?.reasoningSummary === "auto" ? "auto" : undefined
+}
+
+export const encryptedReasoning = (request: LLMRequest) =>
+  options(request)?.includeEncryptedReasoning === true ? true : undefined
+
+export const promptCacheKey = (request: LLMRequest) => {
+  const value = options(request)?.promptCacheKey
+  return typeof value === "string" ? value : undefined
+}
+
+export const textVerbosity = (request: LLMRequest) => {
+  const value = options(request)?.textVerbosity
+  return isTextVerbosity(value) ? value : undefined
+}
+
+export * as OpenAIOptions from "./openai-options"

packages/llm/src/protocols/utils/tool-stream.ts

@@ -0,0 +1,196 @@
+import { Effect } from "effect"
+import { LLMError, type ProviderMetadata, type ToolCall, type ToolInputDelta } from "../../schema"
+import { eventError, parseToolInput, type ToolAccumulator } from "../shared"
+
+type StreamKey = string | number
+
+/**
+ * One pending streamed tool call. Providers emit the tool identity and JSON
+ * argument text across separate chunks; `input` is the raw JSON string collected
+ * so far, not the parsed object.
+ */
+export interface PendingTool extends ToolAccumulator {
+  readonly providerExecuted?: boolean
+  readonly providerMetadata?: ProviderMetadata
+}
+
+/**
+ * Sparse parser state keyed by the provider's stream-local tool identifier.
+ *
+ * This key is not the final tool-call id (`call_...`). It is the id/index the
+ * provider uses while streaming a partial call: OpenAI Chat / Anthropic /
+ * Bedrock use numeric content indexes, while OpenAI Responses uses string
+ * `item_id`s. The generic keeps each protocol internally consistent.
+ */
+export type State<K extends StreamKey> = Partial<Record<K, PendingTool>>
+
+/**
+ * Result of adding argument text to one pending tool call. It returns both the
+ * next `tools` state and the updated `tool` because parsers often need the
+ * current id/name immediately. `event` is present only when new text arrived;
+ * metadata-only deltas update identity without emitting `tool-input-delta`.
+ */
+export interface AppendOutcome<K extends StreamKey> {
+  readonly tools: State<K>
+  readonly tool: PendingTool
+  readonly event?: ToolInputDelta
+}
+
+/** Create empty accumulator state for one provider stream. */
+export const empty = <K extends StreamKey>(): State<K> => ({})
+
+const withTool = <K extends StreamKey>(tools: State<K>, key: K, tool: PendingTool): State<K> => {
+  return { ...tools, [key]: tool }
+}
+
+const withoutTool = <K extends StreamKey>(tools: State<K>, key: K): State<K> => {
+  const next = { ...tools }
+  delete next[key]
+  return next
+}
+
+const inputDelta = (tool: PendingTool, text: string): ToolInputDelta => ({
+  type: "tool-input-delta",
+  id: tool.id,
+  name: tool.name,
+  text,
+  ...(tool.providerMetadata ? { providerMetadata: tool.providerMetadata } : {}),
+})
+
+const toolCall = (route: string, tool: PendingTool, inputOverride?: string) =>
+  parseToolInput(route, tool.name, inputOverride ?? tool.input).pipe(
+    Effect.map(
+      (input): ToolCall =>
+        tool.providerExecuted
+          ? {
+              type: "tool-call",
+              id: tool.id,
+              name: tool.name,
+              input,
+              providerExecuted: true,
+              ...(tool.providerMetadata ? { providerMetadata: tool.providerMetadata } : {}),
+            }
+          : {
+              type: "tool-call",
+              id: tool.id,
+              name: tool.name,
+              input,
+              ...(tool.providerMetadata ? { providerMetadata: tool.providerMetadata } : {}),
+            },
+    ),
+  )
+
+/** Store the updated tool and produce the optional public delta event. */
+const appendTool = <K extends StreamKey>(
+  tools: State<K>,
+  key: K,
+  tool: PendingTool,
+  text: string,
+): AppendOutcome<K> => ({
+  tools: withTool(tools, key, tool),
+  tool,
+  event: text.length === 0 ? undefined : inputDelta(tool, text),
+})
+
+export const isError = <K extends StreamKey>(result: AppendOutcome<K> | LLMError): result is LLMError =>
+  result instanceof LLMError
+
+/**
+ * Register a tool call whose start event arrived before any argument deltas.
+ * Used by Anthropic `content_block_start`, Bedrock `contentBlockStart`, and
+ * OpenAI Responses `response.output_item.added`.
+ */
+export const start = <K extends StreamKey>(
+  tools: State<K>,
+  key: K,
+  tool: Omit<PendingTool, "input"> & { readonly input?: string },
+) => withTool(tools, key, { ...tool, input: tool.input ?? "" })
+
+/**
+ * Append a streamed argument delta, starting the tool if this provider encodes
+ * identity on the first delta instead of a separate start event. OpenAI Chat has
+ * this shape: `tool_calls[].index` is the stream key, and `id` / `name` may only
+ * appear on the first delta for that index.
+ */
+export const appendOrStart = <K extends StreamKey>(
+  route: string,
+  tools: State<K>,
+  key: K,
+  delta: { readonly id?: string; readonly name?: string; readonly text: string },
+  missingToolMessage: string,
+): AppendOutcome<K> | LLMError => {
+  const current = tools[key]
+  const id = delta.id ?? current?.id
+  const name = delta.name ?? current?.name
+  if (!id || !name) return eventError(route, missingToolMessage)
+
+  const tool = {
+    id,
+    name,
+    input: `${current?.input ?? ""}${delta.text}`,
+    providerExecuted: current?.providerExecuted,
+    providerMetadata: current?.providerMetadata,
+  }
+  if (current && delta.text.length === 0 && current.id === id && current.name === name) return { tools, tool: current }
+  return appendTool(tools, key, tool, delta.text)
+}
+
+/**
+ * Append argument text to a tool that must already have been started. This keeps
+ * protocols honest when their stream grammar promises a start event before any
+ * argument delta.
+ */
+export const appendExisting = <K extends StreamKey>(
+  route: string,
+  tools: State<K>,
+  key: K,
+  text: string,
+  missingToolMessage: string,
+): AppendOutcome<K> | LLMError => {
+  const current = tools[key]
+  if (!current) return eventError(route, missingToolMessage)
+  if (text.length === 0) return { tools, tool: current }
+  return appendTool(tools, key, { ...current, input: `${current.input}${text}` }, text)
+}
+
+/**
+ * Finalize one pending tool call: parse the accumulated raw JSON, remove it
+ * from state, and return the optional public `tool-call` event. Missing keys are
+ * a no-op because some providers emit stop events for non-tool content blocks.
+ */
+export const finish = <K extends StreamKey>(route: string, tools: State<K>, key: K) =>
+  Effect.gen(function* () {
+    const tool = tools[key]
+    if (!tool) return { tools }
+    return { tools: withoutTool(tools, key), event: yield* toolCall(route, tool) }
+  })
+
+/**
+ * Finalize one pending tool call with an authoritative final input string.
+ * OpenAI Responses can send accumulated deltas and then repeat the completed
+ * arguments on `response.output_item.done`; the final value wins.
+ */
+export const finishWithInput = <K extends StreamKey>(route: string, tools: State<K>, key: K, input: string) =>
+  Effect.gen(function* () {
+    const tool = tools[key]
+    if (!tool) return { tools }
+    return { tools: withoutTool(tools, key), event: yield* toolCall(route, tool, input) }
+  })
+
+/**
+ * Finalize every pending tool call at once. OpenAI Chat has this shape: it does
+ * not emit per-tool stop events, so all accumulated calls finish when the choice
+ * receives a terminal `finish_reason`.
+ */
+export const finishAll = <K extends StreamKey>(route: string, tools: State<K>) =>
+  Effect.gen(function* () {
+    const pending = Object.values<PendingTool | undefined>(tools).filter(
+      (tool): tool is PendingTool => tool !== undefined,
+    )
+    return {
+      tools: empty<K>(),
+      events: yield* Effect.forEach(pending, (tool) => toolCall(route, tool)),
+    }
+  })
+
+export * as ToolStream from "./tool-stream"