feat(core): flatten provider config and load native packages (#35563)

Co-authored-by: Dax Raad <d@ironbay.co>

packages/llm/src/index.ts

@@ -1,6 +1,7 @@
 export { LLMClient } from "./route/client"
 export { Auth } from "./route/auth"
 export { Provider } from "./provider"
+export { ProviderPackage } from "./provider-package"
 export { isContextOverflow, isContextOverflowFailure } from "./provider-error"
 export type {
   RouteModelInput,
@@ -31,3 +32,4 @@ export type {
   ModelFactory as ProviderModelFactory,
   ModelOptions as ProviderModelOptions,
 } from "./provider"
+export type { Definition as ProviderPackageDefinition, Settings as ProviderPackageSettings } from "./provider-package"

packages/llm/src/provider-package.ts

@@ -0,0 +1,16 @@
+import type { Model } from "./schema"
+
+export interface Settings extends Readonly<Record<string, unknown>> {
+  readonly headers?: Readonly<Record<string, string>>
+  readonly body?: Readonly<Record<string, unknown>>
+  readonly limits?: {
+    readonly context: number
+    readonly output: number
+  }
+}
+
+export interface Definition<ProviderSettings extends Settings = Settings> {
+  readonly model: (modelID: string, settings: ProviderSettings) => Model
+}
+
+export * as ProviderPackage from "./provider-package"

packages/llm/src/providers/amazon-bedrock.ts

@@ -1,5 +1,6 @@
 import type { RouteDefaultsInput } from "../route/client"
 import { Auth } from "../route/auth"
+import type { ProviderPackage } from "../provider-package"
 import { ProviderID, type ModelID } from "../schema"
 import * as BedrockConverse from "../protocols/bedrock-converse"
 import type { BedrockCredentials } from "../protocols/bedrock-converse"
@@ -15,6 +16,15 @@ export type Config = RouteDefaultsInput & {
   /** Override the computed `https://bedrock-runtime.<region>.amazonaws.com` URL. */
   readonly baseURL?: string
 }
+
+export interface Settings extends ProviderPackage.Settings {
+  readonly apiKey?: string
+  readonly auth?: "bearer" | "sigv4"
+  readonly baseURL?: string
+  readonly credentials?: BedrockCredentials
+  readonly region?: string
+  readonly topP?: number
+}
 export const routes = [BedrockConverse.route]
 
 const bedrockBaseURL = (region: string) => `https://bedrock-runtime.${region}.amazonaws.com`
@@ -40,4 +50,19 @@ export const configure = (input: Config = {}) => {
 }
 
 export const provider = configure()
-export const model = provider.model
+export const model: ProviderPackage.Definition<Settings>["model"] = (modelID, settings) => {
+  if (settings.auth === "bearer" && settings.apiKey === undefined)
+    throw new Error("Amazon Bedrock bearer auth requires apiKey")
+  if (settings.auth === "sigv4" && settings.apiKey !== undefined)
+    throw new Error("Amazon Bedrock SigV4 auth does not accept apiKey")
+  return configure({
+    apiKey: settings.auth === "sigv4" ? undefined : settings.apiKey,
+    baseURL: settings.baseURL,
+    credentials: settings.credentials,
+    generation: settings.topP === undefined ? undefined : { topP: settings.topP },
+    headers: settings.headers === undefined ? undefined : { ...settings.headers },
+    http: settings.body === undefined ? undefined : { body: { ...settings.body } },
+    limits: settings.limits,
+    region: settings.region,
+  }).model(modelID)
+}

packages/llm/src/providers/anthropic.ts

@@ -1,6 +1,7 @@
 import type { RouteDefaultsInput } from "../route/client"
 import { Auth } from "../route/auth"
 import type { ProviderAuthOption } from "../route/auth-options"
+import type { ProviderPackage } from "../provider-package"
 import { ProviderID, type ModelID } from "../schema"
 import * as AnthropicMessages from "../protocols/anthropic-messages"
 
@@ -10,6 +11,12 @@ export const routes = [AnthropicMessages.route]
 
 export type Config = RouteDefaultsInput & ProviderAuthOption<"optional"> & { readonly baseURL?: string }
 
+export interface Settings extends ProviderPackage.Settings {
+  readonly apiKey?: string
+  readonly authToken?: string
+  readonly baseURL?: string
+}
+
 const auth = (options: ProviderAuthOption<"optional">) => {
   if ("auth" in options && options.auth) return options.auth
   return Auth.optional("apiKey" in options ? options.apiKey : undefined, "apiKey")
@@ -32,4 +39,11 @@ export const configure = (input: Config = {}) => {
 }
 
 export const provider = configure()
-export const model = provider.model
+export const model: ProviderPackage.Definition<Settings>["model"] = (modelID, settings) =>
+  configure({
+    ...(settings.authToken === undefined ? { apiKey: settings.apiKey } : { auth: Auth.bearer(settings.authToken) }),
+    baseURL: settings.baseURL,
+    headers: settings.headers === undefined ? undefined : { ...settings.headers },
+    http: settings.body === undefined ? undefined : { body: { ...settings.body } },
+    limits: settings.limits,
+  }).model(modelID)

packages/llm/src/providers/openai-compatible.ts

@@ -2,6 +2,7 @@ import { ProviderID, type ModelID } from "../schema"
 import * as OpenAICompatibleChat from "../protocols/openai-compatible-chat"
 import type { RouteDefaultsInput } from "../route/client"
 import { AuthOptions, type ProviderAuthOption } from "../route/auth-options"
+import type { ProviderPackage } from "../provider-package"
 import { profiles, type OpenAICompatibleProfile } from "./openai-compatible-profile"
 
 export const id = ProviderID.make("openai-compatible")
@@ -12,6 +13,12 @@ type GenericModelOptions = RouteDefaultsInput &
     readonly baseURL: string
   }
 
+export interface Settings extends ProviderPackage.Settings {
+  readonly apiKey?: string
+  readonly baseURL: string
+  readonly provider?: string
+}
+
 export type FamilyModelOptions = RouteDefaultsInput &
   ProviderAuthOption<"optional"> & {
     readonly baseURL?: string
@@ -56,6 +63,16 @@ export const provider = {
   configure,
 }
 
+export const model: ProviderPackage.Definition<Settings>["model"] = (modelID, settings) =>
+  configure({
+    apiKey: settings.apiKey,
+    baseURL: settings.baseURL,
+    headers: settings.headers === undefined ? undefined : { ...settings.headers },
+    http: settings.body === undefined ? undefined : { body: { ...settings.body } },
+    limits: settings.limits,
+    provider: settings.provider,
+  }).model(modelID)
+
 export const baseten = define(profiles.baseten)
 export const cerebras = define(profiles.cerebras)
 export const deepinfra = define(profiles.deepinfra)

packages/llm/src/providers/openai.ts

@@ -1,5 +1,6 @@
 import { AuthOptions, type ProviderAuthOption } from "../route/auth-options"
 import type { Route, RouteDefaultsInput } from "../route/client"
+import type { ProviderPackage } from "../provider-package"
 import { ProviderID, type ModelID } from "../schema"
 import * as OpenAIChat from "../protocols/openai-chat"
 import * as OpenAIResponses from "../protocols/openai-responses"
@@ -21,6 +22,16 @@ export type Config = RouteDefaultsInput &
     readonly providerOptions?: OpenAIProviderOptionsInput
   }
 
+export interface Settings extends ProviderPackage.Settings {
+  readonly apiKey?: string
+  readonly baseURL?: string
+  readonly organization?: string
+  readonly project?: string
+  readonly queryParams?: Readonly<Record<string, string>>
+  readonly transport?: "http" | "websocket"
+  readonly providerOptions?: OpenAIProviderOptionsInput
+}
+
 const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "OPENAI_API_KEY")
 
 const defaults = (input: Config) => {
@@ -57,7 +68,32 @@ export const configure = (input: Config = {}) => {
 
 export const provider = configure()
 
-export const model = provider.model
+const config = (settings: Settings): Config => {
+  const headers = {
+    ...(settings.organization === undefined ? {} : { "OpenAI-Organization": settings.organization }),
+    ...(settings.project === undefined ? {} : { "OpenAI-Project": settings.project }),
+    ...settings.headers,
+  }
+  return {
+    apiKey: settings.apiKey,
+    baseURL: settings.baseURL,
+    headers: Object.keys(headers).length === 0 ? undefined : headers,
+    http: settings.body === undefined ? undefined : { body: { ...settings.body } },
+    limits: settings.limits,
+    providerOptions: settings.providerOptions,
+    queryParams: settings.queryParams === undefined ? undefined : { ...settings.queryParams },
+  }
+}
+
+export const model: ProviderPackage.Definition<Settings>["model"] = (modelID, settings) => {
+  const configured = configure(config(settings))
+  if (settings.transport === undefined || settings.transport === "http") return configured.responses(modelID)
+  if (settings.transport === "websocket") return configured.responsesWebSocket(modelID)
+  throw new Error(`Unsupported OpenAI Responses transport: ${String(settings.transport)}`)
+}
+
+export const chatModel: ProviderPackage.Definition<Settings>["model"] = (modelID, settings) =>
+  configure(config(settings)).chat(modelID)
 export const responses = provider.responses
 export const responsesWebSocket = provider.responsesWebSocket
 export const chat = provider.chat

packages/llm/src/providers/openai/chat.ts

@@ -0,0 +1,2 @@
+export { chatModel as model } from "../openai"
+export type { Settings } from "../openai"

packages/llm/src/providers/openai/responses.ts

@@ -0,0 +1,2 @@
+export { model } from "../openai"
+export type { Settings } from "../openai"