feat(ai): add OpenAI image generation (#37714)
Co-authored-by: Aiden Cline <rekram1-node@users.noreply.github.com>
This commit is contained in:
parent
c50554d907
commit
04f0a771a3
17 changed files with 984 additions and 30 deletions
|
|
@ -1,6 +1,6 @@
|
|||
# @opencode-ai/ai
|
||||
|
||||
Schema-first LLM core for opencode. One typed request, response, event, and tool language; provider quirks live in adapters, not in calling code.
|
||||
Schema-first AI primitives for opencode. Provider quirks live in adapters, not in calling code.
|
||||
|
||||
```ts
|
||||
import { Effect } from "effect"
|
||||
|
|
@ -24,6 +24,45 @@ const program = Effect.gen(function* () {
|
|||
|
||||
Run `LLMClient.stream(request)` instead of `generate` when you want incremental `LLMEvent`s. The event stream is provider-neutral — same shape across OpenAI Chat, OpenAI Responses, Anthropic Messages, Gemini, Bedrock Converse, and any OpenAI-compatible deployment.
|
||||
|
||||
## Image generation
|
||||
|
||||
Use `Image.generate` with an image model for direct asset generation:
|
||||
|
||||
```ts
|
||||
import { Image } from "@opencode-ai/ai"
|
||||
import { OpenAI } from "@opencode-ai/ai/providers"
|
||||
|
||||
const program = Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
model: OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY }).image("gpt-image-2"),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
count: 2,
|
||||
size: { width: 1024, height: 1024 },
|
||||
providerOptions: { openai: { quality: "high", outputFormat: "webp" } },
|
||||
})
|
||||
|
||||
return response.images // GeneratedImage[] with owned bytes or a provider URL
|
||||
})
|
||||
```
|
||||
|
||||
Conversational image generation remains part of the LLM interaction. OpenAI Responses exposes it through its hosted image tool:
|
||||
|
||||
```ts
|
||||
const program = Effect.gen(function* () {
|
||||
const response = yield* LLM.generate(
|
||||
LLM.request({
|
||||
model: OpenAI.configure({ apiKey }).responses("gpt-5"),
|
||||
prompt: "Design a solarpunk rooftop garden, then show me.",
|
||||
tools: [OpenAI.imageGeneration({ quality: "high" })],
|
||||
}),
|
||||
)
|
||||
|
||||
return response.message
|
||||
})
|
||||
```
|
||||
|
||||
The hosted result is represented as a provider-executed tool call and tool result. Its image is a `file` content item with a data URI, so retaining `response.message` preserves the generated image for continuation.
|
||||
|
||||
## Public API
|
||||
|
||||
- **`LLM.request({...})`** — build a provider-neutral `LLMRequest`. Accepts ergonomic inputs (`system: string`, `prompt: string`) that normalize into the canonical Schema classes.
|
||||
|
|
@ -32,6 +71,8 @@ Run `LLMClient.stream(request)` instead of `generate` when you want incremental
|
|||
- **`Model.make(...)` / `ToolCallPart.make(...)` / `ToolResultPart.make(...)` / `ToolDefinition.make(...)`** — model and tool-related constructors from the canonical schema model.
|
||||
- **`LLMClient.prepare(request)`** — compile a request through protocol body construction, validation, and HTTP preparation without sending. Useful for inspection and testing.
|
||||
- **`LLMEvent.is.*`** — typed guards (`is.textDelta`, `is.toolCall`, `is.finish`, …) for filtering streams.
|
||||
- **`Image.generate({...})`** — generate images through a provider-neutral image request and response model.
|
||||
- **`ImageClient`** — Effect service and layer for image execution, parallel to `LLMClient`.
|
||||
|
||||
## Caching
|
||||
|
||||
|
|
@ -182,7 +223,7 @@ Adding a new model or deployment is usually 5-15 lines using `Route.make({ proto
|
|||
|
||||
## Effect
|
||||
|
||||
This package is built on Effect. Public methods return `Effect` or `Stream`; provide `LLMClient.layer` for runtime dispatch and import the provider/protocol modules for the routes you use. The example at `example/tutorial.ts` is a runnable walkthrough.
|
||||
This package is built on Effect. Public methods return `Effect` or `Stream`; provide `LLMClient.layer` for LLM dispatch and `ImageClient.layer` for image dispatch, then import the provider/protocol modules for the routes you use. The example at `example/tutorial.ts` is a runnable walkthrough.
|
||||
|
||||
## See also
|
||||
|
||||
|
|
|
|||
34
packages/ai/src/image-client.ts
Normal file
34
packages/ai/src/image-client.ts
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
import { Context, Effect, Layer } from "effect"
|
||||
import { RequestExecutor } from "./route/executor"
|
||||
import type { ImageRequest, ImageResponse } from "./image"
|
||||
import type { LLMError } from "./schema"
|
||||
|
||||
export type Execute = RequestExecutor.Interface["execute"]
|
||||
|
||||
export interface Interface {
|
||||
readonly generate: (request: ImageRequest) => Effect.Effect<ImageResponse, LLMError>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/ImageClient") {}
|
||||
|
||||
export const generate = (request: ImageRequest): Effect.Effect<ImageResponse, LLMError> =>
|
||||
Effect.gen(function* () {
|
||||
const client = yield* Service
|
||||
return yield* client.generate(request)
|
||||
}) as Effect.Effect<ImageResponse, LLMError>
|
||||
|
||||
export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const executor = yield* RequestExecutor.Service
|
||||
return Service.of({
|
||||
generate: (request) => request.model.route.generate(request, executor.execute),
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
export const ImageClient = {
|
||||
Service,
|
||||
layer,
|
||||
generate,
|
||||
} as const
|
||||
116
packages/ai/src/image.ts
Normal file
116
packages/ai/src/image.ts
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
import { Effect, Schema } from "effect"
|
||||
import { HttpOptions, InvalidRequestReason, LLMError, ModelID, ProviderID, ProviderMetadata, Usage } from "./schema"
|
||||
import { ImageClient, type Execute as ImageExecute } from "./image-client"
|
||||
|
||||
export interface ImageRoute {
|
||||
readonly id: string
|
||||
readonly generate: (request: ImageRequest, execute: ImageExecute) => Effect.Effect<ImageResponse, LLMError>
|
||||
}
|
||||
|
||||
export class ImageModel {
|
||||
readonly id: ModelID
|
||||
readonly provider: ProviderID
|
||||
readonly route: ImageRoute
|
||||
readonly defaults?: ImageModelDefaults
|
||||
|
||||
constructor(input: ImageModel.Input) {
|
||||
this.id = input.id
|
||||
this.provider = input.provider
|
||||
this.route = input.route
|
||||
this.defaults = input.defaults
|
||||
}
|
||||
|
||||
static make(input: ImageModel.MakeInput) {
|
||||
return new ImageModel({
|
||||
id: ModelID.make(input.id),
|
||||
provider: ProviderID.make(input.provider),
|
||||
route: input.route,
|
||||
defaults: input.defaults,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export namespace ImageModel {
|
||||
export interface Input {
|
||||
readonly id: ModelID
|
||||
readonly provider: ProviderID
|
||||
readonly route: ImageRoute
|
||||
readonly defaults?: ImageModelDefaults
|
||||
}
|
||||
|
||||
export interface MakeInput extends Omit<Input, "id" | "provider"> {
|
||||
readonly id: string | ModelID
|
||||
readonly provider: string | ProviderID
|
||||
}
|
||||
}
|
||||
|
||||
export interface ImageModelDefaults {
|
||||
readonly providerOptions?: Record<string, Record<string, unknown>>
|
||||
readonly http?: HttpOptions
|
||||
}
|
||||
|
||||
export const ImageModelSchema = Schema.declare((value): value is ImageModel => value instanceof ImageModel, {
|
||||
expected: "Image.Model",
|
||||
})
|
||||
|
||||
export const ImageSize = Schema.Struct({
|
||||
width: Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)),
|
||||
height: Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)),
|
||||
}).annotate({ identifier: "Image.Size" })
|
||||
export type ImageSize = Schema.Schema.Type<typeof ImageSize>
|
||||
|
||||
export class ImageRequest extends Schema.Class<ImageRequest>("Image.Request")({
|
||||
model: ImageModelSchema,
|
||||
prompt: Schema.String,
|
||||
count: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(1))),
|
||||
size: Schema.optional(ImageSize),
|
||||
aspectRatio: Schema.optional(Schema.String),
|
||||
seed: Schema.optional(Schema.Number),
|
||||
providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Record(Schema.String, Schema.Unknown))),
|
||||
http: Schema.optional(HttpOptions),
|
||||
metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
}) {}
|
||||
|
||||
export type ImageRequestInput = Omit<ConstructorParameters<typeof ImageRequest>[0], "http"> & {
|
||||
readonly http?: HttpOptions.Input
|
||||
}
|
||||
|
||||
export class GeneratedImage extends Schema.Class<GeneratedImage>("Image.Generated")({
|
||||
mediaType: Schema.String,
|
||||
data: Schema.Union([Schema.String, Schema.Uint8Array]),
|
||||
providerMetadata: Schema.optional(ProviderMetadata),
|
||||
}) {}
|
||||
|
||||
export class ImageResponse extends Schema.Class<ImageResponse>("Image.Response")({
|
||||
images: Schema.Array(GeneratedImage),
|
||||
usage: Schema.optional(Usage),
|
||||
providerMetadata: Schema.optional(ProviderMetadata),
|
||||
}) {
|
||||
get image() {
|
||||
return this.images[0]
|
||||
}
|
||||
}
|
||||
|
||||
export const request = (input: ImageRequest | ImageRequestInput) => {
|
||||
if (input instanceof ImageRequest) return input
|
||||
return new ImageRequest({
|
||||
...input,
|
||||
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
|
||||
})
|
||||
}
|
||||
|
||||
export const generate = (input: ImageRequest | ImageRequestInput) =>
|
||||
Effect.try({
|
||||
try: () => request(input),
|
||||
catch: (error) =>
|
||||
new LLMError({
|
||||
module: "Image",
|
||||
method: "generate",
|
||||
reason: new InvalidRequestReason({ message: error instanceof Error ? error.message : String(error) }),
|
||||
}),
|
||||
}).pipe(Effect.flatMap(ImageClient.generate))
|
||||
|
||||
export const Image = {
|
||||
request,
|
||||
generate,
|
||||
} as const
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
export { LLMClient } from "./route/client"
|
||||
export { ImageClient } from "./image-client"
|
||||
export { Auth } from "./route/auth"
|
||||
export { Provider } from "./provider"
|
||||
export { ProviderPackage } from "./provider-package"
|
||||
|
|
@ -10,6 +11,9 @@ export type {
|
|||
Service as LLMClientService,
|
||||
} from "./route/client"
|
||||
export * from "./schema"
|
||||
export { GeneratedImage, ImageModel, ImageRequest, ImageResponse, ImageSize } from "./image"
|
||||
export type { ImageModelDefaults, ImageRequestInput, ImageRoute } from "./image"
|
||||
export { Image } from "./image"
|
||||
export { Tool, ToolFailure, toDefinitions } from "./tool"
|
||||
export { ToolRuntime } from "./tool-runtime"
|
||||
export type { DispatchResult as ToolDispatchResult, ToolSettlement } from "./tool-runtime"
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ export * as AnthropicMessages from "./anthropic-messages"
|
|||
export * as BedrockConverse from "./bedrock-converse"
|
||||
export * as Gemini from "./gemini"
|
||||
export * as OpenAIChat from "./openai-chat"
|
||||
export * as OpenAIImages from "./openai-images"
|
||||
export * as OpenAICompatibleChat from "./openai-compatible-chat"
|
||||
export * as OpenAICompatibleResponses from "./openai-compatible-responses"
|
||||
export * as OpenAIResponses from "./openai-responses"
|
||||
|
|
|
|||
208
packages/ai/src/protocols/openai-images.ts
Normal file
208
packages/ai/src/protocols/openai-images.ts
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
import { Effect, Encoding, Schema } from "effect"
|
||||
import { Headers, HttpClientRequest } from "effect/unstable/http"
|
||||
import {
|
||||
ImageModel,
|
||||
GeneratedImage,
|
||||
ImageResponse,
|
||||
type ImageRequest,
|
||||
type ImageModelDefaults,
|
||||
type ImageRoute,
|
||||
} from "../image"
|
||||
import { Auth, type Definition as AuthDefinition } from "../route/auth"
|
||||
import { InvalidProviderOutputReason, LLMError, Usage, mergeHttpOptions, mergeJsonRecords } from "../schema"
|
||||
import { ProviderShared } from "./shared"
|
||||
import { OpenAIImage } from "./utils/openai-image"
|
||||
|
||||
const ADAPTER = "openai-images"
|
||||
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
|
||||
export const PATH = "/images/generations"
|
||||
|
||||
export interface OpenAIImageOptions {
|
||||
readonly quality?: "auto" | "low" | "medium" | "high"
|
||||
readonly background?: "auto" | "opaque" | "transparent"
|
||||
readonly moderation?: "auto" | "low"
|
||||
readonly outputFormat?: "png" | "jpeg" | "webp"
|
||||
readonly outputCompression?: number
|
||||
}
|
||||
|
||||
const OpenAIImageBody = Schema.Struct({
|
||||
model: Schema.String,
|
||||
prompt: Schema.String,
|
||||
n: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(1))),
|
||||
size: Schema.optional(Schema.String),
|
||||
quality: Schema.optional(Schema.Literals(["auto", "low", "medium", "high"])),
|
||||
background: Schema.optional(Schema.Literals(["auto", "opaque", "transparent"])),
|
||||
moderation: Schema.optional(Schema.Literals(["auto", "low"])),
|
||||
output_format: Schema.optional(Schema.Literals(["png", "jpeg", "webp"])),
|
||||
output_compression: Schema.optional(Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 100 }))),
|
||||
})
|
||||
export type OpenAIImageBody = Schema.Schema.Type<typeof OpenAIImageBody>
|
||||
|
||||
const OpenAIImageResponse = Schema.Struct({
|
||||
data: Schema.Array(
|
||||
Schema.Struct({
|
||||
b64_json: Schema.optional(Schema.String),
|
||||
url: Schema.optional(Schema.String),
|
||||
revised_prompt: Schema.optional(Schema.String),
|
||||
}),
|
||||
),
|
||||
output_format: Schema.optional(Schema.String),
|
||||
usage: Schema.optional(
|
||||
Schema.Struct({
|
||||
input_tokens: Schema.optional(Schema.Number),
|
||||
output_tokens: Schema.optional(Schema.Number),
|
||||
total_tokens: Schema.optional(Schema.Number),
|
||||
input_tokens_details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
output_tokens_details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
}),
|
||||
),
|
||||
})
|
||||
|
||||
export interface ModelInput {
|
||||
readonly id: string
|
||||
readonly auth: AuthDefinition
|
||||
readonly baseURL?: string
|
||||
readonly headers?: Record<string, string>
|
||||
readonly defaults?: ImageModelDefaults
|
||||
}
|
||||
|
||||
const providerOptions = (request: ImageRequest): OpenAIImageOptions => ({
|
||||
...request.model.defaults?.providerOptions?.openai,
|
||||
...request.providerOptions?.openai,
|
||||
})
|
||||
|
||||
const body = (request: ImageRequest): OpenAIImageBody => {
|
||||
const options = providerOptions(request)
|
||||
return {
|
||||
model: request.model.id,
|
||||
prompt: request.prompt,
|
||||
n: request.count,
|
||||
size: request.size === undefined ? undefined : `${request.size.width}x${request.size.height}`,
|
||||
quality: options.quality,
|
||||
background: options.background,
|
||||
moderation: options.moderation,
|
||||
output_format: options.outputFormat,
|
||||
output_compression: options.outputCompression,
|
||||
}
|
||||
}
|
||||
|
||||
const invalidOutput = (message: string) =>
|
||||
new LLMError({
|
||||
module: ADAPTER,
|
||||
method: "generate",
|
||||
reason: new InvalidProviderOutputReason({ message, route: ADAPTER }),
|
||||
})
|
||||
|
||||
const applyQuery = (url: string, query: Record<string, string> | undefined) => {
|
||||
if (!query) return url
|
||||
const next = new URL(url)
|
||||
Object.entries(query).forEach(([key, value]) => next.searchParams.set(key, value))
|
||||
return next.toString()
|
||||
}
|
||||
|
||||
const PROTOCOL_BODY_FIELDS = new Set([
|
||||
"model",
|
||||
"prompt",
|
||||
"n",
|
||||
"size",
|
||||
"quality",
|
||||
"background",
|
||||
"moderation",
|
||||
"output_format",
|
||||
"output_compression",
|
||||
])
|
||||
|
||||
const bodyWithOverlay = Effect.fn("OpenAIImages.bodyWithOverlay")(function* (
|
||||
imageBody: OpenAIImageBody,
|
||||
overlay: Record<string, unknown> | undefined,
|
||||
) {
|
||||
if (!overlay) return imageBody
|
||||
const reserved = Object.keys(overlay).filter((key) => PROTOCOL_BODY_FIELDS.has(key))
|
||||
if (reserved.length > 0)
|
||||
return yield* ProviderShared.invalidRequest(
|
||||
`http.body cannot overlay protocol-owned field(s): ${reserved.join(", ")}`,
|
||||
)
|
||||
return mergeJsonRecords(imageBody, overlay) ?? imageBody
|
||||
})
|
||||
|
||||
export const model = (input: ModelInput) => {
|
||||
const route: ImageRoute = {
|
||||
id: ADAPTER,
|
||||
generate: Effect.fn("OpenAIImages.generate")(function* (request: ImageRequest, execute) {
|
||||
if (request.aspectRatio !== undefined)
|
||||
return yield* ProviderShared.invalidRequest("OpenAI Images does not support the common aspectRatio option")
|
||||
if (request.seed !== undefined)
|
||||
return yield* ProviderShared.invalidRequest("OpenAI Images does not support the common seed option")
|
||||
|
||||
const requestBody = yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(OpenAIImageBody))(body(request))
|
||||
const http = mergeHttpOptions(request.model.defaults?.http, request.http)
|
||||
const overlaidBody = yield* bodyWithOverlay(requestBody, http?.body)
|
||||
const text = ProviderShared.encodeJson(overlaidBody)
|
||||
const url = applyQuery(`${(input.baseURL ?? DEFAULT_BASE_URL).replace(/\/$/, "")}${PATH}`, http?.query)
|
||||
const headers = yield* Auth.toEffect(input.auth)({
|
||||
request,
|
||||
method: "POST",
|
||||
url,
|
||||
body: text,
|
||||
headers: Headers.fromInput({ ...input.headers, ...http?.headers }),
|
||||
})
|
||||
const response = yield* execute(
|
||||
HttpClientRequest.post(url).pipe(
|
||||
HttpClientRequest.setHeaders(headers),
|
||||
HttpClientRequest.bodyText(text, "application/json"),
|
||||
),
|
||||
)
|
||||
const payload = yield* response.json.pipe(
|
||||
Effect.mapError(() => invalidOutput("Failed to read the OpenAI Images response")),
|
||||
)
|
||||
const decoded = yield* Schema.decodeUnknownEffect(OpenAIImageResponse)(payload).pipe(
|
||||
Effect.mapError(() => invalidOutput("OpenAI Images returned an invalid response")),
|
||||
)
|
||||
const format = decoded.output_format ?? providerOptions(request).outputFormat ?? "png"
|
||||
const images = yield* Effect.forEach(decoded.data, (item, index) => {
|
||||
if (item.b64_json)
|
||||
return Effect.fromResult(Encoding.decodeBase64(item.b64_json)).pipe(
|
||||
Effect.mapError(() => invalidOutput(`OpenAI Images result ${index} contains invalid base64 data`)),
|
||||
Effect.map(
|
||||
(data) =>
|
||||
new GeneratedImage({
|
||||
mediaType: `image/${format}`,
|
||||
data,
|
||||
providerMetadata:
|
||||
item.revised_prompt === undefined ? undefined : { openai: { revisedPrompt: item.revised_prompt } },
|
||||
}),
|
||||
),
|
||||
)
|
||||
if (item.url)
|
||||
return Effect.succeed(
|
||||
new GeneratedImage({
|
||||
mediaType: `image/${format}`,
|
||||
data: item.url,
|
||||
providerMetadata:
|
||||
item.revised_prompt === undefined ? undefined : { openai: { revisedPrompt: item.revised_prompt } },
|
||||
}),
|
||||
)
|
||||
return Effect.fail(invalidOutput(`OpenAI Images result ${index} has neither image data nor a URL`))
|
||||
})
|
||||
if (images.length === 0) return yield* invalidOutput("OpenAI Images returned no images")
|
||||
return new ImageResponse({
|
||||
images,
|
||||
usage:
|
||||
decoded.usage === undefined
|
||||
? undefined
|
||||
: new Usage({
|
||||
inputTokens: decoded.usage.input_tokens,
|
||||
outputTokens: decoded.usage.output_tokens,
|
||||
totalTokens: decoded.usage.total_tokens,
|
||||
providerMetadata: { openai: decoded.usage },
|
||||
}),
|
||||
providerMetadata: { openai: { outputFormat: format } },
|
||||
})
|
||||
}),
|
||||
}
|
||||
return ImageModel.make({ id: input.id, provider: "openai", route, defaults: input.defaults })
|
||||
}
|
||||
|
||||
export const OpenAIImages = {
|
||||
model,
|
||||
} as const
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import { Effect, Schema } from "effect"
|
||||
import { Effect, Encoding, Schema } from "effect"
|
||||
import { Route } from "../route/client"
|
||||
import { Auth } from "../route/auth"
|
||||
import { Endpoint } from "../route/endpoint"
|
||||
|
|
@ -25,6 +25,7 @@ import { OpenAIOptions } from "./utils/openai-options"
|
|||
import { Lifecycle } from "./utils/lifecycle"
|
||||
import { ToolSchemaProjection } from "./utils/tool-schema"
|
||||
import { ToolStream } from "./utils/tool-stream"
|
||||
import { OpenAIImage } from "./utils/openai-image"
|
||||
|
||||
const ADAPTER = "openai-responses"
|
||||
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
|
||||
|
|
@ -113,11 +114,24 @@ const OpenAIResponsesTool = Schema.Struct({
|
|||
parameters: JsonObject,
|
||||
strict: Schema.optional(Schema.Boolean),
|
||||
})
|
||||
type OpenAIResponsesTool = Schema.Schema.Type<typeof OpenAIResponsesTool>
|
||||
const OpenAIResponsesImageGenerationTool = Schema.Struct({
|
||||
type: Schema.tag("image_generation"),
|
||||
action: Schema.optional(Schema.Literals(["auto", "generate", "edit"])),
|
||||
background: Schema.optional(Schema.Literals(["auto", "opaque", "transparent"])),
|
||||
input_fidelity: Schema.optional(Schema.Literals(["low", "high"])),
|
||||
output_compression: Schema.optional(Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 100 }))),
|
||||
output_format: Schema.optional(Schema.Literals(["png", "jpeg", "webp"])),
|
||||
partial_images: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(0))),
|
||||
quality: Schema.optional(Schema.Literals(["auto", "low", "medium", "high"])),
|
||||
size: Schema.optional(OpenAIImage.Size),
|
||||
})
|
||||
const OpenAIResponsesTools = Schema.Union([OpenAIResponsesTool, OpenAIResponsesImageGenerationTool])
|
||||
type OpenAIResponsesTool = Schema.Schema.Type<typeof OpenAIResponsesTools>
|
||||
|
||||
const OpenAIResponsesToolChoice = Schema.Union([
|
||||
Schema.Literals(["auto", "none", "required"]),
|
||||
Schema.Struct({ type: Schema.tag("function"), name: Schema.String }),
|
||||
Schema.Struct({ type: Schema.tag("image_generation") }),
|
||||
])
|
||||
|
||||
// Fields shared between the HTTP body and the WebSocket `response.create`
|
||||
|
|
@ -128,7 +142,7 @@ const OpenAIResponsesCoreFields = {
|
|||
model: Schema.String,
|
||||
input: Schema.Array(OpenAIResponsesInputItem),
|
||||
instructions: Schema.optional(Schema.String),
|
||||
tools: optionalArray(OpenAIResponsesTool),
|
||||
tools: optionalArray(OpenAIResponsesTools),
|
||||
tool_choice: Schema.optional(OpenAIResponsesToolChoice),
|
||||
store: Schema.optional(Schema.Boolean),
|
||||
service_tier: Schema.optional(OpenAIOptions.OpenAIServiceTier),
|
||||
|
|
@ -194,6 +208,8 @@ const OpenAIResponsesStreamItem = Schema.Struct({
|
|||
outputs: Schema.optional(Schema.Unknown),
|
||||
server_label: Schema.optional(Schema.String),
|
||||
output: Schema.optional(Schema.Unknown),
|
||||
result: Schema.optional(Schema.String),
|
||||
output_format: Schema.optional(Schema.Literals(["png", "jpeg", "webp"])),
|
||||
error: Schema.optional(Schema.Unknown),
|
||||
encrypted_content: optionalNull(Schema.String),
|
||||
})
|
||||
|
|
@ -258,21 +274,41 @@ const invalid = ProviderShared.invalidRequest
|
|||
// =============================================================================
|
||||
// Request Lowering
|
||||
// =============================================================================
|
||||
const lowerTool = (tool: ToolDefinition, inputSchema: JsonSchema): OpenAIResponsesTool => ({
|
||||
type: "function",
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: ToolSchemaProjection.openAI(inputSchema),
|
||||
// TODO: Read this from OpenAI-specific tool options so direct LLM callers can opt into strict schemas.
|
||||
strict: false,
|
||||
const nativeImageToolInput = (tool: ToolDefinition) => {
|
||||
const native = tool.native?.openai
|
||||
return ProviderShared.isRecord(native) && native.type === "image_generation" ? native : undefined
|
||||
}
|
||||
|
||||
const nativeImageTool = (tool: ToolDefinition) => {
|
||||
const native = nativeImageToolInput(tool)
|
||||
return Schema.is(OpenAIResponsesImageGenerationTool)(native) ? native : undefined
|
||||
}
|
||||
|
||||
const lowerTool = Effect.fn("OpenAIResponses.lowerTool")(function* (tool: ToolDefinition, inputSchema: JsonSchema) {
|
||||
const native = nativeImageToolInput(tool)
|
||||
if (native !== undefined) {
|
||||
if (Schema.is(OpenAIResponsesImageGenerationTool)(native)) return native
|
||||
return yield* invalid("OpenAI Responses image generation tool options are invalid")
|
||||
}
|
||||
return {
|
||||
type: "function" as const,
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: ToolSchemaProjection.openAI(inputSchema),
|
||||
// TODO: Read this from OpenAI-specific tool options so direct LLM callers can opt into strict schemas.
|
||||
strict: false,
|
||||
}
|
||||
})
|
||||
|
||||
const lowerToolChoice = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
|
||||
const lowerToolChoice = (toolChoice: NonNullable<LLMRequest["toolChoice"]>, tools: ReadonlyArray<ToolDefinition>) =>
|
||||
ProviderShared.matchToolChoice("OpenAI Responses", toolChoice, {
|
||||
auto: () => "auto" as const,
|
||||
none: () => "none" as const,
|
||||
required: () => "required" as const,
|
||||
tool: (name) => ({ type: "function" as const, name }),
|
||||
tool: (name) =>
|
||||
tools.some((tool) => tool.name === name && nativeImageTool(tool) !== undefined)
|
||||
? ({ type: "image_generation" } as const)
|
||||
: { type: "function" as const, name },
|
||||
})
|
||||
|
||||
const lowerToolCall = (part: ToolCallPart): OpenAIResponsesInputItem => ({
|
||||
|
|
@ -420,6 +456,13 @@ const lowerMessages = Effect.fn("OpenAIResponses.lowerMessages")(function* (requ
|
|||
const itemID = hostedToolItemID(part)
|
||||
if (store !== false && itemID && !hostedToolReferences.has(itemID))
|
||||
input.push({ type: "item_reference", id: itemID })
|
||||
if (store === false && part.name === "image_generation" && part.result.type === "content") {
|
||||
const content: ReadonlyArray<ToolContent> = part.result.value
|
||||
input.push({
|
||||
role: "user",
|
||||
content: yield* Effect.forEach(content, lowerToolResultContentItem),
|
||||
})
|
||||
}
|
||||
if (itemID) hostedToolReferences.add(itemID)
|
||||
continue
|
||||
}
|
||||
|
|
@ -485,10 +528,10 @@ const fromRequest = Effect.fn("OpenAIResponses.fromRequest")(function* (request:
|
|||
tools:
|
||||
request.tools.length === 0
|
||||
? undefined
|
||||
: request.tools.map((tool) =>
|
||||
: yield* Effect.forEach(request.tools, (tool) =>
|
||||
lowerTool(tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, toolSchemaCompatibility)),
|
||||
),
|
||||
tool_choice: request.toolChoice ? yield* lowerToolChoice(request.toolChoice) : undefined,
|
||||
tool_choice: request.toolChoice ? yield* lowerToolChoice(request.toolChoice, request.tools) : undefined,
|
||||
stream: true as const,
|
||||
max_output_tokens: generation?.maxTokens,
|
||||
temperature: generation?.temperature,
|
||||
|
|
@ -574,14 +617,29 @@ const isReasoningItem = (
|
|||
|
||||
// Round-trip the full item as the structured result so consumers can extract
|
||||
// outputs / sources / status without re-decoding.
|
||||
const hostedToolResult = (item: OpenAIResponsesStreamItem) => {
|
||||
const hostedToolResult = Effect.fn("OpenAIResponses.hostedToolResult")(function* (item: OpenAIResponsesStreamItem) {
|
||||
const isError = typeof item.error !== "undefined" && item.error !== null
|
||||
if (item.type === "image_generation_call" && item.result) {
|
||||
yield* Effect.fromResult(Encoding.decodeBase64(item.result)).pipe(
|
||||
Effect.mapError(() => ProviderShared.eventError(ADAPTER, "OpenAI Responses returned invalid image base64")),
|
||||
)
|
||||
return {
|
||||
type: "content" as const,
|
||||
value: [
|
||||
{
|
||||
type: "file" as const,
|
||||
uri: `data:image/${item.output_format ?? "png"};base64,${item.result}`,
|
||||
mime: `image/${item.output_format ?? "png"}`,
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
return isError ? { type: "error" as const, value: item.error } : { type: "json" as const, value: item }
|
||||
}
|
||||
})
|
||||
|
||||
const hostedToolEvents = (
|
||||
const hostedToolEvents = Effect.fn("OpenAIResponses.hostedToolEvents")(function* (
|
||||
item: OpenAIResponsesStreamItem & { type: HostedToolType; id: string },
|
||||
): ReadonlyArray<LLMEvent> => {
|
||||
) {
|
||||
const tool = HOSTED_TOOLS[item.type]
|
||||
const providerMetadata = openaiMetadata({ itemId: item.id })
|
||||
return [
|
||||
|
|
@ -595,12 +653,12 @@ const hostedToolEvents = (
|
|||
LLMEvent.toolResult({
|
||||
id: item.id,
|
||||
name: tool.name,
|
||||
result: hostedToolResult(item),
|
||||
result: yield* hostedToolResult(item),
|
||||
providerExecuted: true,
|
||||
providerMetadata,
|
||||
}),
|
||||
]
|
||||
}
|
||||
})
|
||||
|
||||
type StepResult = readonly [ParserState, ReadonlyArray<LLMEvent>]
|
||||
|
||||
|
|
@ -847,7 +905,7 @@ const onOutputItemDone = Effect.fn("OpenAIResponses.onOutputItemDone")(function*
|
|||
if (isHostedToolItem(item)) {
|
||||
const events: LLMEvent[] = []
|
||||
const lifecycle = Lifecycle.stepStart(state.lifecycle, events)
|
||||
events.push(...hostedToolEvents(item))
|
||||
events.push(...(yield* hostedToolEvents(item)))
|
||||
return [{ ...state, lifecycle }, events] satisfies StepResult
|
||||
}
|
||||
|
||||
|
|
|
|||
20
packages/ai/src/protocols/utils/openai-image.ts
Normal file
20
packages/ai/src/protocols/utils/openai-image.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
import { Schema } from "effect"
|
||||
|
||||
const dimensions = (value: string) => {
|
||||
const match = /^(\d+)x(\d+)$/.exec(value)
|
||||
if (!match) return undefined
|
||||
return { width: Number(match[1]), height: Number(match[2]) }
|
||||
}
|
||||
|
||||
export const Size = Schema.String.check(
|
||||
Schema.makeFilter((value) => {
|
||||
if (value === "auto") return undefined
|
||||
const parsed = dimensions(value)
|
||||
if (!parsed) return "image size must be `auto` or `{width}x{height}`"
|
||||
return parsed.width > 0 && parsed.height > 0 ? undefined : "image dimensions must be positive integers"
|
||||
}),
|
||||
)
|
||||
|
||||
export const OpenAIImage = {
|
||||
Size,
|
||||
} as const
|
||||
|
|
@ -1,12 +1,14 @@
|
|||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options"
|
||||
import type { Route, RouteDefaultsInput } from "../route/client"
|
||||
import type { ProviderPackage } from "../provider-package"
|
||||
import { ProviderID, type ModelID } from "../schema"
|
||||
import { HttpOptions, ProviderID, ToolDefinition, mergeHttpOptions, type ModelID } from "../schema"
|
||||
import * as OpenAIChat from "../protocols/openai-chat"
|
||||
import * as OpenAIResponses from "../protocols/openai-responses"
|
||||
import { withOpenAIOptions, type OpenAIProviderOptionsInput } from "./openai-options"
|
||||
import { OpenAIImages, type OpenAIImageOptions } from "../protocols/openai-images"
|
||||
|
||||
export type { OpenAIOptionsInput, OpenAIResponseIncludable } from "./openai-options"
|
||||
export type { OpenAIImageOptions } from "../protocols/openai-images"
|
||||
|
||||
export const id = ProviderID.make("openai")
|
||||
|
||||
|
|
@ -20,8 +22,44 @@ export type Config = RouteDefaultsInput &
|
|||
readonly baseURL?: string
|
||||
readonly queryParams?: Record<string, string>
|
||||
readonly providerOptions?: OpenAIProviderOptionsInput
|
||||
readonly image?: ImageConfig
|
||||
}
|
||||
|
||||
export interface ImageConfig {
|
||||
readonly providerOptions?: OpenAIImageOptions
|
||||
}
|
||||
|
||||
export interface ImageGenerationOptions {
|
||||
readonly action?: "auto" | "generate" | "edit"
|
||||
readonly background?: "auto" | "opaque" | "transparent"
|
||||
readonly inputFidelity?: "low" | "high"
|
||||
readonly outputCompression?: number
|
||||
readonly outputFormat?: "png" | "jpeg" | "webp"
|
||||
readonly partialImages?: number
|
||||
readonly quality?: "auto" | "low" | "medium" | "high"
|
||||
readonly size?: string
|
||||
}
|
||||
|
||||
export const imageGeneration = (options: ImageGenerationOptions = {}) =>
|
||||
ToolDefinition.make({
|
||||
name: "image_generation",
|
||||
description: "Generate or edit an image using OpenAI's hosted image generation tool.",
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
native: {
|
||||
openai: {
|
||||
type: "image_generation",
|
||||
action: options.action,
|
||||
background: options.background,
|
||||
input_fidelity: options.inputFidelity,
|
||||
output_compression: options.outputCompression,
|
||||
output_format: options.outputFormat,
|
||||
partial_images: options.partialImages,
|
||||
quality: options.quality,
|
||||
size: options.size,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
export interface Settings extends ProviderPackage.Settings {
|
||||
readonly apiKey?: string
|
||||
readonly baseURL?: string
|
||||
|
|
@ -35,7 +73,7 @@ export interface Settings extends ProviderPackage.Settings {
|
|||
const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "OPENAI_API_KEY")
|
||||
|
||||
const defaults = (input: Config) => {
|
||||
const { apiKey: _, auth: _auth, baseURL: _baseURL, queryParams: _queryParams, ...rest } = input
|
||||
const { apiKey: _, auth: _auth, baseURL: _baseURL, queryParams: _queryParams, image: _image, ...rest } = input
|
||||
return rest
|
||||
}
|
||||
|
||||
|
|
@ -55,6 +93,21 @@ export const configure = (input: Config = {}) => {
|
|||
const responsesWebSocket = (id: string | ModelID) =>
|
||||
responsesWebSocketRoute.with(withOpenAIOptions(id, modelDefaults, { textVerbosity: true })).model({ id })
|
||||
const chat = (id: string | ModelID) => chatRoute.with(withOpenAIOptions(id, modelDefaults)).model({ id })
|
||||
const image = (modelID: string | ModelID) =>
|
||||
OpenAIImages.model({
|
||||
id: modelID,
|
||||
auth: auth(input),
|
||||
baseURL: input.baseURL,
|
||||
headers: input.headers,
|
||||
defaults: {
|
||||
providerOptions:
|
||||
input.image?.providerOptions === undefined ? undefined : { openai: { ...input.image.providerOptions } },
|
||||
http: mergeHttpOptions(
|
||||
input.http === undefined ? undefined : HttpOptions.make(input.http),
|
||||
input.queryParams === undefined ? undefined : new HttpOptions({ query: input.queryParams }),
|
||||
),
|
||||
},
|
||||
})
|
||||
|
||||
return {
|
||||
id,
|
||||
|
|
@ -62,6 +115,7 @@ export const configure = (input: Config = {}) => {
|
|||
responses,
|
||||
responsesWebSocket,
|
||||
chat,
|
||||
image,
|
||||
configure,
|
||||
}
|
||||
}
|
||||
|
|
@ -97,3 +151,4 @@ export const chatModel: ProviderPackage.Definition<Settings>["model"] = (modelID
|
|||
export const responses = provider.responses
|
||||
export const responsesWebSocket = provider.responsesWebSocket
|
||||
export const chat = provider.chat
|
||||
export const image = provider.image
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { Config, Effect, Redacted } from "effect"
|
||||
import { Headers } from "effect/unstable/http"
|
||||
import { AuthenticationReason, InvalidRequestReason, LLMError, type LLMRequest } from "../schema"
|
||||
import { AuthenticationReason, InvalidRequestReason, LLMError, type HttpOptions } from "../schema"
|
||||
|
||||
export class MissingCredentialError extends Error {
|
||||
readonly _tag = "MissingCredentialError"
|
||||
|
|
@ -15,7 +15,7 @@ export type AuthError = CredentialError | LLMError
|
|||
type Secret = string | Redacted.Redacted | Config.Config<string | Redacted.Redacted>
|
||||
|
||||
export interface AuthInput {
|
||||
readonly request: LLMRequest
|
||||
readonly request: { readonly http?: HttpOptions }
|
||||
readonly method: "POST" | "GET"
|
||||
readonly url: string
|
||||
readonly body: string
|
||||
|
|
|
|||
32
packages/ai/test/fixtures/recordings/openai-images/generates-an-image.json
vendored
Normal file
32
packages/ai/test/fixtures/recordings/openai-images/generates-an-image.json
vendored
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
95
packages/ai/test/image.test.ts
Normal file
95
packages/ai/test/image.test.ts
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
import { describe, expect } from "bun:test"
|
||||
import { Effect, Layer } from "effect"
|
||||
import { HttpClientRequest } from "effect/unstable/http"
|
||||
import { Image, ImageClient } from "../src"
|
||||
import { OpenAI } from "../src/providers"
|
||||
import { it } from "./lib/effect"
|
||||
import { dynamicResponse } from "./lib/http"
|
||||
|
||||
describe("Image", () => {
|
||||
it.effect("generates images through the OpenAI Images API", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
model: OpenAI.configure({
|
||||
apiKey: "test",
|
||||
baseURL: "https://api.openai.test/v1",
|
||||
queryParams: { "api-version": "v1" },
|
||||
http: { body: { deployment: "test" }, headers: { "x-default": "yes" } },
|
||||
}).image("gpt-image-2"),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
count: 2,
|
||||
size: { width: 1024, height: 1024 },
|
||||
providerOptions: {
|
||||
openai: { quality: "high", outputFormat: "webp" },
|
||||
},
|
||||
http: {
|
||||
body: { request_metadata: "value" },
|
||||
headers: { "x-request": "yes" },
|
||||
query: { trace: "1" },
|
||||
},
|
||||
})
|
||||
|
||||
expect(response.images).toHaveLength(2)
|
||||
expect(response.image?.mediaType).toBe("image/webp")
|
||||
expect(response.image?.data).toEqual(Uint8Array.from([1, 2, 3]))
|
||||
expect(response.image?.providerMetadata).toEqual({ openai: { revisedPrompt: "A precise robot" } })
|
||||
expect(response.usage?.totalTokens).toBe(12)
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
ImageClient.layer.pipe(
|
||||
Layer.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(request.url).toBe("https://api.openai.test/v1/images/generations?api-version=v1&trace=1")
|
||||
expect(request.headers.get("authorization")).toBe("Bearer test")
|
||||
expect(request.headers.get("x-default")).toBe("yes")
|
||||
expect(request.headers.get("x-request")).toBe("yes")
|
||||
expect(JSON.parse(input.text)).toEqual({
|
||||
model: "gpt-image-2",
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
n: 2,
|
||||
size: "1024x1024",
|
||||
quality: "high",
|
||||
output_format: "webp",
|
||||
deployment: "test",
|
||||
request_metadata: "value",
|
||||
})
|
||||
return input.respond(
|
||||
JSON.stringify({
|
||||
data: [{ b64_json: "AQID", revised_prompt: "A precise robot" }, { b64_json: "BAUG" }],
|
||||
output_format: "webp",
|
||||
usage: { input_tokens: 4, output_tokens: 8, total_tokens: 12 },
|
||||
}),
|
||||
{ headers: { "content-type": "application/json" } },
|
||||
)
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("rejects invalid common and OpenAI image options locally", () =>
|
||||
Image.generate({
|
||||
model: OpenAI.configure({ apiKey: "test", baseURL: "https://api.openai.test/v1" }).image("gpt-image-2"),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
count: -1,
|
||||
size: { width: -1, height: 0.5 },
|
||||
providerOptions: { openai: { outputCompression: 101 } },
|
||||
}).pipe(
|
||||
Effect.flip,
|
||||
Effect.tap((error) =>
|
||||
Effect.sync(() => {
|
||||
expect(error.reason._tag).toBe("InvalidRequest")
|
||||
}),
|
||||
),
|
||||
Effect.provide(
|
||||
ImageClient.layer.pipe(
|
||||
Layer.provide(dynamicResponse(() => Effect.die("invalid request should not reach the provider"))),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
})
|
||||
40
packages/ai/test/provider/openai-images.recorded.test.ts
Normal file
40
packages/ai/test/provider/openai-images.recorded.test.ts
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { Image } from "../../src"
|
||||
import { OpenAI } from "../../src/providers"
|
||||
import { recordedTests } from "../recorded-test"
|
||||
|
||||
const model = OpenAI.configure({
|
||||
apiKey: process.env.OPENAI_API_KEY ?? "fixture",
|
||||
image: {
|
||||
providerOptions: {
|
||||
quality: "low",
|
||||
outputFormat: "jpeg",
|
||||
outputCompression: 10,
|
||||
},
|
||||
},
|
||||
}).image("gpt-image-1-mini")
|
||||
|
||||
const recorded = recordedTests({
|
||||
prefix: "openai-images",
|
||||
provider: "openai",
|
||||
protocol: "openai-images",
|
||||
requires: ["OPENAI_API_KEY"],
|
||||
})
|
||||
|
||||
describe("OpenAI Images recorded", () => {
|
||||
recorded.effect("generates an image", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
model,
|
||||
prompt: "A simple flat black circle centered on a plain white background.",
|
||||
size: { width: 1024, height: 1024 },
|
||||
})
|
||||
|
||||
expect(response.images).toHaveLength(1)
|
||||
expect(response.image?.mediaType).toBe("image/jpeg")
|
||||
expect(response.image?.data).toBeInstanceOf(Uint8Array)
|
||||
expect(response.image?.data.length).toBeGreaterThan(0)
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { LLM, LLMEvent, Message } from "../../src"
|
||||
import { OpenAI } from "../../src/providers"
|
||||
import { recordedTests } from "../recorded-test"
|
||||
|
||||
const openai = OpenAI.configure({
|
||||
apiKey: process.env.OPENAI_API_KEY ?? "fixture",
|
||||
})
|
||||
|
||||
const recorded = recordedTests({
|
||||
prefix: "openai-responses-images",
|
||||
provider: "openai",
|
||||
protocol: "openai-responses",
|
||||
requires: ["OPENAI_API_KEY"],
|
||||
})
|
||||
|
||||
describe("OpenAI Responses image generation recorded", () => {
|
||||
recorded.effect("generates and edits an image with the hosted tool", () =>
|
||||
Effect.gen(function* () {
|
||||
const initial = Message.user("Generate a simple flat black triangle centered on a plain white background.")
|
||||
const tools = [
|
||||
OpenAI.imageGeneration({
|
||||
action: "auto",
|
||||
quality: "low",
|
||||
size: "1024x1024",
|
||||
outputFormat: "jpeg",
|
||||
outputCompression: 10,
|
||||
partialImages: 0,
|
||||
}),
|
||||
]
|
||||
const response = yield* LLM.generate(
|
||||
LLM.request({
|
||||
model: openai.responses("gpt-5-mini"),
|
||||
messages: [initial],
|
||||
tools,
|
||||
toolChoice: "image_generation",
|
||||
}),
|
||||
)
|
||||
|
||||
const result = response.events.find(LLMEvent.is.toolResult)
|
||||
expect(result).toBeDefined()
|
||||
expect(result?.providerExecuted).toBe(true)
|
||||
expect(result?.result.type).toBe("content")
|
||||
if (result?.result.type !== "content") return
|
||||
expect(result.result.value).toHaveLength(1)
|
||||
expect(result.result.value[0]?.type).toBe("file")
|
||||
if (result.result.value[0]?.type !== "file") return
|
||||
expect(result.result.value[0].mime).toBe("image/jpeg")
|
||||
expect(result.result.value[0].uri.startsWith("data:image/jpeg;base64,")).toBe(true)
|
||||
|
||||
const edited = yield* LLM.generate(
|
||||
LLM.request({
|
||||
model: openai.responses("gpt-5-mini"),
|
||||
messages: [initial, response.message, Message.user("Now make the triangle blue.")],
|
||||
tools,
|
||||
toolChoice: "image_generation",
|
||||
}),
|
||||
)
|
||||
const editedResult = edited.events.find(LLMEvent.is.toolResult)
|
||||
expect(editedResult?.result.type).toBe("content")
|
||||
if (editedResult?.result.type !== "content") return
|
||||
expect(editedResult.result.value[0]?.type).toBe("file")
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
import { describe, expect } from "bun:test"
|
||||
import { ConfigProvider, Effect, Layer, Stream } from "effect"
|
||||
import { Headers, HttpClientRequest } from "effect/unstable/http"
|
||||
import { LLM, LLMError, LLMEvent, Message, Model, ToolCallPart, Usage } from "../../src"
|
||||
import { LLM, LLMError, LLMEvent, Message, Model, ToolCallPart, ToolResultPart, Usage } from "../../src"
|
||||
import { Auth, LLMClient, RequestExecutor, WebSocketExecutor } from "../../src/route"
|
||||
import * as Azure from "../../src/providers/azure"
|
||||
import * as OpenAI from "../../src/providers/openai"
|
||||
|
|
@ -58,6 +58,39 @@ describe("OpenAI Responses route", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("lowers the hosted OpenAI image generation tool", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
|
||||
LLM.request({
|
||||
model,
|
||||
prompt: "Show me a rooftop garden.",
|
||||
tools: [OpenAI.imageGeneration({ action: "generate", quality: "high", size: "1024x1024" })],
|
||||
toolChoice: "image_generation",
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body.tools).toEqual([
|
||||
{ type: "image_generation", action: "generate", quality: "high", size: "1024x1024" },
|
||||
])
|
||||
expect(prepared.body.tool_choice).toEqual({ type: "image_generation" })
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects invalid hosted image generation options locally", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
model,
|
||||
prompt: "Show me a rooftop garden.",
|
||||
tools: [OpenAI.imageGeneration({ outputCompression: -1, partialImages: 4, size: "bogus" })],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
|
||||
expect(error.reason._tag).toBe("InvalidRequest")
|
||||
expect(error.message).toContain("image generation tool options are invalid")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("lowers semantic service tier options", () =>
|
||||
Effect.gen(function* () {
|
||||
const input = LLM.updateRequest(request, { providerOptions: { openai: { serviceTier: "priority" } } })
|
||||
|
|
@ -1103,6 +1136,48 @@ describe("OpenAI Responses route", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("continues stateless hosted image generation with the generated image", () =>
|
||||
Effect.gen(function* () {
|
||||
const imageTool = OpenAI.imageGeneration({ action: "edit" })
|
||||
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
|
||||
LLM.request({
|
||||
model,
|
||||
messages: [
|
||||
Message.user("Generate a black triangle."),
|
||||
Message.assistant([
|
||||
ToolCallPart.make({
|
||||
id: "ig_1",
|
||||
name: "image_generation",
|
||||
input: {},
|
||||
providerExecuted: true,
|
||||
providerMetadata: { openai: { itemId: "ig_1" } },
|
||||
}),
|
||||
ToolResultPart.make({
|
||||
id: "ig_1",
|
||||
name: "image_generation",
|
||||
result: {
|
||||
type: "content",
|
||||
value: [{ type: "file", uri: "data:image/png;base64,AQID", mime: "image/png" }],
|
||||
},
|
||||
providerExecuted: true,
|
||||
providerMetadata: { openai: { itemId: "ig_1" } },
|
||||
}),
|
||||
]),
|
||||
Message.user("Make it blue."),
|
||||
],
|
||||
tools: [imageTool],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body.store).toBe(false)
|
||||
expect(prepared.body.input).toEqual([
|
||||
{ role: "user", content: [{ type: "input_text", text: "Generate a black triangle." }] },
|
||||
{ role: "user", content: [{ type: "input_image", image_url: "data:image/png;base64,AQID" }] },
|
||||
{ role: "user", content: [{ type: "input_text", text: "Make it blue." }] },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("joins streamed summary blocks into one continuation reasoning item", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
|
||||
|
|
@ -1361,6 +1436,59 @@ describe("OpenAI Responses route", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("decodes image generation output as image content", () =>
|
||||
Effect.gen(function* () {
|
||||
const item = {
|
||||
type: "image_generation_call",
|
||||
id: "ig_1",
|
||||
status: "completed",
|
||||
result: "AQID",
|
||||
}
|
||||
const response = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse(
|
||||
sseEvents(
|
||||
{ type: "response.output_item.done", item },
|
||||
{ type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } },
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(response.events.find(LLMEvent.is.toolResult)).toMatchObject({
|
||||
id: "ig_1",
|
||||
name: "image_generation",
|
||||
providerExecuted: true,
|
||||
result: {
|
||||
type: "content",
|
||||
value: [{ type: "file", uri: "data:image/png;base64,AQID", mime: "image/png" }],
|
||||
},
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects malformed image generation base64", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse(
|
||||
sseEvents(
|
||||
{
|
||||
type: "response.output_item.done",
|
||||
item: { type: "image_generation_call", id: "ig_bad", status: "completed", result: "%%%" },
|
||||
},
|
||||
{ type: "response.completed", response: {} },
|
||||
),
|
||||
),
|
||||
),
|
||||
Effect.flip,
|
||||
)
|
||||
|
||||
expect(error.reason._tag).toBe("InvalidProviderOutput")
|
||||
expect(error.message).toContain("invalid image base64")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("decodes code_interpreter_call as provider-executed events with code input", () =>
|
||||
Effect.gen(function* () {
|
||||
const item = {
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ import { Layer } from "effect"
|
|||
import * as path from "node:path"
|
||||
import { fileURLToPath } from "node:url"
|
||||
import { LLMClient, RequestExecutor, WebSocketExecutor } from "../src/route"
|
||||
import { ImageClient } from "../src/image-client"
|
||||
import type { Service as ImageClientService } from "../src/image-client"
|
||||
import type { Service as LLMClientService } from "../src/route/client"
|
||||
import type { Service as RequestExecutorService } from "../src/route/executor"
|
||||
import type { Service as WebSocketExecutorService } from "../src/route/transport/websocket"
|
||||
|
|
@ -15,7 +17,7 @@ import {
|
|||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
const FIXTURES_DIR = path.resolve(__dirname, "fixtures", "recordings")
|
||||
|
||||
type RecordedEnv = RequestExecutorService | WebSocketExecutorService | LLMClientService
|
||||
type RecordedEnv = RequestExecutorService | WebSocketExecutorService | LLMClientService | ImageClientService
|
||||
|
||||
type RecordedTestsOptions = RecordedGroupOptions & {
|
||||
readonly options?: HttpRecorder.RecorderOptions
|
||||
|
|
@ -81,6 +83,10 @@ export const recordedTests = (options: RecordedTestsOptions) =>
|
|||
),
|
||||
)
|
||||
const deps = Layer.mergeAll(requestExecutor, WebSocketExecutor.layer)
|
||||
return Layer.mergeAll(deps, LLMClient.layer.pipe(Layer.provide(deps)))
|
||||
return Layer.mergeAll(
|
||||
deps,
|
||||
LLMClient.layer.pipe(Layer.provide(deps)),
|
||||
ImageClient.layer.pipe(Layer.provide(deps)),
|
||||
)
|
||||
},
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue