feat(ai): add Google image generation (#37781)
Co-authored-by: Aiden Cline <rekram1-node@users.noreply.github.com>
This commit is contained in:
opencode-agent[bot]
GitHub
parent
642950a763
commit
8cb9361149
7 changed files with 672 additions and 5 deletions
|
|
@ -81,6 +81,35 @@ yield *
|
|||
})
|
||||
```
|
||||
|
||||
Google's current Gemini image models use the same direct API:
|
||||
|
||||
```ts
|
||||
import { Google } from "@opencode-ai/ai/providers"
|
||||
|
||||
const googleProgram = Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
model: Google.configure({ apiKey }).image("any-model-id"),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
options: {
|
||||
aspectRatio: "16:9",
|
||||
imageSize: "2K",
|
||||
seed: 42,
|
||||
thinkingLevel: "HIGH",
|
||||
includeThoughts: true,
|
||||
futureOption: true,
|
||||
},
|
||||
http,
|
||||
})
|
||||
|
||||
return response.images
|
||||
})
|
||||
```
|
||||
|
||||
Google image options are request-scoped and inferred from the selected model. Known fields autocomplete while
|
||||
future string values and arbitrary native Gemini `generationConfig` fields remain available. Native fields override
|
||||
their mapped aliases, and `http.body` is the final deep overlay. The selected model ID is sent to Gemini
|
||||
`generateContent` without a local allowlist.
|
||||
|
||||
Conversational image generation remains part of the LLM interaction. OpenAI Responses exposes it through its hosted image tool:
|
||||
|
||||
```ts
|
||||
|
|
|
|||
283
packages/ai/src/protocols/google-images.ts
Normal file
283
packages/ai/src/protocols/google-images.ts
Normal file
|
|
@ -0,0 +1,283 @@
|
|||
import { Effect, Encoding, Schema } from "effect"
|
||||
import { Headers, HttpClientRequest } from "effect/unstable/http"
|
||||
import { GeneratedImage, ImageModel, ImageResponse, type ImageRequestFor, type ImageRoute } from "../image"
|
||||
import { Auth, type Definition as AuthDefinition } from "../route/auth"
|
||||
import {
|
||||
InvalidProviderOutputReason,
|
||||
LLMError,
|
||||
Usage,
|
||||
mergeHttpOptions,
|
||||
mergeJsonRecords,
|
||||
type HttpOptions,
|
||||
type ProviderMetadata,
|
||||
} from "../schema"
|
||||
import { ProviderShared } from "./shared"
|
||||
|
||||
const ADAPTER = "google-images"
|
||||
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
|
||||
|
||||
export type GoogleImageString<Known extends string> = Known | (string & {})
|
||||
|
||||
export type GoogleImageOptions = {
|
||||
readonly aspectRatio?: GoogleImageString<
|
||||
"1:1" | "2:3" | "3:2" | "3:4" | "4:3" | "4:5" | "5:4" | "9:16" | "16:9" | "21:9"
|
||||
>
|
||||
readonly imageSize?: GoogleImageString<"1K" | "2K" | "4K">
|
||||
readonly seed?: number
|
||||
readonly thinkingLevel?: GoogleImageString<"MINIMAL" | "LOW" | "MEDIUM" | "HIGH">
|
||||
readonly includeThoughts?: boolean
|
||||
} & Record<string, unknown>
|
||||
|
||||
export type GoogleImageBody = Record<string, unknown> & {
|
||||
readonly contents: ReadonlyArray<{
|
||||
readonly role: "user"
|
||||
readonly parts: ReadonlyArray<{ readonly text: string }>
|
||||
}>
|
||||
readonly generationConfig: Record<string, unknown>
|
||||
}
|
||||
|
||||
const GoogleUsage = Schema.StructWithRest(
|
||||
Schema.Struct({
|
||||
cachedContentTokenCount: Schema.optional(Schema.Number),
|
||||
thoughtsTokenCount: Schema.optional(Schema.Number),
|
||||
promptTokenCount: Schema.optional(Schema.Number),
|
||||
candidatesTokenCount: Schema.optional(Schema.Number),
|
||||
totalTokenCount: Schema.optional(Schema.Number),
|
||||
promptTokensDetails: Schema.optional(Schema.Unknown),
|
||||
candidatesTokensDetails: Schema.optional(Schema.Unknown),
|
||||
}),
|
||||
[Schema.Record(Schema.String, Schema.Unknown)],
|
||||
)
|
||||
|
||||
const GoogleImageResponse = Schema.Struct({
|
||||
candidates: Schema.optional(
|
||||
Schema.Array(
|
||||
Schema.Struct({
|
||||
index: Schema.optional(Schema.Number),
|
||||
content: Schema.optional(
|
||||
Schema.Struct({
|
||||
parts: Schema.Array(
|
||||
Schema.Struct({
|
||||
text: Schema.optional(Schema.String),
|
||||
thought: Schema.optional(Schema.Boolean),
|
||||
thoughtSignature: Schema.optional(Schema.String),
|
||||
inlineData: Schema.optional(
|
||||
Schema.Struct({
|
||||
mimeType: Schema.String,
|
||||
data: Schema.String,
|
||||
}),
|
||||
),
|
||||
}),
|
||||
),
|
||||
}),
|
||||
),
|
||||
finishReason: Schema.optional(Schema.String),
|
||||
finishMessage: Schema.optional(Schema.String),
|
||||
safetyRatings: Schema.optional(Schema.Unknown),
|
||||
citationMetadata: Schema.optional(Schema.Unknown),
|
||||
groundingMetadata: Schema.optional(Schema.Unknown),
|
||||
}),
|
||||
),
|
||||
),
|
||||
usageMetadata: Schema.optional(GoogleUsage),
|
||||
modelVersion: Schema.optional(Schema.String),
|
||||
responseId: Schema.optional(Schema.String),
|
||||
promptFeedback: Schema.optional(Schema.Unknown),
|
||||
})
|
||||
|
||||
export interface ModelInput {
|
||||
readonly id: string
|
||||
readonly auth: AuthDefinition
|
||||
readonly baseURL?: string
|
||||
readonly headers?: Record<string, string>
|
||||
readonly http?: HttpOptions
|
||||
}
|
||||
|
||||
const nativeOptions = (options: GoogleImageOptions | undefined) => {
|
||||
const { aspectRatio, imageSize, seed, thinkingLevel, includeThoughts, ...native } = options ?? {}
|
||||
const image = {
|
||||
aspectRatio,
|
||||
imageSize,
|
||||
}
|
||||
const thinkingConfig = {
|
||||
thinkingLevel,
|
||||
includeThoughts,
|
||||
}
|
||||
return (
|
||||
mergeJsonRecords(
|
||||
{
|
||||
responseModalities: ["IMAGE"],
|
||||
imageConfig: Object.values(image).some((value) => value !== undefined) ? image : undefined,
|
||||
seed,
|
||||
thinkingConfig: Object.values(thinkingConfig).some((value) => value !== undefined) ? thinkingConfig : undefined,
|
||||
},
|
||||
native,
|
||||
) ?? { responseModalities: ["IMAGE"] }
|
||||
)
|
||||
}
|
||||
|
||||
const body = (request: ImageRequestFor<GoogleImageOptions>, overlay: Record<string, unknown> | undefined) =>
|
||||
mergeJsonRecords(
|
||||
{
|
||||
contents: [{ role: "user", parts: [{ text: request.prompt }] }],
|
||||
generationConfig: nativeOptions(request.options),
|
||||
},
|
||||
overlay,
|
||||
) as GoogleImageBody
|
||||
|
||||
const invalidOutput = (message: string, providerMetadata?: ProviderMetadata) =>
|
||||
new LLMError({
|
||||
module: ADAPTER,
|
||||
method: "generate",
|
||||
reason: new InvalidProviderOutputReason({ message, route: ADAPTER, providerMetadata }),
|
||||
})
|
||||
|
||||
const applyQuery = (url: string, query: Record<string, string> | undefined) => {
|
||||
if (!query) return url
|
||||
const next = new URL(url)
|
||||
Object.entries(query).forEach(([key, value]) => next.searchParams.set(key, value))
|
||||
return next.toString()
|
||||
}
|
||||
|
||||
export const model = (input: ModelInput) => {
|
||||
const route: ImageRoute<GoogleImageOptions> = {
|
||||
id: ADAPTER,
|
||||
generate: Effect.fn("GoogleImages.generate")(function* (request: ImageRequestFor<GoogleImageOptions>, execute) {
|
||||
const http = mergeHttpOptions(request.model.http, request.http)
|
||||
const text = ProviderShared.encodeJson(body(request, http?.body))
|
||||
const url = applyQuery(
|
||||
`${(input.baseURL ?? DEFAULT_BASE_URL).replace(/\/$/, "")}/models/${request.model.id}:generateContent`,
|
||||
http?.query,
|
||||
)
|
||||
const headers = yield* Auth.toEffect(input.auth)({
|
||||
request,
|
||||
method: "POST",
|
||||
url,
|
||||
body: text,
|
||||
headers: Headers.fromInput({ ...input.headers, ...http?.headers }),
|
||||
})
|
||||
const response = yield* execute(
|
||||
HttpClientRequest.post(url).pipe(
|
||||
HttpClientRequest.setHeaders(headers),
|
||||
HttpClientRequest.bodyText(text, "application/json"),
|
||||
),
|
||||
)
|
||||
const payload = yield* response.json.pipe(
|
||||
Effect.mapError(() => invalidOutput("Failed to read the Google Images response")),
|
||||
)
|
||||
const decoded = yield* Schema.decodeUnknownEffect(GoogleImageResponse)(payload).pipe(
|
||||
Effect.mapError(() => invalidOutput("Google Images returned an invalid response")),
|
||||
)
|
||||
const candidates = decoded.candidates ?? []
|
||||
const candidateMetadata = candidates.map((candidate, candidateIndex) => ({
|
||||
index: candidate.index ?? candidateIndex,
|
||||
finishReason: candidate.finishReason,
|
||||
finishMessage: candidate.finishMessage,
|
||||
safetyRatings: candidate.safetyRatings,
|
||||
citationMetadata: candidate.citationMetadata,
|
||||
groundingMetadata: candidate.groundingMetadata,
|
||||
parts: (candidate.content?.parts ?? []).map((part) =>
|
||||
part.inlineData === undefined
|
||||
? {
|
||||
type: "text",
|
||||
text: part.text,
|
||||
thought: part.thought,
|
||||
thoughtSignature: part.thoughtSignature,
|
||||
}
|
||||
: {
|
||||
type: "inlineData",
|
||||
mediaType: part.inlineData.mimeType,
|
||||
thought: part.thought,
|
||||
thoughtSignature: part.thoughtSignature,
|
||||
},
|
||||
),
|
||||
}))
|
||||
const encoded = candidates.flatMap((candidate, candidateIndex) =>
|
||||
(candidate.content?.parts ?? []).flatMap((part, partIndex) =>
|
||||
part.inlineData === undefined || part.thought === true
|
||||
? []
|
||||
: [{ candidate, candidateIndex, partIndex, inlineData: part.inlineData }],
|
||||
),
|
||||
)
|
||||
const images = yield* Effect.forEach(encoded, (item) =>
|
||||
Effect.fromResult(Encoding.decodeBase64(item.inlineData.data)).pipe(
|
||||
Effect.mapError(() =>
|
||||
invalidOutput(
|
||||
`Google Images candidate ${item.candidateIndex} part ${item.partIndex} contains invalid base64 data`,
|
||||
),
|
||||
),
|
||||
Effect.map(
|
||||
(data) =>
|
||||
new GeneratedImage({
|
||||
mediaType: item.inlineData.mimeType,
|
||||
data,
|
||||
providerMetadata: {
|
||||
google: {
|
||||
candidateIndex: item.candidate.index ?? item.candidateIndex,
|
||||
partIndex: item.partIndex,
|
||||
finishReason: item.candidate.finishReason,
|
||||
safetyRatings: item.candidate.safetyRatings,
|
||||
citationMetadata: item.candidate.citationMetadata,
|
||||
groundingMetadata: item.candidate.groundingMetadata,
|
||||
thoughtSignature: item.candidate.content?.parts[item.partIndex]?.thoughtSignature,
|
||||
},
|
||||
},
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
if (images.length === 0) {
|
||||
const finishReasons = candidates.flatMap((candidate) =>
|
||||
candidate.finishReason === undefined ? [] : [candidate.finishReason],
|
||||
)
|
||||
return yield* invalidOutput(
|
||||
`Google Images returned no final images${
|
||||
finishReasons.length === 0 ? "" : ` (finish reasons: ${finishReasons.join(", ")})`
|
||||
}; inspect reason.providerMetadata.google for prompt feedback and candidate details`,
|
||||
{
|
||||
google: {
|
||||
promptFeedback: decoded.promptFeedback,
|
||||
candidates: candidateMetadata,
|
||||
},
|
||||
},
|
||||
)
|
||||
}
|
||||
const usage = decoded.usageMetadata
|
||||
const outputTokens =
|
||||
usage?.candidatesTokenCount === undefined
|
||||
? undefined
|
||||
: usage.candidatesTokenCount + (usage.thoughtsTokenCount ?? 0)
|
||||
return new ImageResponse({
|
||||
images,
|
||||
usage:
|
||||
usage === undefined
|
||||
? undefined
|
||||
: new Usage({
|
||||
inputTokens: usage.promptTokenCount,
|
||||
outputTokens,
|
||||
nonCachedInputTokens: ProviderShared.subtractTokens(
|
||||
usage.promptTokenCount,
|
||||
usage.cachedContentTokenCount,
|
||||
),
|
||||
cacheReadInputTokens: usage.cachedContentTokenCount,
|
||||
reasoningTokens: usage.thoughtsTokenCount,
|
||||
totalTokens: ProviderShared.totalTokens(usage.promptTokenCount, outputTokens, usage.totalTokenCount),
|
||||
providerMetadata: { google: usage },
|
||||
}),
|
||||
providerMetadata: {
|
||||
google: {
|
||||
modelVersion: decoded.modelVersion,
|
||||
responseId: decoded.responseId,
|
||||
promptFeedback: decoded.promptFeedback,
|
||||
candidates: candidateMetadata,
|
||||
},
|
||||
},
|
||||
})
|
||||
}),
|
||||
}
|
||||
return ImageModel.make<GoogleImageOptions>({ id: input.id, provider: "google", route, http: input.http })
|
||||
}
|
||||
|
||||
export const GoogleImages = {
|
||||
model,
|
||||
} as const
|
||||
|
|
@ -2,14 +2,20 @@ import type { RouteDefaultsInput } from "../route/client"
|
|||
import { Auth } from "../route/auth"
|
||||
import type { ProviderAuthOption } from "../route/auth-options"
|
||||
import type { ProviderPackage } from "../provider-package"
|
||||
import { ProviderID, type ModelID, type ProviderOptions } from "../schema"
|
||||
import * as Gemini from "../protocols/gemini"
|
||||
import { HttpOptions, ProviderID, mergeHttpOptions, type ModelID, type ProviderOptions } from "../schema"
|
||||
import { Gemini } from "../protocols/gemini"
|
||||
import { GoogleImages } from "../protocols/google-images"
|
||||
|
||||
export type { GoogleImageOptions } from "../protocols/google-images"
|
||||
|
||||
export const id = ProviderID.make("google")
|
||||
|
||||
export const routes = [Gemini.route]
|
||||
|
||||
export type Config = RouteDefaultsInput & ProviderAuthOption<"optional"> & { readonly baseURL?: string }
|
||||
export type Config = RouteDefaultsInput &
|
||||
ProviderAuthOption<"optional"> & {
|
||||
readonly baseURL?: string
|
||||
}
|
||||
|
||||
export interface Settings extends ProviderPackage.Settings {
|
||||
readonly apiKey?: string
|
||||
|
|
@ -31,9 +37,18 @@ const configuredRoute = (input: Config) => {
|
|||
|
||||
export const configure = (input: Config = {}) => {
|
||||
const route = configuredRoute(input)
|
||||
const image = (modelID: string | ModelID) =>
|
||||
GoogleImages.model({
|
||||
id: modelID,
|
||||
auth: auth(input),
|
||||
baseURL: input.baseURL,
|
||||
headers: input.headers,
|
||||
http: mergeHttpOptions(input.http === undefined ? undefined : HttpOptions.make(input.http)),
|
||||
})
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) => route.model({ id: modelID }),
|
||||
image,
|
||||
configure,
|
||||
}
|
||||
}
|
||||
|
|
@ -48,3 +63,5 @@ export const model: ProviderPackage.Definition<Settings>["model"] = (modelID, se
|
|||
limits: settings.limits,
|
||||
providerOptions: settings.providerOptions,
|
||||
}).model(modelID)
|
||||
|
||||
export const image = provider.image
|
||||
|
|
|
|||
32
packages/ai/test/fixtures/recordings/google-images/generates-an-image.json
vendored
Normal file
32
packages/ai/test/fixtures/recordings/google-images/generates-an-image.json
vendored
Normal file
File diff suppressed because one or more lines are too long
|
|
@ -2,7 +2,7 @@ import { describe, expect } from "bun:test"
|
|||
import { Effect, Layer } from "effect"
|
||||
import { HttpClientRequest } from "effect/unstable/http"
|
||||
import { Image, ImageClient } from "../src"
|
||||
import { OpenAI } from "../src/providers"
|
||||
import { Google, OpenAI } from "../src/providers"
|
||||
import { it } from "./lib/effect"
|
||||
import { dynamicResponse } from "./lib/http"
|
||||
|
||||
|
|
@ -124,4 +124,250 @@ describe("Image", () => {
|
|||
),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("generates images through the Google generateContent API", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
model: Google.configure({
|
||||
apiKey: "test",
|
||||
baseURL: "https://generativelanguage.test/v1beta/",
|
||||
headers: { "x-default": "yes" },
|
||||
http: { body: { labels: { deployment: "test" } }, query: { api: "v1" } },
|
||||
}).image("any-model-id"),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
options: {
|
||||
aspectRatio: "16:9",
|
||||
imageSize: "2K",
|
||||
seed: 42,
|
||||
thinkingLevel: "HIGH",
|
||||
includeThoughts: true,
|
||||
futureOption: true,
|
||||
imageConfig: { aspectRatio: "4:3", nativeImageOption: true },
|
||||
thinkingConfig: { thinkingLevel: "LOW", nativeThinkingOption: true },
|
||||
},
|
||||
http: {
|
||||
body: {
|
||||
safetySettings: [],
|
||||
generationConfig: {
|
||||
imageConfig: { aspectRatio: "3:2", httpImageOption: true },
|
||||
thinkingConfig: { includeThoughts: false, httpThinkingOption: true },
|
||||
futureOption: "http",
|
||||
httpOption: true,
|
||||
},
|
||||
},
|
||||
headers: { "x-request": "yes" },
|
||||
query: { trace: "1" },
|
||||
},
|
||||
})
|
||||
|
||||
expect(response.images).toHaveLength(3)
|
||||
expect(response.images.map((image) => image.data)).toEqual([
|
||||
Uint8Array.from([1, 2, 3]),
|
||||
Uint8Array.from([4, 5, 6]),
|
||||
Uint8Array.from([7, 8, 9]),
|
||||
])
|
||||
expect(response.images.map((image) => image.mediaType)).toEqual(["image/png", "image/jpeg", "image/webp"])
|
||||
expect(response.images[0].providerMetadata).toMatchObject({ google: { thoughtSignature: "signature-1" } })
|
||||
expect(response.images[1].providerMetadata).toMatchObject({
|
||||
google: { candidateIndex: 0, partIndex: 3, finishReason: "STOP" },
|
||||
})
|
||||
expect(response.images[2].providerMetadata).toMatchObject({ google: { candidateIndex: 7, partIndex: 0 } })
|
||||
expect(response.usage?.inputTokens).toBe(5)
|
||||
expect(response.usage?.outputTokens).toBe(10)
|
||||
expect(response.usage?.reasoningTokens).toBe(3)
|
||||
expect(response.usage?.providerMetadata).toMatchObject({ google: { serviceTier: "STANDARD" } })
|
||||
expect(response.providerMetadata).toEqual({
|
||||
google: {
|
||||
modelVersion: "gemini-3.1-flash-image",
|
||||
responseId: "response-1",
|
||||
promptFeedback: undefined,
|
||||
candidates: [
|
||||
{
|
||||
index: 0,
|
||||
finishReason: "STOP",
|
||||
finishMessage: undefined,
|
||||
safetyRatings: [{ category: "safe" }],
|
||||
citationMetadata: undefined,
|
||||
groundingMetadata: undefined,
|
||||
parts: [
|
||||
{
|
||||
type: "inlineData",
|
||||
mediaType: "image/png",
|
||||
thought: undefined,
|
||||
thoughtSignature: "signature-1",
|
||||
},
|
||||
{ type: "text", text: "planning", thought: true, thoughtSignature: "text-signature" },
|
||||
{
|
||||
type: "inlineData",
|
||||
mediaType: "image/png",
|
||||
thought: true,
|
||||
thoughtSignature: "draft-signature",
|
||||
},
|
||||
{
|
||||
type: "inlineData",
|
||||
mediaType: "image/jpeg",
|
||||
thought: undefined,
|
||||
thoughtSignature: undefined,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
index: 7,
|
||||
finishReason: undefined,
|
||||
finishMessage: undefined,
|
||||
safetyRatings: undefined,
|
||||
citationMetadata: undefined,
|
||||
groundingMetadata: undefined,
|
||||
parts: [
|
||||
{
|
||||
type: "inlineData",
|
||||
mediaType: "image/webp",
|
||||
thought: undefined,
|
||||
thoughtSignature: undefined,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
ImageClient.layer.pipe(
|
||||
Layer.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(request.url).toBe(
|
||||
"https://generativelanguage.test/v1beta/models/any-model-id:generateContent?api=v1&trace=1",
|
||||
)
|
||||
expect(request.headers.get("x-goog-api-key")).toBe("test")
|
||||
expect(request.headers.get("x-default")).toBe("yes")
|
||||
expect(request.headers.get("x-request")).toBe("yes")
|
||||
expect(JSON.parse(input.text)).toEqual({
|
||||
contents: [{ role: "user", parts: [{ text: "A robot tending a rooftop garden" }] }],
|
||||
generationConfig: {
|
||||
responseModalities: ["IMAGE"],
|
||||
imageConfig: {
|
||||
aspectRatio: "3:2",
|
||||
imageSize: "2K",
|
||||
nativeImageOption: true,
|
||||
httpImageOption: true,
|
||||
},
|
||||
seed: 42,
|
||||
thinkingConfig: {
|
||||
thinkingLevel: "LOW",
|
||||
includeThoughts: false,
|
||||
nativeThinkingOption: true,
|
||||
httpThinkingOption: true,
|
||||
},
|
||||
futureOption: "http",
|
||||
httpOption: true,
|
||||
},
|
||||
labels: { deployment: "test" },
|
||||
safetySettings: [],
|
||||
})
|
||||
return input.respond(
|
||||
JSON.stringify({
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
parts: [
|
||||
{
|
||||
inlineData: { mimeType: "image/png", data: "AQID" },
|
||||
thoughtSignature: "signature-1",
|
||||
},
|
||||
{ text: "planning", thought: true, thoughtSignature: "text-signature" },
|
||||
{
|
||||
inlineData: { mimeType: "image/png", data: "CgsM" },
|
||||
thought: true,
|
||||
thoughtSignature: "draft-signature",
|
||||
},
|
||||
{ inlineData: { mimeType: "image/jpeg", data: "BAUG" } },
|
||||
],
|
||||
},
|
||||
finishReason: "STOP",
|
||||
safetyRatings: [{ category: "safe" }],
|
||||
},
|
||||
{
|
||||
index: 7,
|
||||
content: { parts: [{ inlineData: { mimeType: "image/webp", data: "BwgJ" } }] },
|
||||
},
|
||||
],
|
||||
usageMetadata: {
|
||||
promptTokenCount: 5,
|
||||
candidatesTokenCount: 7,
|
||||
thoughtsTokenCount: 3,
|
||||
totalTokenCount: 15,
|
||||
serviceTier: "STANDARD",
|
||||
},
|
||||
modelVersion: "gemini-3.1-flash-image",
|
||||
responseId: "response-1",
|
||||
}),
|
||||
{ headers: { "content-type": "application/json" } },
|
||||
)
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("includes Google diagnostics when no final image is returned", () =>
|
||||
Image.generate({
|
||||
model: Google.configure({ apiKey: "test", baseURL: "https://generativelanguage.test/v1beta" }).image(
|
||||
"gemini-3.1-flash-image",
|
||||
),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
}).pipe(
|
||||
Effect.flip,
|
||||
Effect.tap((error) =>
|
||||
Effect.sync(() => {
|
||||
expect(error.reason._tag).toBe("InvalidProviderOutput")
|
||||
if (error.reason._tag !== "InvalidProviderOutput") return
|
||||
expect(error.reason.message).toContain("finish reasons: IMAGE_SAFETY")
|
||||
expect(error.reason.providerMetadata).toEqual({
|
||||
google: {
|
||||
promptFeedback: { blockReason: "SAFETY" },
|
||||
candidates: [
|
||||
{
|
||||
index: 0,
|
||||
finishReason: "IMAGE_SAFETY",
|
||||
finishMessage: "The generated image was blocked by safety filters.",
|
||||
safetyRatings: [{ category: "HARM_CATEGORY_DANGEROUS_CONTENT", blocked: true }],
|
||||
citationMetadata: undefined,
|
||||
groundingMetadata: undefined,
|
||||
parts: [{ type: "text", text: "blocked", thought: false, thoughtSignature: undefined }],
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
}),
|
||||
),
|
||||
Effect.provide(
|
||||
ImageClient.layer.pipe(
|
||||
Layer.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.succeed(
|
||||
input.respond(
|
||||
JSON.stringify({
|
||||
candidates: [
|
||||
{
|
||||
content: { parts: [{ text: "blocked", thought: false }] },
|
||||
finishReason: "IMAGE_SAFETY",
|
||||
finishMessage: "The generated image was blocked by safety filters.",
|
||||
safetyRatings: [{ category: "HARM_CATEGORY_DANGEROUS_CONTENT", blocked: true }],
|
||||
},
|
||||
],
|
||||
promptFeedback: { blockReason: "SAFETY" },
|
||||
}),
|
||||
{ headers: { "content-type": "application/json" } },
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
})
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import {
|
|||
type ImageRequestFor,
|
||||
type ImageRoute,
|
||||
} from "../src"
|
||||
import { OpenAI, XAI } from "../src/providers"
|
||||
import { Google, OpenAI, XAI } from "../src/providers"
|
||||
|
||||
type GoogleLikeOptions = {
|
||||
readonly aspectRatio?: "1:1" | "16:9"
|
||||
|
|
@ -25,6 +25,33 @@ Image.generate({
|
|||
options: { aspectRatio: "16:9", imageSize: "2K", futureOption: true },
|
||||
})
|
||||
|
||||
const googleProvider = Google.configure({ apiKey: "test" }).image("any-model-id")
|
||||
Image.generate({
|
||||
model: googleProvider,
|
||||
prompt: "A lighthouse",
|
||||
options: {
|
||||
aspectRatio: "16:9",
|
||||
imageSize: "2K",
|
||||
seed: 42,
|
||||
thinkingLevel: "HIGH",
|
||||
includeThoughts: true,
|
||||
futureOption: true,
|
||||
},
|
||||
})
|
||||
Image.generate({
|
||||
model: googleProvider,
|
||||
prompt: "A lighthouse",
|
||||
options: { aspectRatio: "future-ratio", imageSize: "8K", thinkingLevel: "FUTURE" },
|
||||
})
|
||||
// @ts-expect-error Image generation options are request-scoped, not provider configuration.
|
||||
Google.configure({ image: { providerOptions: { imageSize: "2K" } } })
|
||||
// @ts-expect-error Known Google string options retain their value kind.
|
||||
Image.generate({ model: googleProvider, prompt: "A lighthouse", options: { imageSize: 2 } })
|
||||
// @ts-expect-error Known Google numeric options retain their value kind.
|
||||
Image.generate({ model: googleProvider, prompt: "A lighthouse", options: { seed: "42" } })
|
||||
// @ts-expect-error Known Google boolean options retain their value kind.
|
||||
Image.generate({ model: googleProvider, prompt: "A lighthouse", options: { includeThoughts: "yes" } })
|
||||
|
||||
const openai = OpenAI.image("gpt-image-2")
|
||||
// @ts-expect-error Image generation options are request-scoped, not provider configuration.
|
||||
OpenAI.configure({ image: { options: { quality: "medium" } } })
|
||||
|
|
|
|||
33
packages/ai/test/provider/google-images.recorded.test.ts
Normal file
33
packages/ai/test/provider/google-images.recorded.test.ts
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { Image } from "../../src"
|
||||
import { Google } from "../../src/providers"
|
||||
import { recordedTests } from "../recorded-test"
|
||||
|
||||
const model = Google.configure({
|
||||
apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY ?? "fixture",
|
||||
}).image("gemini-3.1-flash-image")
|
||||
|
||||
const recorded = recordedTests({
|
||||
prefix: "google-images",
|
||||
provider: "google",
|
||||
protocol: "google-images",
|
||||
requires: ["GOOGLE_GENERATIVE_AI_API_KEY"],
|
||||
})
|
||||
|
||||
describe("Google Images recorded", () => {
|
||||
recorded.effect("generates an image", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
model,
|
||||
prompt: "A simple flat blue circle centered on a plain white background.",
|
||||
options: { aspectRatio: "1:1" },
|
||||
})
|
||||
|
||||
expect(response.images).toHaveLength(1)
|
||||
expect(response.image?.mediaType).toMatch(/^image\//)
|
||||
expect(response.image?.data).toBeInstanceOf(Uint8Array)
|
||||
expect(response.image?.data.length).toBeGreaterThan(0)
|
||||
}),
|
||||
)
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue