feat(session): add native OpenAI runtime opt-in
This commit is contained in:
Kit Langton
parent
9bbb688459
commit
a878036f63
3 changed files with 241 additions and 79 deletions
|
|
@ -11,6 +11,8 @@ type ToolInput = {
|
|||
|
||||
export type RequestInput = {
|
||||
readonly model: Provider.Model
|
||||
readonly apiKey?: string
|
||||
readonly baseURL?: string
|
||||
readonly system?: readonly string[]
|
||||
readonly messages: readonly ModelMessage[]
|
||||
readonly tools?: Record<string, ToolInput>
|
||||
|
|
@ -154,14 +156,16 @@ const baseURL = (model: Provider.Model) => {
|
|||
throw new Error(`Native LLM request adapter requires a base URL for ${model.providerID}/${model.id}`)
|
||||
}
|
||||
|
||||
export const model = (model: Provider.Model, headers?: Record<string, string>) => {
|
||||
export const model = (input: Provider.Model | RequestInput, headers?: Record<string, string>) => {
|
||||
const model = "model" in input ? input.model : input
|
||||
const route = ROUTE[model.api.npm]
|
||||
if (!route) throw new Error(`Native LLM request adapter does not support provider package ${model.api.npm}`)
|
||||
return LLM.model({
|
||||
id: model.api.id,
|
||||
provider: model.providerID,
|
||||
route,
|
||||
baseURL: baseURL(model),
|
||||
baseURL: "model" in input && input.baseURL ? input.baseURL : baseURL(model),
|
||||
apiKey: "model" in input ? input.apiKey : undefined,
|
||||
headers: Object.keys({ ...model.headers, ...headers }).length === 0 ? undefined : { ...model.headers, ...headers },
|
||||
limits: {
|
||||
context: model.limit.context,
|
||||
|
|
@ -173,7 +177,7 @@ export const model = (model: Provider.Model, headers?: Record<string, string>) =
|
|||
export const request = (input: RequestInput) => {
|
||||
const converted = messages(input.messages)
|
||||
return LLM.request({
|
||||
model: model(input.model, input.headers),
|
||||
model: model(input, input.headers),
|
||||
system: [...(input.system ?? []).map(SystemPart.make), ...converted.system],
|
||||
messages: converted.messages,
|
||||
tools: tools(input.tools),
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { Context, Effect, Layer, Record } from "effect"
|
|||
import * as Stream from "effect/Stream"
|
||||
import { streamText, wrapLanguageModel, type ModelMessage, type Tool, tool, jsonSchema } from "ai"
|
||||
import type { LLMEvent } from "@opencode-ai/llm"
|
||||
import { LLMClient, RequestExecutor } from "@opencode-ai/llm/route"
|
||||
import { mergeDeep } from "remeda"
|
||||
import { GitLabWorkflowLanguageModel } from "gitlab-ai-provider"
|
||||
import { ProviderTransform } from "@/provider/transform"
|
||||
|
|
@ -20,12 +21,12 @@ import { Bus } from "@/bus"
|
|||
import { Wildcard } from "@/util/wildcard"
|
||||
import { SessionID } from "@/session/schema"
|
||||
import { Auth } from "@/auth"
|
||||
import { Installation } from "@/installation"
|
||||
import { InstallationVersion } from "@opencode-ai/core/installation/version"
|
||||
import { EffectBridge } from "@/effect/bridge"
|
||||
import * as Option from "effect/Option"
|
||||
import * as OtelTracer from "@effect/opentelemetry/Tracer"
|
||||
import { LLMAISDK } from "./llm-ai-sdk"
|
||||
import { LLMNative } from "./llm-native"
|
||||
|
||||
const log = Log.create({ service: "llm" })
|
||||
export const OUTPUT_TOKEN_MAX = ProviderTransform.OUTPUT_TOKEN_MAX
|
||||
|
|
@ -34,6 +35,8 @@ export const OUTPUT_TOKEN_MAX = ProviderTransform.OUTPUT_TOKEN_MAX
|
|||
const mergeOptions = (target: Record<string, any>, source: Record<string, any> | undefined): Record<string, any> =>
|
||||
mergeDeep(target, source ?? {}) as Record<string, any>
|
||||
|
||||
const runtime = () => (process.env.OPENCODE_LLM_RUNTIME === "native" ? "native" : "ai-sdk")
|
||||
|
||||
export type StreamInput = {
|
||||
user: MessageV2.User
|
||||
sessionID: string
|
||||
|
|
@ -333,86 +336,123 @@ const live: Layer.Layer<
|
|||
? (yield* InstanceState.context).project.id
|
||||
: undefined
|
||||
|
||||
return streamText({
|
||||
onError(error) {
|
||||
l.error("stream error", {
|
||||
error,
|
||||
})
|
||||
},
|
||||
async experimental_repairToolCall(failed) {
|
||||
const lower = failed.toolCall.toolName.toLowerCase()
|
||||
if (lower !== failed.toolCall.toolName && sortedTools[lower]) {
|
||||
l.info("repairing tool call", {
|
||||
tool: failed.toolCall.toolName,
|
||||
repaired: lower,
|
||||
const requestHeaders = {
|
||||
...(input.model.providerID.startsWith("opencode")
|
||||
? {
|
||||
...(opencodeProjectID ? { "x-opencode-project": opencodeProjectID } : {}),
|
||||
"x-opencode-session": input.sessionID,
|
||||
"x-opencode-request": input.user.id,
|
||||
"x-opencode-client": Flag.OPENCODE_CLIENT,
|
||||
"User-Agent": `opencode/${InstallationVersion}`,
|
||||
}
|
||||
: {
|
||||
"x-session-affinity": input.sessionID,
|
||||
...(input.parentSessionID ? { "x-parent-session-id": input.parentSessionID } : {}),
|
||||
"User-Agent": `opencode/${InstallationVersion}`,
|
||||
}),
|
||||
...input.model.headers,
|
||||
...headers,
|
||||
}
|
||||
|
||||
if (runtime() === "native") {
|
||||
if (input.model.providerID !== "openai" || input.model.api.npm !== "@ai-sdk/openai") {
|
||||
return yield* Effect.fail(new Error("Native LLM runtime currently only supports OpenAI models"))
|
||||
}
|
||||
if (Object.keys(sortedTools).length > 0) {
|
||||
return yield* Effect.fail(new Error("Native LLM runtime does not support tools yet"))
|
||||
}
|
||||
const apiKey =
|
||||
info?.type === "api" ? info.key : typeof item.options.apiKey === "string" ? item.options.apiKey : undefined
|
||||
if (!apiKey) return yield* Effect.fail(new Error("Native LLM runtime requires API key auth for OpenAI"))
|
||||
const baseURL = typeof item.options.baseURL === "string" ? item.options.baseURL : undefined
|
||||
return {
|
||||
type: "native" as const,
|
||||
stream: LLMClient.stream(
|
||||
LLMNative.request({
|
||||
model: input.model,
|
||||
apiKey,
|
||||
baseURL,
|
||||
system: isOpenaiOauth ? system : [],
|
||||
messages: ProviderTransform.message(messages, input.model, options),
|
||||
toolChoice: input.toolChoice,
|
||||
temperature: params.temperature,
|
||||
topP: params.topP,
|
||||
topK: params.topK,
|
||||
maxOutputTokens: params.maxOutputTokens,
|
||||
providerOptions: ProviderTransform.providerOptions(input.model, params.options),
|
||||
headers: requestHeaders,
|
||||
}),
|
||||
).pipe(Stream.provide(LLMClient.layer), Stream.provide(RequestExecutor.defaultLayer)),
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
type: "ai-sdk" as const,
|
||||
result: streamText({
|
||||
onError(error) {
|
||||
l.error("stream error", {
|
||||
error,
|
||||
})
|
||||
},
|
||||
async experimental_repairToolCall(failed) {
|
||||
const lower = failed.toolCall.toolName.toLowerCase()
|
||||
if (lower !== failed.toolCall.toolName && sortedTools[lower]) {
|
||||
l.info("repairing tool call", {
|
||||
tool: failed.toolCall.toolName,
|
||||
repaired: lower,
|
||||
})
|
||||
return {
|
||||
...failed.toolCall,
|
||||
toolName: lower,
|
||||
}
|
||||
}
|
||||
return {
|
||||
...failed.toolCall,
|
||||
toolName: lower,
|
||||
}
|
||||
}
|
||||
return {
|
||||
...failed.toolCall,
|
||||
input: JSON.stringify({
|
||||
tool: failed.toolCall.toolName,
|
||||
error: failed.error.message,
|
||||
}),
|
||||
toolName: "invalid",
|
||||
}
|
||||
},
|
||||
temperature: params.temperature,
|
||||
topP: params.topP,
|
||||
topK: params.topK,
|
||||
providerOptions: ProviderTransform.providerOptions(input.model, params.options),
|
||||
activeTools: Object.keys(sortedTools).filter((x) => x !== "invalid"),
|
||||
tools: sortedTools,
|
||||
toolChoice: input.toolChoice,
|
||||
maxOutputTokens: params.maxOutputTokens,
|
||||
abortSignal: input.abort,
|
||||
headers: {
|
||||
...(input.model.providerID.startsWith("opencode")
|
||||
? {
|
||||
"x-opencode-project": opencodeProjectID,
|
||||
"x-opencode-session": input.sessionID,
|
||||
"x-opencode-request": input.user.id,
|
||||
"x-opencode-client": Flag.OPENCODE_CLIENT,
|
||||
"User-Agent": `opencode/${InstallationVersion}`,
|
||||
}
|
||||
: {
|
||||
"x-session-affinity": input.sessionID,
|
||||
...(input.parentSessionID ? { "x-parent-session-id": input.parentSessionID } : {}),
|
||||
"User-Agent": `opencode/${InstallationVersion}`,
|
||||
input: JSON.stringify({
|
||||
tool: failed.toolCall.toolName,
|
||||
error: failed.error.message,
|
||||
}),
|
||||
...input.model.headers,
|
||||
...headers,
|
||||
},
|
||||
maxRetries: input.retries ?? 0,
|
||||
messages,
|
||||
model: wrapLanguageModel({
|
||||
model: language,
|
||||
middleware: [
|
||||
{
|
||||
specificationVersion: "v3" as const,
|
||||
async transformParams(args) {
|
||||
if (args.type === "stream") {
|
||||
// @ts-expect-error
|
||||
args.params.prompt = ProviderTransform.message(args.params.prompt, input.model, options)
|
||||
}
|
||||
return args.params
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
experimental_telemetry: {
|
||||
isEnabled: cfg.experimental?.openTelemetry,
|
||||
functionId: "session.llm",
|
||||
tracer: telemetryTracer,
|
||||
metadata: {
|
||||
userId: cfg.username ?? "unknown",
|
||||
sessionId: input.sessionID,
|
||||
toolName: "invalid",
|
||||
}
|
||||
},
|
||||
},
|
||||
})
|
||||
temperature: params.temperature,
|
||||
topP: params.topP,
|
||||
topK: params.topK,
|
||||
providerOptions: ProviderTransform.providerOptions(input.model, params.options),
|
||||
activeTools: Object.keys(sortedTools).filter((x) => x !== "invalid"),
|
||||
tools: sortedTools,
|
||||
toolChoice: input.toolChoice,
|
||||
maxOutputTokens: params.maxOutputTokens,
|
||||
abortSignal: input.abort,
|
||||
headers: requestHeaders,
|
||||
maxRetries: input.retries ?? 0,
|
||||
messages,
|
||||
model: wrapLanguageModel({
|
||||
model: language,
|
||||
middleware: [
|
||||
{
|
||||
specificationVersion: "v3" as const,
|
||||
async transformParams(args) {
|
||||
if (args.type === "stream") {
|
||||
// @ts-expect-error
|
||||
args.params.prompt = ProviderTransform.message(args.params.prompt, input.model, options)
|
||||
}
|
||||
return args.params
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
experimental_telemetry: {
|
||||
isEnabled: cfg.experimental?.openTelemetry,
|
||||
functionId: "session.llm",
|
||||
tracer: telemetryTracer,
|
||||
metadata: {
|
||||
userId: cfg.username ?? "unknown",
|
||||
sessionId: input.sessionID,
|
||||
},
|
||||
},
|
||||
}),
|
||||
}
|
||||
})
|
||||
|
||||
const stream: Interface["stream"] = (input) =>
|
||||
|
|
@ -426,8 +466,12 @@ const live: Layer.Layer<
|
|||
|
||||
const result = yield* run({ ...input, abort: ctrl.signal })
|
||||
|
||||
if (result.type === "native") return result.stream
|
||||
|
||||
const state = LLMAISDK.adapterState()
|
||||
return Stream.fromAsyncIterable(result.fullStream, (e) => (e instanceof Error ? e : new Error(String(e)))).pipe(
|
||||
return Stream.fromAsyncIterable(result.result.fullStream, (e) =>
|
||||
e instanceof Error ? e : new Error(String(e)),
|
||||
).pipe(
|
||||
Stream.mapEffect((event) => LLMAISDK.toLLMEvents(state, event)),
|
||||
Stream.flatMap((events) => Stream.fromIterable(events)),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import { Cause, Effect, Exit, Stream } from "effect"
|
|||
import z from "zod"
|
||||
import { makeRuntime } from "../../src/effect/run-service"
|
||||
import { LLM } from "../../src/session/llm"
|
||||
import { Instance } from "../../src/project/instance"
|
||||
import { WithInstance } from "../../src/project/with-instance"
|
||||
import { Provider } from "@/provider/provider"
|
||||
import { ProviderTransform } from "@/provider/transform"
|
||||
|
|
@ -688,6 +687,121 @@ describe("session.llm.stream", () => {
|
|||
})
|
||||
})
|
||||
|
||||
test("streams OpenAI through native runtime when opted in", async () => {
|
||||
const server = state.server
|
||||
if (!server) {
|
||||
throw new Error("Server not initialized")
|
||||
}
|
||||
|
||||
const source = await loadFixture("openai", "gpt-5.2")
|
||||
const model = source.model
|
||||
const chunks = [
|
||||
{
|
||||
type: "response.created",
|
||||
response: {
|
||||
id: "resp-native",
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "response.output_item.added",
|
||||
item: { type: "message", id: "item-native", status: "in_progress" },
|
||||
},
|
||||
{
|
||||
type: "response.output_text.delta",
|
||||
item_id: "item-native",
|
||||
delta: "Hello native",
|
||||
},
|
||||
{
|
||||
type: "response.completed",
|
||||
response: {
|
||||
incomplete_details: null,
|
||||
usage: {
|
||||
input_tokens: 1,
|
||||
input_tokens_details: null,
|
||||
output_tokens: 1,
|
||||
output_tokens_details: null,
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
const request = waitRequest("/responses", createEventResponse(chunks, true))
|
||||
|
||||
await using tmp = await tmpdir({
|
||||
init: async (dir) => {
|
||||
await Bun.write(
|
||||
path.join(dir, "opencode.json"),
|
||||
JSON.stringify({
|
||||
$schema: "https://opencode.ai/config.json",
|
||||
enabled_providers: ["openai"],
|
||||
provider: {
|
||||
openai: {
|
||||
name: "OpenAI",
|
||||
env: ["OPENAI_API_KEY"],
|
||||
npm: "@ai-sdk/openai",
|
||||
api: "https://api.openai.com/v1",
|
||||
models: {
|
||||
[model.id]: model,
|
||||
},
|
||||
options: {
|
||||
apiKey: "test-openai-key",
|
||||
baseURL: `${server.url.origin}/v1`,
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
)
|
||||
},
|
||||
})
|
||||
|
||||
await WithInstance.provide({
|
||||
directory: tmp.path,
|
||||
fn: async () => {
|
||||
const previous = process.env.OPENCODE_LLM_RUNTIME
|
||||
process.env.OPENCODE_LLM_RUNTIME = "native"
|
||||
try {
|
||||
const resolved = await getModel(ProviderID.openai, ModelID.make(model.id))
|
||||
const sessionID = SessionID.make("session-test-native")
|
||||
const agent = {
|
||||
name: "test",
|
||||
mode: "primary",
|
||||
options: {},
|
||||
permission: [{ permission: "*", pattern: "*", action: "allow" }],
|
||||
temperature: 0.2,
|
||||
} satisfies Agent.Info
|
||||
|
||||
await drain({
|
||||
user: {
|
||||
id: MessageID.make("msg_user-native"),
|
||||
sessionID,
|
||||
role: "user",
|
||||
time: { created: Date.now() },
|
||||
agent: agent.name,
|
||||
model: { providerID: ProviderID.make("openai"), modelID: resolved.id, variant: "high" },
|
||||
} satisfies MessageV2.User,
|
||||
sessionID,
|
||||
model: resolved,
|
||||
agent,
|
||||
system: ["You are a helpful assistant."],
|
||||
messages: [{ role: "user", content: "Hello" }],
|
||||
tools: {},
|
||||
})
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.OPENCODE_LLM_RUNTIME
|
||||
else process.env.OPENCODE_LLM_RUNTIME = previous
|
||||
}
|
||||
|
||||
const capture = await request
|
||||
expect(capture.url.pathname.endsWith("/responses")).toBe(true)
|
||||
expect(capture.headers.get("Authorization")).toBe("Bearer test-openai-key")
|
||||
expect(capture.body.model).toBe(model.id)
|
||||
expect(capture.body.stream).toBe(true)
|
||||
expect((capture.body.reasoning as { effort?: string } | undefined)?.effort).toBe("high")
|
||||
expect(JSON.stringify(capture.body.input)).toContain("You are a helpful assistant.")
|
||||
expect(capture.body.input).toContainEqual({ role: "user", content: [{ type: "input_text", text: "Hello" }] })
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
test("accepts user image attachments as data URLs for OpenAI models", async () => {
|
||||
const server = state.server
|
||||
if (!server) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue