From a878036f63a510fcc6057058a8d8b4d5b2169bb2 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Mon, 11 May 2026 17:44:09 -0400 Subject: [PATCH] feat(session): add native OpenAI runtime opt-in --- packages/opencode/src/session/llm-native.ts | 10 +- packages/opencode/src/session/llm.ts | 198 ++++++++++++-------- packages/opencode/test/session/llm.test.ts | 116 +++++++++++- 3 files changed, 243 insertions(+), 81 deletions(-) diff --git a/packages/opencode/src/session/llm-native.ts b/packages/opencode/src/session/llm-native.ts index 74be5988ee..817f29832d 100644 --- a/packages/opencode/src/session/llm-native.ts +++ b/packages/opencode/src/session/llm-native.ts @@ -11,6 +11,8 @@ type ToolInput = { export type RequestInput = { readonly model: Provider.Model + readonly apiKey?: string + readonly baseURL?: string readonly system?: readonly string[] readonly messages: readonly ModelMessage[] readonly tools?: Record @@ -154,14 +156,16 @@ const baseURL = (model: Provider.Model) => { throw new Error(`Native LLM request adapter requires a base URL for ${model.providerID}/${model.id}`) } -export const model = (model: Provider.Model, headers?: Record) => { +export const model = (input: Provider.Model | RequestInput, headers?: Record) => { + const model = "model" in input ? input.model : input const route = ROUTE[model.api.npm] if (!route) throw new Error(`Native LLM request adapter does not support provider package ${model.api.npm}`) return LLM.model({ id: model.api.id, provider: model.providerID, route, - baseURL: baseURL(model), + baseURL: "model" in input && input.baseURL ? input.baseURL : baseURL(model), + apiKey: "model" in input ? input.apiKey : undefined, headers: Object.keys({ ...model.headers, ...headers }).length === 0 ? undefined : { ...model.headers, ...headers }, limits: { context: model.limit.context, @@ -173,7 +177,7 @@ export const model = (model: Provider.Model, headers?: Record) = export const request = (input: RequestInput) => { const converted = messages(input.messages) return LLM.request({ - model: model(input.model, input.headers), + model: model(input, input.headers), system: [...(input.system ?? []).map(SystemPart.make), ...converted.system], messages: converted.messages, tools: tools(input.tools), diff --git a/packages/opencode/src/session/llm.ts b/packages/opencode/src/session/llm.ts index 28e05aba8f..3460a04c02 100644 --- a/packages/opencode/src/session/llm.ts +++ b/packages/opencode/src/session/llm.ts @@ -4,6 +4,7 @@ import { Context, Effect, Layer, Record } from "effect" import * as Stream from "effect/Stream" import { streamText, wrapLanguageModel, type ModelMessage, type Tool, tool, jsonSchema } from "ai" import type { LLMEvent } from "@opencode-ai/llm" +import { LLMClient, RequestExecutor } from "@opencode-ai/llm/route" import { mergeDeep } from "remeda" import { GitLabWorkflowLanguageModel } from "gitlab-ai-provider" import { ProviderTransform } from "@/provider/transform" @@ -20,12 +21,12 @@ import { Bus } from "@/bus" import { Wildcard } from "@/util/wildcard" import { SessionID } from "@/session/schema" import { Auth } from "@/auth" -import { Installation } from "@/installation" import { InstallationVersion } from "@opencode-ai/core/installation/version" import { EffectBridge } from "@/effect/bridge" import * as Option from "effect/Option" import * as OtelTracer from "@effect/opentelemetry/Tracer" import { LLMAISDK } from "./llm-ai-sdk" +import { LLMNative } from "./llm-native" const log = Log.create({ service: "llm" }) export const OUTPUT_TOKEN_MAX = ProviderTransform.OUTPUT_TOKEN_MAX @@ -34,6 +35,8 @@ export const OUTPUT_TOKEN_MAX = ProviderTransform.OUTPUT_TOKEN_MAX const mergeOptions = (target: Record, source: Record | undefined): Record => mergeDeep(target, source ?? {}) as Record +const runtime = () => (process.env.OPENCODE_LLM_RUNTIME === "native" ? "native" : "ai-sdk") + export type StreamInput = { user: MessageV2.User sessionID: string @@ -333,86 +336,123 @@ const live: Layer.Layer< ? (yield* InstanceState.context).project.id : undefined - return streamText({ - onError(error) { - l.error("stream error", { - error, - }) - }, - async experimental_repairToolCall(failed) { - const lower = failed.toolCall.toolName.toLowerCase() - if (lower !== failed.toolCall.toolName && sortedTools[lower]) { - l.info("repairing tool call", { - tool: failed.toolCall.toolName, - repaired: lower, + const requestHeaders = { + ...(input.model.providerID.startsWith("opencode") + ? { + ...(opencodeProjectID ? { "x-opencode-project": opencodeProjectID } : {}), + "x-opencode-session": input.sessionID, + "x-opencode-request": input.user.id, + "x-opencode-client": Flag.OPENCODE_CLIENT, + "User-Agent": `opencode/${InstallationVersion}`, + } + : { + "x-session-affinity": input.sessionID, + ...(input.parentSessionID ? { "x-parent-session-id": input.parentSessionID } : {}), + "User-Agent": `opencode/${InstallationVersion}`, + }), + ...input.model.headers, + ...headers, + } + + if (runtime() === "native") { + if (input.model.providerID !== "openai" || input.model.api.npm !== "@ai-sdk/openai") { + return yield* Effect.fail(new Error("Native LLM runtime currently only supports OpenAI models")) + } + if (Object.keys(sortedTools).length > 0) { + return yield* Effect.fail(new Error("Native LLM runtime does not support tools yet")) + } + const apiKey = + info?.type === "api" ? info.key : typeof item.options.apiKey === "string" ? item.options.apiKey : undefined + if (!apiKey) return yield* Effect.fail(new Error("Native LLM runtime requires API key auth for OpenAI")) + const baseURL = typeof item.options.baseURL === "string" ? item.options.baseURL : undefined + return { + type: "native" as const, + stream: LLMClient.stream( + LLMNative.request({ + model: input.model, + apiKey, + baseURL, + system: isOpenaiOauth ? system : [], + messages: ProviderTransform.message(messages, input.model, options), + toolChoice: input.toolChoice, + temperature: params.temperature, + topP: params.topP, + topK: params.topK, + maxOutputTokens: params.maxOutputTokens, + providerOptions: ProviderTransform.providerOptions(input.model, params.options), + headers: requestHeaders, + }), + ).pipe(Stream.provide(LLMClient.layer), Stream.provide(RequestExecutor.defaultLayer)), + } + } + + return { + type: "ai-sdk" as const, + result: streamText({ + onError(error) { + l.error("stream error", { + error, }) + }, + async experimental_repairToolCall(failed) { + const lower = failed.toolCall.toolName.toLowerCase() + if (lower !== failed.toolCall.toolName && sortedTools[lower]) { + l.info("repairing tool call", { + tool: failed.toolCall.toolName, + repaired: lower, + }) + return { + ...failed.toolCall, + toolName: lower, + } + } return { ...failed.toolCall, - toolName: lower, - } - } - return { - ...failed.toolCall, - input: JSON.stringify({ - tool: failed.toolCall.toolName, - error: failed.error.message, - }), - toolName: "invalid", - } - }, - temperature: params.temperature, - topP: params.topP, - topK: params.topK, - providerOptions: ProviderTransform.providerOptions(input.model, params.options), - activeTools: Object.keys(sortedTools).filter((x) => x !== "invalid"), - tools: sortedTools, - toolChoice: input.toolChoice, - maxOutputTokens: params.maxOutputTokens, - abortSignal: input.abort, - headers: { - ...(input.model.providerID.startsWith("opencode") - ? { - "x-opencode-project": opencodeProjectID, - "x-opencode-session": input.sessionID, - "x-opencode-request": input.user.id, - "x-opencode-client": Flag.OPENCODE_CLIENT, - "User-Agent": `opencode/${InstallationVersion}`, - } - : { - "x-session-affinity": input.sessionID, - ...(input.parentSessionID ? { "x-parent-session-id": input.parentSessionID } : {}), - "User-Agent": `opencode/${InstallationVersion}`, + input: JSON.stringify({ + tool: failed.toolCall.toolName, + error: failed.error.message, }), - ...input.model.headers, - ...headers, - }, - maxRetries: input.retries ?? 0, - messages, - model: wrapLanguageModel({ - model: language, - middleware: [ - { - specificationVersion: "v3" as const, - async transformParams(args) { - if (args.type === "stream") { - // @ts-expect-error - args.params.prompt = ProviderTransform.message(args.params.prompt, input.model, options) - } - return args.params - }, - }, - ], - }), - experimental_telemetry: { - isEnabled: cfg.experimental?.openTelemetry, - functionId: "session.llm", - tracer: telemetryTracer, - metadata: { - userId: cfg.username ?? "unknown", - sessionId: input.sessionID, + toolName: "invalid", + } }, - }, - }) + temperature: params.temperature, + topP: params.topP, + topK: params.topK, + providerOptions: ProviderTransform.providerOptions(input.model, params.options), + activeTools: Object.keys(sortedTools).filter((x) => x !== "invalid"), + tools: sortedTools, + toolChoice: input.toolChoice, + maxOutputTokens: params.maxOutputTokens, + abortSignal: input.abort, + headers: requestHeaders, + maxRetries: input.retries ?? 0, + messages, + model: wrapLanguageModel({ + model: language, + middleware: [ + { + specificationVersion: "v3" as const, + async transformParams(args) { + if (args.type === "stream") { + // @ts-expect-error + args.params.prompt = ProviderTransform.message(args.params.prompt, input.model, options) + } + return args.params + }, + }, + ], + }), + experimental_telemetry: { + isEnabled: cfg.experimental?.openTelemetry, + functionId: "session.llm", + tracer: telemetryTracer, + metadata: { + userId: cfg.username ?? "unknown", + sessionId: input.sessionID, + }, + }, + }), + } }) const stream: Interface["stream"] = (input) => @@ -426,8 +466,12 @@ const live: Layer.Layer< const result = yield* run({ ...input, abort: ctrl.signal }) + if (result.type === "native") return result.stream + const state = LLMAISDK.adapterState() - return Stream.fromAsyncIterable(result.fullStream, (e) => (e instanceof Error ? e : new Error(String(e)))).pipe( + return Stream.fromAsyncIterable(result.result.fullStream, (e) => + e instanceof Error ? e : new Error(String(e)), + ).pipe( Stream.mapEffect((event) => LLMAISDK.toLLMEvents(state, event)), Stream.flatMap((events) => Stream.fromIterable(events)), ) diff --git a/packages/opencode/test/session/llm.test.ts b/packages/opencode/test/session/llm.test.ts index f078e352ac..16a693dfa0 100644 --- a/packages/opencode/test/session/llm.test.ts +++ b/packages/opencode/test/session/llm.test.ts @@ -5,7 +5,6 @@ import { Cause, Effect, Exit, Stream } from "effect" import z from "zod" import { makeRuntime } from "../../src/effect/run-service" import { LLM } from "../../src/session/llm" -import { Instance } from "../../src/project/instance" import { WithInstance } from "../../src/project/with-instance" import { Provider } from "@/provider/provider" import { ProviderTransform } from "@/provider/transform" @@ -688,6 +687,121 @@ describe("session.llm.stream", () => { }) }) + test("streams OpenAI through native runtime when opted in", async () => { + const server = state.server + if (!server) { + throw new Error("Server not initialized") + } + + const source = await loadFixture("openai", "gpt-5.2") + const model = source.model + const chunks = [ + { + type: "response.created", + response: { + id: "resp-native", + }, + }, + { + type: "response.output_item.added", + item: { type: "message", id: "item-native", status: "in_progress" }, + }, + { + type: "response.output_text.delta", + item_id: "item-native", + delta: "Hello native", + }, + { + type: "response.completed", + response: { + incomplete_details: null, + usage: { + input_tokens: 1, + input_tokens_details: null, + output_tokens: 1, + output_tokens_details: null, + }, + }, + }, + ] + const request = waitRequest("/responses", createEventResponse(chunks, true)) + + await using tmp = await tmpdir({ + init: async (dir) => { + await Bun.write( + path.join(dir, "opencode.json"), + JSON.stringify({ + $schema: "https://opencode.ai/config.json", + enabled_providers: ["openai"], + provider: { + openai: { + name: "OpenAI", + env: ["OPENAI_API_KEY"], + npm: "@ai-sdk/openai", + api: "https://api.openai.com/v1", + models: { + [model.id]: model, + }, + options: { + apiKey: "test-openai-key", + baseURL: `${server.url.origin}/v1`, + }, + }, + }, + }), + ) + }, + }) + + await WithInstance.provide({ + directory: tmp.path, + fn: async () => { + const previous = process.env.OPENCODE_LLM_RUNTIME + process.env.OPENCODE_LLM_RUNTIME = "native" + try { + const resolved = await getModel(ProviderID.openai, ModelID.make(model.id)) + const sessionID = SessionID.make("session-test-native") + const agent = { + name: "test", + mode: "primary", + options: {}, + permission: [{ permission: "*", pattern: "*", action: "allow" }], + temperature: 0.2, + } satisfies Agent.Info + + await drain({ + user: { + id: MessageID.make("msg_user-native"), + sessionID, + role: "user", + time: { created: Date.now() }, + agent: agent.name, + model: { providerID: ProviderID.make("openai"), modelID: resolved.id, variant: "high" }, + } satisfies MessageV2.User, + sessionID, + model: resolved, + agent, + system: ["You are a helpful assistant."], + messages: [{ role: "user", content: "Hello" }], + tools: {}, + }) + } finally { + if (previous === undefined) delete process.env.OPENCODE_LLM_RUNTIME + else process.env.OPENCODE_LLM_RUNTIME = previous + } + + const capture = await request + expect(capture.url.pathname.endsWith("/responses")).toBe(true) + expect(capture.headers.get("Authorization")).toBe("Bearer test-openai-key") + expect(capture.body.model).toBe(model.id) + expect(capture.body.stream).toBe(true) + expect((capture.body.reasoning as { effort?: string } | undefined)?.effort).toBe("high") + expect(JSON.stringify(capture.body.input)).toContain("You are a helpful assistant.") + expect(capture.body.input).toContainEqual({ role: "user", content: [{ type: "input_text", text: "Hello" }] }) + }, + }) + }) + test("accepts user image attachments as data URLs for OpenAI models", async () => { const server = state.server if (!server) {