From 08c5a2a5e8c565da9796261f5beaf8887b31c7a6 Mon Sep 17 00:00:00 2001 From: Shoubhit Dash Date: Mon, 29 Jun 2026 19:24:37 +0530 Subject: [PATCH] feat(llm): enforce request precedence (#34440) --- .../llm/src/protocols/anthropic-messages.ts | 3 +- packages/llm/src/route/client.ts | 15 +- packages/llm/test/prepare.test.ts | 162 ++++++++++++++++++ 3 files changed, 173 insertions(+), 7 deletions(-) create mode 100644 packages/llm/test/prepare.test.ts diff --git a/packages/llm/src/protocols/anthropic-messages.ts b/packages/llm/src/protocols/anthropic-messages.ts index a37cd2c9a7..d51da9851d 100644 --- a/packages/llm/src/protocols/anthropic-messages.ts +++ b/packages/llm/src/protocols/anthropic-messages.ts @@ -504,6 +504,7 @@ const lowerThinking = Effect.fn("AnthropicMessages.lowerThinking")(function* (re const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (request: LLMRequest) { const toolChoice = request.toolChoice ? yield* lowerToolChoice(request.toolChoice) : undefined const generation = request.generation + const outputLimit = request.model.defaults?.limits?.output ?? request.model.route.defaults.limits?.output ?? 4096 // Allocate the 4-breakpoint budget in invalidation order: tools → system → // messages. Tools live highest in the cache hierarchy, so when callers // over-mark we keep their tool hints and shed the message-tail ones first. @@ -533,7 +534,7 @@ const fromRequest = Effect.fn("AnthropicMessages.fromRequest")(function* (reques tools, tool_choice: toolChoice, stream: true as const, - max_tokens: generation?.maxTokens ?? request.model.route.defaults.limits?.output ?? 4096, + max_tokens: generation?.maxTokens ?? outputLimit, temperature: generation?.temperature, top_p: generation?.topP, top_k: generation?.topK, diff --git a/packages/llm/src/route/client.ts b/packages/llm/src/route/client.ts index b261cd7f23..183f171dcd 100644 --- a/packages/llm/src/route/client.ts +++ b/packages/llm/src/route/client.ts @@ -164,13 +164,16 @@ export interface GenerateMethod { export class Service extends Context.Service()("@opencode/LLMClient") {} -const resolveRequestOptions = (request: LLMRequest) => - LLMRequest.update(request, { - generation: - mergeGenerationOptions(request.model.route.defaults.generation, request.generation) ?? new GenerationOptions({}), - providerOptions: mergeProviderOptions(request.model.route.defaults.providerOptions, request.providerOptions), - http: mergeHttpOptions(request.model.route.defaults.http, request.http), +const resolveRequestOptions = (request: LLMRequest) => { + const routeDefaults = request.model.route.defaults + const modelDefaults = request.model.defaults + const generation = mergeGenerationOptions(routeDefaults.generation, modelDefaults?.generation, request.generation) + return LLMRequest.update(request, { + generation: generation ?? new GenerationOptions({}), + providerOptions: mergeProviderOptions(routeDefaults.providerOptions, modelDefaults?.providerOptions, request.providerOptions), + http: mergeHttpOptions(routeDefaults.http, modelDefaults?.http, request.http), }) +} export interface MakeInput { /** Route id used in diagnostics and prepared request metadata. */ diff --git a/packages/llm/test/prepare.test.ts b/packages/llm/test/prepare.test.ts new file mode 100644 index 0000000000..d6006095f4 --- /dev/null +++ b/packages/llm/test/prepare.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, test } from "bun:test" +import { Effect, Schema } from "effect" +import { HttpClientRequest } from "effect/unstable/http" +import { LLM, mergeProviderOptions } from "../src" +import { AnthropicMessages, OpenAIChat } from "../src/protocols" +import { Auth, LLMClient } from "../src/route" +import { it } from "./lib/effect" +import { dynamicResponse } from "./lib/http" +import { deltaChunk } from "./lib/openai-chunks" +import { sseEvents } from "./lib/sse" + +const TargetJson = Schema.fromJsonString(Schema.Unknown) +const decodeJson = Schema.decodeUnknownSync(TargetJson) + +describe("request option precedence", () => { + test("deep-merges provider option records and replaces arrays, primitives, and null", () => { + const merged = mergeProviderOptions( + { + openai: { + include: ["route"], + metadata: { route: true, shared: "route" }, + nullable: "route", + primitive: "route", + }, + }, + { + openai: { + include: ["model"], + metadata: { model: true, shared: "model" }, + nullable: null, + primitive: "model", + }, + }, + { openai: { metadata: { request: true }, primitive: false } }, + ) + + expect(merged).toEqual({ + openai: { + include: ["model"], + metadata: { route: true, model: true, request: true, shared: "model" }, + nullable: null, + primitive: false, + }, + }) + }) + + it.effect("prepares bodies with route defaults, model defaults, and call options in order", () => + Effect.gen(function* () { + const route = OpenAIChat.route.with({ + endpoint: { baseURL: "https://api.openai.test/v1/" }, + auth: Auth.bearer("test"), + generation: { maxTokens: 10, temperature: 1, stop: ["route"] }, + providerOptions: { openai: { store: false, reasoningEffort: "low" } }, + }) + const model = route.model({ + id: "gpt-4o-mini", + defaults: { + generation: { maxTokens: 20, temperature: 0.5, frequencyPenalty: 0.25, stop: ["model"] }, + providerOptions: { openai: { reasoningEffort: "medium" } }, + }, + }) + const prepared = yield* LLMClient.prepare( + LLM.request({ + model, + prompt: "Say hello.", + generation: { maxTokens: 30, topP: 0.9, stop: ["request"] }, + providerOptions: { openai: { store: true } }, + }), + ) + + expect(prepared.body).toMatchObject({ + model: "gpt-4o-mini", + stream: true, + max_tokens: 30, + temperature: 0.5, + top_p: 0.9, + frequency_penalty: 0.25, + store: true, + reasoning_effort: "medium", + }) + expect(prepared.body.stop).toEqual(["request"]) + }), + ) + + it.effect("applies model HTTP defaults before request HTTP overlays", () => + LLMClient.generate( + LLM.request({ + model: OpenAIChat.route + .with({ + endpoint: { baseURL: "https://api.openai.test/v1/" }, + auth: Auth.bearer("fresh-key"), + http: { + body: { metadata: { route: true, shared: "route" }, value: "route" }, + headers: { "x-route": "route", "x-shared": "route" }, + query: { route: "1", shared: "route" }, + }, + }) + .model({ + id: "gpt-4o-mini", + defaults: { + http: { + body: { metadata: { model: true, shared: "model" }, value: "model" }, + headers: { "x-model": "model", "x-shared": "model" }, + query: { model: "1", shared: "model" }, + }, + }, + }), + prompt: "Say hello.", + http: { + body: { metadata: { request: true }, value: null }, + headers: { "x-request": "request" }, + query: { request: "1" }, + }, + }), + ).pipe( + Effect.provide( + dynamicResponse((input) => + Effect.gen(function* () { + const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie) + const url = new URL(web.url) + expect(url.searchParams.get("route")).toBe("1") + expect(url.searchParams.get("model")).toBe("1") + expect(url.searchParams.get("request")).toBe("1") + expect(url.searchParams.get("shared")).toBe("model") + expect(web.headers.get("authorization")).toBe("Bearer fresh-key") + expect(web.headers.get("x-route")).toBe("route") + expect(web.headers.get("x-model")).toBe("model") + expect(web.headers.get("x-request")).toBe("request") + expect(web.headers.get("x-shared")).toBe("model") + expect(decodeJson(input.text)).toMatchObject({ + metadata: { route: true, model: true, request: true, shared: "model" }, + value: null, + }) + return input.respond(sseEvents(deltaChunk({}, "stop")), { + headers: { "content-type": "text/event-stream" }, + }) + }), + ), + ), + ), + ) + + it.effect("uses model output limits after route limits and before call maxTokens", () => + Effect.gen(function* () { + const route = AnthropicMessages.route.with({ + endpoint: { baseURL: "https://api.anthropic.test/v1/" }, + auth: Auth.header("x-api-key", "test"), + limits: { output: 128 }, + }) + const model = route.model({ id: "claude-sonnet-4-5", defaults: { limits: { output: 64 } } }) + const withoutMaxTokens = yield* LLMClient.prepare( + LLM.request({ model, prompt: "Say hello.", cache: "none" }), + ) + const withMaxTokens = yield* LLMClient.prepare( + LLM.request({ model, prompt: "Say hello.", cache: "none", generation: { maxTokens: 32 } }), + ) + + expect(withoutMaxTokens.body.max_tokens).toBe(64) + expect(withMaxTokens.body.max_tokens).toBe(32) + }), + ) +})