fix(core): compact before inference body limit

packages/core/src/session/compaction.ts

@@ -19,6 +19,7 @@ const DEFAULT_KEEP_TOKENS = 8_000
 const TOOL_OUTPUT_MAX_CHARS = 2_000
 const COMPACTION_CHUNK_TOKENS = 32_000
 const SUMMARY_OUTPUT_TOKENS = 4_096
+const REQUEST_BODY_COMPACTION_BYTES = 8 * 1024 * 1024
 const SUMMARY_TEMPLATE = `Output exactly the Markdown structure shown inside <template> and keep the section order unchanged. Do not include the <template> tags in your response.
 <template>
 ## Objective
@@ -70,6 +71,7 @@ export type AutoInput = {
   readonly sessionID: SessionSchema.ID
   readonly messages: readonly SessionMessage.Info[]
   readonly model: Model
+  readonly requestBytes: number
 }
 
 export type ManualInput = {
@@ -327,6 +329,7 @@ const make = (dependencies: Dependencies) => {
   })
   const required = (input: AutoInput) => {
     if (!config.auto) return false
+    if (input.requestBytes >= REQUEST_BODY_COMPACTION_BYTES) return true
     const context = input.model.route.defaults.limits?.context
     if (context === undefined || context <= 0) return false
     const last = input.messages.findLast(

packages/core/src/session/runner/llm.ts

@@ -186,12 +186,6 @@ const layer = Layer.effect(
       const providerMetadataKey = model.route.providerMetadataKey ?? model.provider
       const history = yield* SessionHistory.entriesForRunner(db, session.id, instructions)
       const context = history.entries.map((entry) => entry.message)
-      const compactionInput = { sessionID: session.id, messages: context, model }
-      if (compaction.required(compactionInput) && !(yield* SessionPending.compaction(db, session.id))) {
-        const compacted = yield* compaction.compact(compactionInput)
-        if (compacted.status === "completed") return { _tag: "RestartAfterCompaction", step: currentStep } as const
-        return yield* new StepFailedError({ error: compacted.error })
-      }
       const isLastStep = agentInfo.steps !== undefined && currentStep >= agentInfo.steps
       const toolMaterialization = isLastStep ? undefined : yield* tools.materialize(agentInfo.permissions)
       const promptCacheKey = /^ses_[0-9a-f]{64}$/.test(session.id) ? session.id.slice(4) : session.id
@@ -208,6 +202,19 @@ const layer = Layer.effect(
         tools: toolMaterialization?.definitions ?? [],
         toolChoice: isLastStep ? "none" : undefined,
       })
+      const compactionInput = {
+        sessionID: session.id,
+        messages: context,
+        model,
+        requestBytes: new TextEncoder().encode(
+          JSON.stringify({ system: request.system, messages: request.messages, tools: request.tools }),
+        ).length,
+      }
+      if (compaction.required(compactionInput) && !(yield* SessionPending.compaction(db, session.id))) {
+        const compacted = yield* compaction.compact(compactionInput)
+        if (compacted.status === "completed") return { _tag: "RestartAfterCompaction", step: currentStep } as const
+        return yield* new StepFailedError({ error: compacted.error })
+      }
       const toolFibers = yield* FiberSet.make<void, ToolOutputStore.Error>()
       const ownedToolFibers: Array<Fiber.Fiber<void, ToolOutputStore.Error>> = []
       let needsContinuation = false
@@ -323,8 +330,7 @@ const layer = Layer.effect(
             recoverOverflow &&
             !publisher.hasRetryEvidence() &&
             isContextOverflowFailure(overflowFailure ?? streamFailure) &&
-            (yield* restore(recoverOverflow({ sessionID: session.id, messages: context, model }))).status ===
-              "completed"
+            (yield* restore(recoverOverflow(compactionInput))).status === "completed"
           )
             return { _tag: "RestartAfterOverflowCompaction", step: currentStep } as const
 

packages/core/test/session-compaction.test.ts

@@ -96,6 +96,20 @@ test("compaction prompt requires the checkpoint headings in order", () => {
   expect(prompt).toContain("Keep every section, even when empty.")
 })
 
+it.effect("requires compaction before a request reaches the inference body limit", () =>
+  Effect.gen(function* () {
+    const compaction = yield* SessionCompaction.Service
+    expect(
+      compaction.required({
+        sessionID: SessionV2.ID.make("ses_request_body_compaction"),
+        messages: [],
+        model,
+        requestBytes: 8 * 1024 * 1024,
+      }),
+    ).toBe(true)
+  }),
+)
+
 it.effect("manual compaction summarizes short context instead of no-op", () =>
   Effect.gen(function* () {
     requests = []