fix(core): bound web tool failures (#33259)

packages/core/src/tool/http-body.ts

@@ -0,0 +1,30 @@
+import { Effect, Stream } from "effect"
+import { HttpClientResponse } from "effect/unstable/http"
+
+export const collectBoundedResponseBody = (
+  response: HttpClientResponse.HttpClientResponse,
+  maximumBytes: number,
+  tooLarge: () => Error,
+) =>
+  Effect.gen(function* () {
+    const contentLength = response.headers["content-length"]
+    const parsedSize = contentLength ? Number.parseInt(contentLength, 10) : undefined
+    const declaredSize =
+      parsedSize !== undefined && Number.isSafeInteger(parsedSize) && parsedSize >= 0 ? parsedSize : undefined
+    if (declaredSize !== undefined && declaredSize > maximumBytes) return yield* Effect.fail(tooLarge())
+    let body = Buffer.allocUnsafe(Math.min(maximumBytes, declaredSize || 64 * 1024))
+    let size = 0
+    yield* Stream.runForEach(response.stream, (chunk) => {
+      if (chunk.byteLength === 0) return Effect.void
+      if (size + chunk.byteLength > maximumBytes) return Effect.fail(tooLarge())
+      if (size + chunk.byteLength > body.byteLength) {
+        const grown = Buffer.allocUnsafe(Math.min(maximumBytes, Math.max(size + chunk.byteLength, body.byteLength * 2)))
+        body.copy(grown, 0, 0, size)
+        body = grown
+      }
+      body.set(chunk, size)
+      size += chunk.byteLength
+      return Effect.void
+    })
+    return body.subarray(0, size)
+  })

packages/core/src/tool/webfetch.ts

@@ -1,11 +1,12 @@
 export * as WebFetchTool from "./webfetch"
 
 import { ToolFailure } from "@opencode-ai/llm"
-import { Duration, Effect, Layer, Schema, Stream } from "effect"
+import { Duration, Effect, Layer, Schema } from "effect"
 import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
 import { Parser } from "htmlparser2"
 import TurndownService from "turndown"
 import { PermissionV2 } from "../permission"
+import { collectBoundedResponseBody } from "./http-body"
 import { Tool } from "./tool"
 import { Tools } from "./tools"
 
@@ -86,24 +87,11 @@ const execute = (http: HttpClient.HttpClient, url: string, format: Format, userA
   http.execute(request(url, format, userAgent)).pipe(Effect.flatMap(HttpClientResponse.filterStatusOk))
 
 const collectBody = (response: HttpClientResponse.HttpClientResponse) =>
-  Effect.gen(function* () {
-    const contentLength = response.headers["content-length"]
-    if (contentLength && Number.parseInt(contentLength, 10) > MAX_RESPONSE_BYTES) {
-      return yield* Effect.fail(new Error(`Response too large (exceeds ${MAX_RESPONSE_BYTES} byte limit)`))
-    }
-    const chunks: Uint8Array[] = []
-    let size = 0
-    yield* Stream.runForEach(response.stream, (chunk) =>
-      Effect.gen(function* () {
-        size += chunk.byteLength
-        if (size > MAX_RESPONSE_BYTES)
-          return yield* Effect.fail(new Error(`Response too large (exceeds ${MAX_RESPONSE_BYTES} byte limit)`))
-        chunks.push(chunk)
-        return undefined
-      }),
-    )
-    return Buffer.concat(chunks, size)
-  })
+  collectBoundedResponseBody(
+    response,
+    MAX_RESPONSE_BYTES,
+    () => new Error(`Response too large (exceeds ${MAX_RESPONSE_BYTES} byte limit)`),
+  )
 
 const mimeFrom = (contentType: string) => contentType.split(";", 1)[0]?.trim().toLowerCase() ?? ""
 const isImageAttachment = (mime: string) =>
@@ -171,12 +159,16 @@ export const layer = Layer.effectDiscard(
                   orElse: () => Effect.fail(new Error("Request timed out")),
                 }),
               )
-              const content = convert(new TextDecoder().decode(body), contentType, input.format)
+              const content = new TextDecoder().decode(body)
+              const output = yield* Effect.try({
+                try: () => convert(content, contentType, input.format),
+                catch: (error) => error,
+              })
               return {
                 url: input.url,
                 contentType,
                 format: input.format,
-                output: content,
+                output,
               }
             }).pipe(Effect.mapError(() => new ToolFailure({ message: `Unable to fetch ${input.url}` }))),
         }),

packages/core/src/tool/websearch.ts

@@ -9,6 +9,7 @@ import { PositiveInt } from "../schema"
 import { PermissionV2 } from "../permission"
 import { Tool } from "./tool"
 import { Tools } from "./tools"
+import { collectBoundedResponseBody } from "./http-body"
 import { checksum } from "../util/encode"
 
 export const name = "websearch"
@@ -164,10 +165,12 @@ const callMcp = <F extends Schema.Struct.Fields>(
     )
     return yield* Effect.gen(function* () {
       const response = yield* HttpClient.filterStatusOk(http).execute(request)
-      const body = yield* response.text
-      if (Buffer.byteLength(body, "utf8") > MAX_RESPONSE_BYTES)
-        return yield* Effect.fail(new Error(`${tool} response exceeded ${MAX_RESPONSE_BYTES} bytes`))
-      return yield* parseResponse(body)
+      const body = yield* collectBoundedResponseBody(
+        response,
+        MAX_RESPONSE_BYTES,
+        () => new Error(`${tool} response exceeded ${MAX_RESPONSE_BYTES} bytes`),
+      )
+      return yield* parseResponse(body.toString("utf8"))
     }).pipe(
       Effect.timeoutOrElse({
         duration: Duration.seconds(25),