feat(http-recorder): sync recorder v0.3 (#35619)

packages/http-recorder/src/http/matching.ts

@@ -0,0 +1,77 @@
+import { HashSet, Option } from "effect"
+import type { RequestMatcher, RequestSnapshot } from "../api.js"
+import { canonicalizeJson, decodeJson, isJsonRecord, jsonBody, safeText } from "../replay/comparison.js"
+import type { HttpInteraction } from "./model.js"
+
+export type { RequestMatcher } from "../api.js"
+
+export const canonicalSnapshot = (snapshot: RequestSnapshot): string =>
+  JSON.stringify({
+    method: snapshot.method,
+    url: snapshot.url,
+    headers: canonicalizeJson(snapshot.headers),
+    body: Option.match(decodeJson(snapshot.body), { onNone: () => snapshot.body, onSome: canonicalizeJson }),
+  })
+export const defaultMatcher: RequestMatcher = (incoming, recorded) =>
+  canonicalSnapshot(incoming) === canonicalSnapshot(recorded)
+
+const valueDiffs = (expected: unknown, received: unknown, base = "$", limit = 8): ReadonlyArray<string> => {
+  if (Object.is(expected, received)) return []
+  if (isJsonRecord(expected) && isJsonRecord(received))
+    return [...new Set([...Object.keys(expected), ...Object.keys(received)])]
+      .toSorted()
+      .flatMap((key) => valueDiffs(expected[key], received[key], `${base}.${key}`, limit))
+      .slice(0, limit)
+  if (Array.isArray(expected) && Array.isArray(received))
+    return Array.from({ length: Math.max(expected.length, received.length) }, (_, index) => index)
+      .flatMap((index) => valueDiffs(expected[index], received[index], `${base}[${index}]`, limit))
+      .slice(0, limit)
+  return [`${base} expected ${safeText(expected)}, received ${safeText(received)}`]
+}
+
+const headerDiffs = (expected: Record<string, string>, received: Record<string, string>) =>
+  [...new Set([...Object.keys(expected), ...Object.keys(received)])].toSorted().flatMap((key) => {
+    if (expected[key] === received[key]) return []
+    if (expected[key] === undefined) return [`  ${key} unexpected ${safeText(received[key])}`]
+    if (received[key] === undefined) return [`  ${key} missing expected ${safeText(expected[key])}`]
+    return [`  ${key} expected ${safeText(expected[key])}, received ${safeText(received[key])}`]
+  })
+
+export const requestDiff = (expected: RequestSnapshot, received: RequestSnapshot): ReadonlyArray<string> => {
+  const lines: string[] = []
+  if (expected.method !== received.method)
+    lines.push("method:", `  expected ${expected.method}, received ${received.method}`)
+  if (expected.url !== received.url) lines.push("url:", `  expected ${expected.url}`, `  received ${received.url}`)
+  const headers = headerDiffs(expected.headers, received.headers)
+  if (headers.length > 0) lines.push("headers:", ...headers.slice(0, 8))
+  const expectedBody = jsonBody(expected.body)
+  const receivedBody = jsonBody(received.body)
+  const body =
+    expectedBody !== undefined && receivedBody !== undefined
+      ? valueDiffs(expectedBody, receivedBody).map((line) => `  ${line}`)
+      : expected.body === received.body
+        ? []
+        : [`  expected ${safeText(expected.body)}, received ${safeText(received.body)}`]
+  if (body.length > 0) lines.push("body:", ...body)
+  return lines
+}
+
+export const selectFirstMatching = (
+  interactions: ReadonlyArray<HttpInteraction>,
+  incoming: RequestSnapshot,
+  match: RequestMatcher,
+  used: HashSet.HashSet<number>,
+): { readonly _tag: "Matched"; readonly index: number } | { readonly _tag: "Unmatched"; readonly detail: string } => {
+  let firstUnused: HttpInteraction | undefined
+  for (let index = 0; index < interactions.length; index++) {
+    if (HashSet.has(used, index)) continue
+    const interaction = interactions[index]
+    firstUnused ??= interaction
+    if (match(incoming, interaction.request)) return { _tag: "Matched", index }
+  }
+  if (firstUnused === undefined)
+    return { _tag: "Unmatched", detail: `all ${interactions.length} recorded interactions have already been consumed` }
+  return { _tag: "Unmatched", detail: requestDiff(firstUnused.request, incoming).join("\n") }
+}
+
+export * as HttpMatching from "./matching.js"

packages/http-recorder/src/http/model.ts

@@ -0,0 +1,30 @@
+import { Schema } from "effect"
+import type { RequestSnapshot } from "../api.js"
+
+export const RequestSnapshotSchema = Schema.Struct({
+  method: Schema.String,
+  url: Schema.String,
+  headers: Schema.Record(Schema.String, Schema.String),
+  body: Schema.String,
+})
+
+export type { RequestSnapshot } from "../api.js"
+
+export const ResponseSnapshotSchema = Schema.Struct({
+  status: Schema.Number,
+  headers: Schema.Record(Schema.String, Schema.String),
+  body: Schema.String,
+  bodyEncoding: Schema.optional(Schema.Literals(["text", "base64"])),
+})
+
+export interface ResponseSnapshot extends Schema.Schema.Type<typeof ResponseSnapshotSchema> {}
+
+export const HttpInteractionSchema = Schema.Struct({
+  transport: Schema.tag("http"),
+  request: RequestSnapshotSchema,
+  response: ResponseSnapshotSchema,
+})
+
+export interface HttpInteraction extends Schema.Schema.Type<typeof HttpInteractionSchema> {}
+
+export * as HttpModel from "./model.js"

packages/http-recorder/src/http/recorder.ts

@@ -0,0 +1,176 @@
+import { NodeFileSystem } from "@effect/platform-node-shared"
+import { Deferred, Effect, Layer, Ref } from "effect"
+import {
+  FetchHttpClient,
+  HttpClient,
+  HttpClientError,
+  HttpClientRequest,
+  HttpClientResponse,
+} from "effect/unstable/http"
+import { fileSystem, Service } from "../cassette/store.js"
+import type { RecorderOptions } from "../options.js"
+import { make, redactUrl, type Redactor } from "../redaction/redactor.js"
+import { makeReplayPoolState, resolveAutoMode } from "../replay/state.js"
+import { httpInteractions, type CassetteMetadata } from "../cassette/model.js"
+import { defaultMatcher, selectFirstMatching, type RequestMatcher } from "./matching.js"
+import type { HttpInteraction, ResponseSnapshot } from "./model.js"
+
+export { defaultMatcher }
+export type RecordReplayMode = "auto" | "record" | "replay" | "passthrough"
+export interface RecordReplayOptions {
+  readonly mode?: RecordReplayMode
+  readonly directory?: string
+  readonly metadata?: CassetteMetadata
+  readonly redactor?: Redactor
+  readonly match?: RequestMatcher
+}
+
+const TEXT_CONTENT_TYPES = new Set([
+  "application/graphql",
+  "application/javascript",
+  "application/json",
+  "application/sql",
+  "application/x-www-form-urlencoded",
+  "application/xml",
+  "application/yaml",
+  "image/svg+xml",
+])
+const isTextContentType = (contentType: string | undefined) => {
+  const mediaType = contentType?.split(";", 1)[0]?.trim().toLowerCase()
+  if (!mediaType) return false
+  return (
+    mediaType.startsWith("text/") ||
+    mediaType.endsWith("+json") ||
+    mediaType.endsWith("+xml") ||
+    TEXT_CONTENT_TYPES.has(mediaType)
+  )
+}
+const captureResponseBody = (response: HttpClientResponse.HttpClientResponse, contentType: string | undefined) =>
+  response.arrayBuffer.pipe(
+    Effect.map((bytes) =>
+      isTextContentType(contentType)
+        ? { body: new TextDecoder().decode(bytes) }
+        : { body: Buffer.from(bytes).toString("base64"), bodyEncoding: "base64" as const },
+    ),
+  )
+const decodeResponseBody = (snapshot: ResponseSnapshot) =>
+  snapshot.bodyEncoding === "base64" ? Buffer.from(snapshot.body, "base64") : snapshot.body
+const responseFromSnapshot = (request: HttpClientRequest.HttpClientRequest, snapshot: ResponseSnapshot) =>
+  HttpClientResponse.fromWeb(
+    request,
+    new Response(
+      request.method === "HEAD" || snapshot.status === 204 || snapshot.status === 205 || snapshot.status === 304
+        ? null
+        : decodeResponseBody(snapshot),
+      snapshot,
+    ),
+  )
+
+export const redactedErrorRequest = (
+  request: HttpClientRequest.HttpClientRequest,
+  redactedUrl = redactUrl(request.url),
+) => HttpClientRequest.make(request.method)(redactedUrl)
+const transportError = (request: HttpClientRequest.HttpClientRequest, description: string, redactedUrl?: string) =>
+  new HttpClientError.HttpClientError({
+    reason: new HttpClientError.TransportError({ request: redactedErrorRequest(request, redactedUrl), description }),
+  })
+
+export const recordingLayer = (
+  name: string,
+  options: Omit<RecordReplayOptions, "directory"> = {},
+): Layer.Layer<HttpClient.HttpClient, never, HttpClient.HttpClient | Service> =>
+  Layer.effect(
+    HttpClient.HttpClient,
+    Effect.gen(function* () {
+      const upstream = yield* HttpClient.HttpClient
+      const cassette = yield* Service
+      const redactor = options.redactor ?? make()
+      const match = options.match ?? defaultMatcher
+      const requested = options.mode ?? "auto"
+      const mode = requested === "auto" ? yield* resolveAutoMode(cassette, name) : requested
+      const snapshotRequest = (request: HttpClientRequest.HttpClientRequest) =>
+        Effect.gen(function* () {
+          const web = yield* HttpClientRequest.toWeb(request).pipe(Effect.orDie)
+          return redactor.request({
+            method: web.method,
+            url: web.url,
+            headers: Object.fromEntries(web.headers.entries()),
+            body: yield* Effect.promise(() => web.text()),
+          })
+        })
+      if (mode === "passthrough") return upstream
+      if (mode === "record") {
+        const initial = yield* Deferred.make<void>()
+        yield* Deferred.succeed(initial, undefined)
+        const tail = yield* Ref.make(initial)
+        return HttpClient.make((request) =>
+          Effect.gen(function* () {
+            const completed = yield* Deferred.make<void>()
+            const previous = yield* Ref.modify(tail, (current) => [current, completed])
+            return yield* Effect.gen(function* () {
+              const incoming = yield* snapshotRequest(request)
+              const requestError = (description: string) => transportError(request, description, incoming.url)
+              const response = yield* upstream.execute(request)
+              const captured = yield* captureResponseBody(response, response.headers["content-type"])
+              const responseSnapshot: ResponseSnapshot = {
+                status: response.status,
+                headers: response.headers as Record<string, string>,
+                ...captured,
+              }
+              const interaction: HttpInteraction = {
+                transport: "http",
+                request: incoming,
+                response: redactor.response(responseSnapshot),
+              }
+              yield* Deferred.await(previous)
+              yield* cassette
+                .append(name, interaction, options.metadata)
+                .pipe(Effect.catchTag("UnsafeCassetteError", (error) => Effect.fail(requestError(error.message))))
+              return responseFromSnapshot(request, responseSnapshot)
+            }).pipe(Effect.ensuring(Deferred.succeed(completed, undefined)))
+          }),
+        )
+      }
+      const replay = yield* makeReplayPoolState(cassette, name, httpInteractions)
+      return HttpClient.make((request) =>
+        Effect.gen(function* () {
+          const incoming = yield* snapshotRequest(request)
+          const requestError = (description: string) => transportError(request, description, incoming.url)
+          const claimed = yield* replay
+            .claim((interactions, used) => {
+              const result = selectFirstMatching(interactions, incoming, match, used)
+              if (result._tag === "Matched") return Effect.succeed(result.index)
+              return Effect.fail(
+                requestError(`Fixture "${name}" does not match the current request: ${result.detail}.`),
+              )
+            })
+            .pipe(
+              Effect.mapError((error) =>
+                error._tag === "CassetteNotFoundError"
+                  ? requestError(`Fixture "${name}" not found. Run locally to record it (CI=true forces replay).`)
+                  : requestError(error.message),
+              ),
+            )
+          return responseFromSnapshot(request, claimed.interaction.response)
+        }),
+      )
+    }),
+  )
+
+export const cassetteLayer = (name: string, options: RecordReplayOptions = {}): Layer.Layer<HttpClient.HttpClient> =>
+  recordingLayer(name, options).pipe(
+    Layer.provide(fileSystem({ directory: options.directory })),
+    Layer.provide(FetchHttpClient.layer),
+    Layer.provide(NodeFileSystem.layer),
+  )
+
+export const layer = (
+  name: string,
+  options: RecorderOptions = {},
+): Layer.Layer<HttpClient.HttpClient, never, HttpClient.HttpClient> =>
+  recordingLayer(name, { metadata: options.metadata, redactor: make(options.redact), match: options.match }).pipe(
+    Layer.provide(fileSystem({ directory: options.directory })),
+    Layer.provide(NodeFileSystem.layer),
+  )
+export const layerFetch = (name: string, options: RecorderOptions = {}): Layer.Layer<HttpClient.HttpClient> =>
+  layer(name, options).pipe(Layer.provide(FetchHttpClient.layer))