fix(data): oom crash in stats sync

infra/stats.ts

@@ -185,7 +185,9 @@ export const statSync = new sst.aws.Service("StatsSyncService", {
   cluster: lakeCluster,
   architecture: "arm64",
   cpu: "0.25 vCPU",
-  memory: "0.5 GB",
+  // 0.5 GB caused an OOM crash loop: every restart immediately re-ran the 4 Athena
+  // stats queries (~$5/pass) every ~5 minutes instead of hourly.
+  memory: "2 GB",
   image: {
     context: ".",
     dockerfile: "packages/stats/server/Dockerfile",

packages/stats/core/src/athena.ts

@@ -116,12 +116,15 @@ const poll: (
 const results: (
   client: AwsAthenaClient,
   queryExecutionId: string,
-  nextToken?: string,
 ) => Effect.Effect<AthenaData[], AthenaQueryError> = Effect.fn("Athena.results")(function* (
   client: AwsAthenaClient,
   queryExecutionId: string,
-  nextToken?: string,
 ) {
+  // Accumulate pages iteratively; recursive spreads copied every previously
+  // fetched row per page and blew up memory on large result sets.
+  const rows: AthenaData[] = []
+  let nextToken: string | undefined
+  while (true) {
     const result = yield* Effect.tryPromise({
       try: () =>
         client.send(
@@ -131,13 +134,15 @@ const results: (
             MaxResults: ATHENA_PAGE_SIZE,
           }),
         ),
-    catch: (cause) => new AthenaQueryError({ message: "Failed to read Athena stats results", queryExecutionId, cause }),
+      catch: (cause) =>
+        new AthenaQueryError({ message: "Failed to read Athena stats results", queryExecutionId, cause }),
     })
     const columns = result.ResultSet?.ResultSetMetadata?.ColumnInfo?.map((item) => item.Name ?? "") ?? []
-  const rows = (result.ResultSet?.Rows ?? []).slice(nextToken ? 0 : 1).map((row) => rowData(columns, row))
+    // The first page starts with the header row.
-
+    for (const row of (result.ResultSet?.Rows ?? []).slice(nextToken ? 0 : 1)) rows.push(rowData(columns, row))
     if (!result.NextToken) return rows
-  return [...rows, ...(yield* results(client, queryExecutionId, result.NextToken))]
+    nextToken = result.NextToken
+  }
 })
 
 function rowData(columns: string[], row: Row): AthenaData {