ceos/opencode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(data): cut athena stats sync cost

Browse source 
Compute all stat dimensions and both grains in one GROUPING SETS query
so each sync pass scans the source table once instead of eight times.
Hourly passes now only recompute the current ISO week; a daily full pass
refreshes the whole display window. Restarted daemons resume the hourly
cadence from the last completed sync instead of immediately re-running a
pass, and the workgroup kills any query scanning more than 2 TB.
This commit is contained in:
Adam 2026-07-10 06:36:17 -05:00
commit 08a2a623dc
No known key found for this signature in database
GPG key ID: 9CB48779AF150E75
5 changed files with 122 additions and 77 deletions
Download patch file Download diff file Expand all files Collapse all files

67
packages/stats/core/src/stat-sync.ts
View file

@ -12,24 +12,22 @@ const DATALAKE_INGESTION_LAG_MS = 5 * 60_000
const STATS_DATA_START_MS = new Date("2026-05-28T00:00:00.000Z").getTime()
const WEEK_MS = 7 * 86_400_000
const DISPLAY_WINDOW_MS = 56 * 86_400_000
// Anchor incremental passes to the ISO week containing this lookback, so the pass
// after a week boundary still recomputes the previous week's final aggregates even
// if the boundary pass itself failed.
const INCREMENTAL_LOOKBACK_MS = 2 * 3_600_000
export type SyncStatsResult = { ok: true; rows: number; startedAt: string; periodStart: string; periodEnd: string }
export type SyncStatsError = AthenaQueryError | AthenaQueryTimeoutError | DatabaseError
export const syncStats: () => Effect.Effect<
export const syncStats: (options?: { full?: boolean }) => Effect.Effect<
SyncStatsResult,
SyncStatsError,
Athena | ModelStatRepo | ProviderStatRepo | GeoStatRepo
> = Effect.fn("StatSync.sync")(function* () {
> = Effect.fn("StatSync.sync")(function* (options?: { full?: boolean }) {
const startedAt = yield* DateTime.nowAsDate
const periodEnd = new Date(Math.floor((startedAt.getTime() - DATALAKE_INGESTION_LAG_MS) / 60_000) * 60_000)
// May 27 was partial, so keep Athena stats anchored at the first complete day.
const periodStart = new Date(
Math.max(
Math.min(startOfIsoWeek(periodEnd).getTime() - WEEK_MS, periodEnd.getTime() - DISPLAY_WINDOW_MS),
STATS_DATA_START_MS,
),
)
const periodStart = options?.full ? fullPeriodStart(periodEnd) : incrementalPeriodStart(periodEnd)
const athena = yield* Athena
const modelStats = yield* ModelStatRepo
const providerStats = yield* ProviderStatRepo
@ -37,26 +35,16 @@ export const syncStats: () => Effect.Effect<
yield* logRuntimeCheck()
const [modelAggregates, providerAggregates, geoAggregates, geoModelAggregates] = yield* Effect.all(
[
athena
.query(buildStatsQuery(periodStart, periodEnd, "model"))
.pipe(Effect.map((rows) => rows.flatMap(toModelAggregate))),
athena
.query(buildStatsQuery(periodStart, periodEnd, "provider"))
.pipe(Effect.map((rows) => rows.flatMap(toProviderAggregate))),
athena
.query(buildStatsQuery(periodStart, periodEnd, "geo"))
.pipe(Effect.map((rows) => rows.flatMap(toGeoAggregate))),
athena
.query(buildStatsQuery(periodStart, periodEnd, "geo_model"))
.pipe(Effect.map((rows) => rows.flatMap(toGeoAggregate))),
],
{ concurrency: "unbounded" },
const rows = yield* athena.query(buildStatsQuery(periodStart, periodEnd))
const modelRows = modelRowsFromAggregates(
rows.filter((row) => row.dimension === "model").flatMap(toModelAggregate),
)
const providerRows = providerRowsFromAggregates(
rows.filter((row) => row.dimension === "provider").flatMap(toProviderAggregate),
)
const geoRows = geoRowsFromAggregates(
rows.filter((row) => row.dimension === "geo" || row.dimension === "geo_model").flatMap(toGeoAggregate),
)
const modelRows = modelRowsFromAggregates(modelAggregates)
const providerRows = providerRowsFromAggregates(providerAggregates)
const geoRows = geoRowsFromAggregates([...geoAggregates, ...geoModelAggregates])
yield* Effect.all([modelStats.upsert(modelRows), providerStats.upsert(providerRows), geoStats.upsert(geoRows)], {
concurrency: "unbounded",
@ -92,6 +80,29 @@ export const syncStats: () => Effect.Effect<
}
})
// May 27 was partial, so keep Athena stats anchored at the first complete day.
function fullPeriodStart(periodEnd: Date) {
return new Date(
Math.max(
Math.min(startOfIsoWeek(periodEnd).getTime() - WEEK_MS, periodEnd.getTime() - DISPLAY_WINDOW_MS),
STATS_DATA_START_MS,
),
)
}
// Events are append-only, so completed periods never change once synced; hourly
// passes only recompute the periods the current ISO week can still touch. The daily
// full pass refreshes the whole display window (normalization changes, retired
// dimension cleanup).
function incrementalPeriodStart(periodEnd: Date) {
return new Date(
Math.max(
startOfIsoWeek(new Date(periodEnd.getTime() - INCREMENTAL_LOOKBACK_MS)).getTime(),
STATS_DATA_START_MS,
),
)
}
function logRuntimeCheck() {
return Effect.logInfo(
`athena stats runtime check ${JSON.stringify({