chore: merge dev into v2 (#36312)
Co-authored-by: opencode-agent[bot] <219766164+opencode-agent[bot]@users.noreply.github.com> Co-authored-by: LukeParkerDev <10430890+Hona@users.noreply.github.com> Co-authored-by: opencode-agent[bot] <opencode-agent[bot]@users.noreply.github.com> Co-authored-by: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Co-authored-by: Brendan Allan <14191578+Brendonovich@users.noreply.github.com> Co-authored-by: Aarav Sareen <96787824+arvsrn@users.noreply.github.com> Co-authored-by: Julian Coy <julian@ex-machina.co> Co-authored-by: Brendan Allan <git@brendonovich.dev> Co-authored-by: usrnk1 <7547651+usrnk1@users.noreply.github.com> Co-authored-by: opencode <opencode@sst.dev> Co-authored-by: Vladimir Glafirov <vglafirov@gitlab.com> Co-authored-by: Adam <2363879+adamdotdevin@users.noreply.github.com> Co-authored-by: Frank <frank@anoma.ly> Co-authored-by: Jay <53023+jayair@users.noreply.github.com> Co-authored-by: Dustin Deus <deusdustin@gmail.com> Co-authored-by: Kit Langton <kit.langton@gmail.com> Co-authored-by: James Long <longster@gmail.com> Co-authored-by: Simon Klee <hello@simonklee.dk> Co-authored-by: Jay <air@live.ca> Co-authored-by: Jack <jack@anoma.ly> Co-authored-by: David Hill <1879069+iamdavidhill@users.noreply.github.com> Co-authored-by: Aiden Cline <aidenpcline@gmail.com> Co-authored-by: James Long <jlongster@users.noreply.github.com> Co-authored-by: 冯基魁 <56265583+fengjikui@users.noreply.github.com> Co-authored-by: Aiden Cline <rekram1-node@users.noreply.github.com> Co-authored-by: Victor Navarro <vn4varro@gmail.com>
This commit is contained in:
parent
9028c2d8f8
commit
43ecf3ff1b
82 changed files with 4519 additions and 1231 deletions
|
|
@ -113,32 +113,32 @@ const poll: (
|
|||
return yield* poll(client, queryExecutionId, attempt + 1)
|
||||
})
|
||||
|
||||
const results: (
|
||||
client: AwsAthenaClient,
|
||||
queryExecutionId: string,
|
||||
nextToken?: string,
|
||||
) => Effect.Effect<AthenaData[], AthenaQueryError> = Effect.fn("Athena.results")(function* (
|
||||
client: AwsAthenaClient,
|
||||
queryExecutionId: string,
|
||||
nextToken?: string,
|
||||
) {
|
||||
const result = yield* Effect.tryPromise({
|
||||
try: () =>
|
||||
client.send(
|
||||
new GetQueryResultsCommand({
|
||||
QueryExecutionId: queryExecutionId,
|
||||
NextToken: nextToken,
|
||||
MaxResults: ATHENA_PAGE_SIZE,
|
||||
}),
|
||||
),
|
||||
catch: (cause) => new AthenaQueryError({ message: "Failed to read Athena stats results", queryExecutionId, cause }),
|
||||
const results: (client: AwsAthenaClient, queryExecutionId: string) => Effect.Effect<AthenaData[], AthenaQueryError> =
|
||||
Effect.fn("Athena.results")(function* (client: AwsAthenaClient, queryExecutionId: string) {
|
||||
// Accumulate pages iteratively; recursive spreads copied every previously
|
||||
// fetched row per page and blew up memory on large result sets.
|
||||
const rows: AthenaData[] = []
|
||||
let nextToken: string | undefined
|
||||
while (true) {
|
||||
const result = yield* Effect.tryPromise({
|
||||
try: () =>
|
||||
client.send(
|
||||
new GetQueryResultsCommand({
|
||||
QueryExecutionId: queryExecutionId,
|
||||
NextToken: nextToken,
|
||||
MaxResults: ATHENA_PAGE_SIZE,
|
||||
}),
|
||||
),
|
||||
catch: (cause) =>
|
||||
new AthenaQueryError({ message: "Failed to read Athena stats results", queryExecutionId, cause }),
|
||||
})
|
||||
const columns = result.ResultSet?.ResultSetMetadata?.ColumnInfo?.map((item) => item.Name ?? "") ?? []
|
||||
// The first page starts with the header row.
|
||||
for (const row of (result.ResultSet?.Rows ?? []).slice(nextToken ? 0 : 1)) rows.push(rowData(columns, row))
|
||||
if (!result.NextToken) return rows
|
||||
nextToken = result.NextToken
|
||||
}
|
||||
})
|
||||
const columns = result.ResultSet?.ResultSetMetadata?.ColumnInfo?.map((item) => item.Name ?? "") ?? []
|
||||
const rows = (result.ResultSet?.Rows ?? []).slice(nextToken ? 0 : 1).map((row) => rowData(columns, row))
|
||||
|
||||
if (!result.NextToken) return rows
|
||||
return [...rows, ...(yield* results(client, queryExecutionId, result.NextToken))]
|
||||
})
|
||||
|
||||
function rowData(columns: string[], row: Row): AthenaData {
|
||||
return Object.fromEntries(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue