diff --git a/infra/console.ts b/infra/console.ts index 5fbf35de00..122ce30f3c 100644 --- a/infra/console.ts +++ b/infra/console.ts @@ -233,6 +233,7 @@ const bucket = new sst.cloudflare.Bucket("ZenData") const bucketNew = new sst.cloudflare.Bucket("ZenDataNew") const DISCORD_INCIDENT_WEBHOOK_URL = new sst.Secret("DISCORD_INCIDENT_WEBHOOK_URL") +const SLACK_INCIDENT_WEBHOOK_URL = new sst.Secret("SLACK_INCIDENT_WEBHOOK_URL") const AWS_SES_ACCESS_KEY_ID = new sst.Secret("AWS_SES_ACCESS_KEY_ID") const AWS_SES_SECRET_ACCESS_KEY = new sst.Secret("AWS_SES_SECRET_ACCESS_KEY") @@ -258,6 +259,7 @@ new sst.cloudflare.x.SolidStart("Console", { STRIPE_WEBHOOK_SECRET, SECRET.SupportApiKey, DISCORD_INCIDENT_WEBHOOK_URL, + SLACK_INCIDENT_WEBHOOK_URL, SECRET.HoneycombWebhookSecret, STRIPE_SECRET_KEY, EMAILOCTOPUS_API_KEY, diff --git a/packages/console/app/src/routes/honeycomb/webhook.ts b/packages/console/app/src/routes/honeycomb/webhook.ts index 0cae3629b7..8ea28b7d7e 100644 --- a/packages/console/app/src/routes/honeycomb/webhook.ts +++ b/packages/console/app/src/routes/honeycomb/webhook.ts @@ -37,31 +37,29 @@ const honeycombWebhookPayload = z.discriminatedUnion("type", [ }), ]) -const postDiscordMessage = async (payload: z.infer) => { - const names = - payload.type === "custom" - ? [] - : payload.groups.flatMap((item) => - item.group.map((g) => { - const result = item.result == null ? undefined : Number(item.result) - return `- ${g.value}${ - result !== undefined && Number.isFinite(result) - ? payload.type === "model_low_tps" - ? ` (${Math.round(result)} TPS)` - : ` (${Math.round(result * 100)}% errors)` - : "" - }` - }), - ) +const alertDetails = (payload: z.infer) => + payload.type === "custom" + ? [] + : payload.groups.flatMap((item) => + item.group.map((group) => { + const result = item.result == null ? undefined : Number(item.result) + return `- ${group.value}${ + result !== undefined && Number.isFinite(result) + ? payload.type === "model_low_tps" + ? ` (${Math.round(result)} TPS)` + : ` (${Math.round(result * 100)}% errors)` + : "" + }` + }), + ) +const postDiscordMessage = async (payload: z.infer) => { const content = [ `[**${payload.isTest ? "[TEST] " : ""}${payload.name ?? "Honeycomb alert"}**](${payload.url})`, - ...names, + ...alertDetails(payload), "", `<@&${DISCORD_ALERT_ROLE_ID}>`, - ] - .filter((line) => line !== undefined) - .join("\n") + ].join("\n") return fetch(Resource.DISCORD_INCIDENT_WEBHOOK_URL.value, { method: "POST", @@ -74,6 +72,21 @@ const postDiscordMessage = async (payload: z.infer) => { + const text = [ + `<${payload.url}|*${payload.isTest ? "[TEST] " : ""}${payload.name ?? "Honeycomb alert"}*>`, + ...alertDetails(payload), + "", + "", + ].join("\n") + + return fetch(Resource.SLACK_INCIDENT_WEBHOOK_URL.value, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ text, unfurl_links: false }), + }) +} + export async function POST(input: APIEvent) { const token = input.request.headers.get("X-Honeycomb-Webhook-Token") if (!safeEqual(token ?? "", Resource.HoneycombWebhookSecret.value)) { @@ -96,9 +109,10 @@ export async function POST(input: APIEvent) { return Response.json({ message: "ignored" }, { status: 200 }) } - const response = await postDiscordMessage(parsed.data) - if (!response.ok) { - return Response.json({ message: "discord webhook failed" }, { status: 502 }) + const [discord, slack] = await Promise.all([postDiscordMessage(parsed.data), postSlackMessage(parsed.data)]) + if (!discord.ok || !slack.ok) { + console.error("Honeycomb alert delivery failed", { discord: discord.status, slack: slack.status }) + return Response.json({ message: "alert webhook failed" }, { status: 502 }) } return Response.json({ message: "sent" }, { status: 200 }) diff --git a/sst-env.d.ts b/sst-env.d.ts index 109cb6d81b..cb59540b29 100644 --- a/sst-env.d.ts +++ b/sst-env.d.ts @@ -120,6 +120,10 @@ declare module "sst" { "type": "sst.sst.Secret" "value": string } + "SLACK_INCIDENT_WEBHOOK_URL": { + "type": "sst.sst.Secret" + "value": string + } "STRIPE_PUBLISHABLE_KEY": { "type": "sst.sst.Secret" "value": string @@ -309,4 +313,4 @@ declare module "sst" { } import "sst" -export {} \ No newline at end of file +export {}