ceos/opencode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore: simplify honeycomb alerts (#26142)

Browse source
This commit is contained in:
Victor Navarro 2026-05-07 09:56:10 +02:00 • committed by GitHub
commit f8aa4a3be0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 185 additions and 408 deletions
Download patch file Download diff file Expand all files Collapse all files

6
infra/console.ts
View file

@ -1,5 +1,6 @@
import { domain } from "./stage"
import { EMAILOCTOPUS_API_KEY } from "./app"
import { SECRET } from "./secret"
////////////////
// DATABASE
@ -221,8 +222,6 @@ const AUTH_API_URL = new sst.Linkable("AUTH_API_URL", {
const STRIPE_WEBHOOK_SECRET = new sst.Linkable("STRIPE_WEBHOOK_SECRET", {
properties: { value: stripeWebhook.secret },
})
const INCIDENT_WEBHOOK_SIGNING_SECRET = new sst.Secret("INCIDENT_WEBHOOK_SIGNING_SECRET")
const DISCORD_INCIDENT_WEBHOOK_URL = new sst.Secret("DISCORD_INCIDENT_WEBHOOK_URL")
const gatewayKv = new sst.cloudflare.Kv("GatewayKv")
@ -233,6 +232,7 @@ const gatewayKv = new sst.cloudflare.Kv("GatewayKv")
const bucket = new sst.cloudflare.Bucket("ZenData")
const bucketNew = new sst.cloudflare.Bucket("ZenDataNew")
const DISCORD_INCIDENT_WEBHOOK_URL = new sst.Secret("DISCORD_INCIDENT_WEBHOOK_URL")
const AWS_SES_ACCESS_KEY_ID = new sst.Secret("AWS_SES_ACCESS_KEY_ID")
const AWS_SES_SECRET_ACCESS_KEY = new sst.Secret("AWS_SES_SECRET_ACCESS_KEY")
@ -254,8 +254,8 @@ new sst.cloudflare.x.SolidStart("Console", {
database,
AUTH_API_URL,
STRIPE_WEBHOOK_SECRET,
INCIDENT_WEBHOOK_SIGNING_SECRET,
DISCORD_INCIDENT_WEBHOOK_URL,
SECRET.HoneycombWebhookSecret,
STRIPE_SECRET_KEY,
EMAILOCTOPUS_API_KEY,
AWS_SES_ACCESS_KEY_ID,

369
infra/monitoring.ts
View file

@ -1,318 +1,91 @@
const displayName = (s: string) =>
s
.split("-")
.map((w) => w.charAt(0).toUpperCase() + w.slice(1))
.join(" ")
.replace(/(?<=\d) (?=\d)/g, ".")
import { SECRET } from "./secret"
import { domain } from "./stage"
const resourceName = (s: string) => displayName(s).replace(/[^a-zA-Z0-9]/g, "")
const varSpec = (label: string, name: string) =>
$jsonStringify({
content: [
{
content: [
{
attrs: {
name,
label,
missing: false,
},
type: "varSpec",
},
],
type: "paragraph",
},
],
type: "doc",
})
const fields = {
model: incident.getAlertAttributeOutput({ name: "Model" }),
product: incident.getAlertAttributeOutput({ name: "Product" }),
}
const alertSource = new incident.AlertSource("HoneycombAlertSource", {
name: $app.stage === "production" ? "Honeycomb" : `Honeycomb (${$app.stage})`,
sourceType: "honeycomb",
template: {
title: {
literal: varSpec("Payload -> Title", "title"),
},
description: {
literal: varSpec("Payload -> Description", "description"),
},
attributes: [
{
alertAttributeId: fields.model.id,
binding: {
value: {
reference: 'expressions["model"]',
},
mergeStrategy: "first_wins",
},
},
{
alertAttributeId: fields.product.id,
binding: {
value: {
reference: 'expressions["product"]',
},
mergeStrategy: "first_wins",
},
},
],
expressions: [
{
label: "Model",
operations: [
{
operationType: "parse",
parse: {
returns: {
array: false,
type: fields.model.type,
},
source: "$['model']",
},
},
],
reference: "model",
rootReference: "payload",
},
{
label: "Product",
operations: [
{
operationType: "parse",
parse: {
returns: {
array: false,
type: fields.product.type,
},
source: "$['product']",
},
},
],
reference: "product",
rootReference: "payload",
},
],
},
})
const webhookRecipient = new honeycomb.WebhookRecipient(`IncidentWebhook`, {
name: $app.stage === "production" ? "Incident.io" : `Incident.io (${$app.stage})`,
url: alertSource.alertEventsUrl,
secret: alertSource.secretToken,
const webhookRecipient = new honeycomb.WebhookRecipient("DiscordAlerts", {
name: $app.stage === "production" ? "Discord Alerts" : `Discord Alerts (${$app.stage})`,
url: `https://${domain}/honeycomb/webhook`,
secret: SECRET.HoneycombWebhookSecret.result,
templates: [
{
type: "trigger",
body: $jsonStringify({
title: "{{ .Name }}",
description: "{{ .Description }}",
status: "{{ .Alert.Status }}",
deduplication_key: "{{ .Alert.InstanceID }}",
source_url: "{{ .Result.URL }}",
model: "{{ .Vars.model }}",
product: "{{ .Vars.product }}",
}),
body: `{
"url": {{ .Result.URL | quote }},
"type": {{ .Vars.type | quote }},
"name": {{ .Name | quote }},
"status": {{ .Alert.Status | quote }},
"isTest": {{ .Alert.IsTest }},
"groups": {{ .Result.GroupsTriggered | toJson }}
}`,
},
],
variables: [
{
name: "model",
},
{
name: "product",
name: "type",
},
],
})
new incident.AlertRoute("HoneycombAlertRoute", {
name: $app.stage === "production" ? "Honeycomb" : `Honeycomb (${$app.stage})`,
enabled: true,
isPrivate: false,
alertSources: [
{
alertSourceId: alertSource.id,
conditionGroups: [
{
conditions: [
{
subject: "alert.title",
operation: "is_set",
paramBindings: [],
},
],
},
],
},
],
conditionGroups: [
{
conditions: [
{
subject: "alert.title",
operation: "is_set",
paramBindings: [],
},
],
},
],
expressions: [],
escalationConfig: {
autoCancelEscalations: true,
escalationTargets: [],
},
incidentConfig: {
autoDeclineEnabled: true,
enabled: true,
conditionGroups: [],
deferTimeSeconds: 0,
groupingKeys: [
const modelHttpErrorsQuery = (product: "go" | "zen") => {
const filters = [
{ column: "model", op: "exists" },
{ column: "event_type", op: "=", value: "completions" },
{ column: "user_agent", op: "contains", value: "opencode" },
{ column: "isGoTier", op: "=", value: product === "go" ? "true" : "false" },
]
return honeycomb.getQuerySpecificationOutput({
breakdowns: ["model"],
calculatedFields: [
{
reference: $interpolate`alert.attributes.${fields.model.id}`,
},
{
reference: $interpolate`alert.attributes.${fields.product.id}`,
name: "is_failed_http_status",
expression: `IF(AND(GTE($status, "400"), NOT(EQUALS($status, "401"))), 1, 0)`,
},
],
groupingWindowSeconds: 3600,
},
incidentTemplate: {
name: {
value: {
literal: varSpec("Alert -> Title", "alert.title"),
},
},
summary: {
value: {
literal: varSpec("Alert -> Description", "alert.description"),
},
},
startInTriage: {
value: {
literal: "true",
},
},
severity: {
mergeStrategy: "first-wins",
},
incidentMode: {
value: {
literal: $app.stage === "production" ? "standard" : "test",
},
},
},
})
type Product = "go" | "zen"
type Trigger = (opts: { model: string; product: Product }) => {
id: string
title: string
description: string
json: honeycomb.GetQuerySpecificationOutputArgs
threshold: { op: ">=" | "<="; value: number }
}
type Model = { id: string; products: Product[]; triggers: Trigger[] }
const httpErrors: Trigger = ({ model, product }) => ({
id: "increased-http-errors",
title: `Increased HTTP Errors for ${displayName(model)} on ${displayName(product)}`,
description: `Detected increased rate of HTTP errors for ${displayName(model)} on OpenCode ${displayName(product)}`,
json: {
calculations: [
{
op: "COUNT",
name: "TOTAL",
filterCombination: "AND",
filters: [
{ column: "model", op: "=", value: model },
{ column: "event_type", op: "=", value: "completions" },
{ column: "user_agent", op: "contains", value: "opencode" },
{ column: "isGoTier", op: "=", value: product === "go" ? "true" : "false" },
],
},
{
op: "COUNT",
name: "FAILED",
filterCombination: "AND",
filters: [
{ column: "model", op: "=", value: model },
{ column: "event_type", op: "=", value: "completions" },
{ column: "user_agent", op: "contains", value: "opencode" },
{ column: "isGoTier", op: "=", value: product === "go" ? "true" : "false" },
{ column: "status", op: ">=", value: "400" },
{ column: "status", op: "!=", value: "401" },
],
},
{ op: "COUNT", name: "TOTAL", filterCombination: "AND", filters },
{ op: "SUM", name: "FAILED", column: "is_failed_http_status", filterCombination: "AND", filters },
],
formulas: [{ name: "ERROR", expression: "$FAILED / $TOTAL" }],
formulas: [{ name: "ERROR", expression: "IF(GTE($TOTAL, 2500), DIV($FAILED, $TOTAL), 0)" }],
timeRange: 900,
},
threshold: { op: ">=", value: 0.8 },
}).json
}
const description = "Managed by SST (Don't edit in Honeycomb UI)"
new honeycomb.Trigger("IncreasedModelHttpErrorsGo", {
name: "Increased Model HTTP Errors [Go]",
description,
queryJson: modelHttpErrorsQuery("go"),
alertType: "on_change",
frequency: 300,
thresholds: [{ op: ">=", value: 0.8, exceededLimit: 1 }],
recipients: [
// {
// id: webhookRecipient.id,
// notificationDetails: [
// {
// variables: [{ name: "type", value: "model_http_errors" }],
// },
// ],
// },
],
})
const models: Model[] = [
{ id: "kimi-k2.6", products: ["go", "zen"], triggers: [httpErrors] },
{ id: "kimi-k2.5", products: ["go", "zen"], triggers: [httpErrors] },
{ id: "deepseek-v4-flash", products: ["go", "zen"], triggers: [httpErrors] },
{ id: "deepseek-v4-pro", products: ["go", "zen"], triggers: [httpErrors] },
{ id: "glm-5.1", products: ["go", "zen"], triggers: [httpErrors] },
// { id: "glm-5", products: ["go"], triggers: [httpErrors] },
{ id: "qwen3.6-plus", products: ["go", "zen"], triggers: [httpErrors] },
{ id: "qwen3.5-plus", products: ["go"], triggers: [httpErrors] },
{ id: "minimax-m2.7", products: ["go", "zen"], triggers: [httpErrors] },
// { id: "minimax-m2.5", products: ["go", "zen"], triggers: [httpErrors] },
{ id: "mimo-v2.5-pro", products: ["go"], triggers: [httpErrors] },
// { id: "mimo-v2.5", products: ["go"], triggers: [httpErrors] },
// { id: "mimo-v2-omni", products: ["go"], triggers: [httpErrors] },
// { id: "mimo-v2-pro", products: ["go"], triggers: [httpErrors] },
{ id: "claude-opus-4-7", products: ["zen"], triggers: [httpErrors] },
// { id: "claude-opus-4-6", products: ["zen"], triggers: [httpErrors] },
// { id: "claude-sonnet-4-6", products: ["zen"], triggers: [httpErrors] },
{ id: "gpt-5.5", products: ["zen"], triggers: [httpErrors] },
{ id: "big-pickle", products: ["zen"], triggers: [httpErrors] },
// { id: "minimax-m2.5-free", products: ["zen"], triggers: [httpErrors] },
// { id: "hy3-preview-free", products: ["zen"], triggers: [httpErrors] },
// { id: "nemotron-3-super-free", products: ["zen"], triggers: [httpErrors] },
// { id: "trinity-large-preview-free", products: ["zen"], triggers: [httpErrors] },
// { id: "ling-2.6-flash-free", products: ["zen"], triggers: [httpErrors] },
]
if ($app.stage !== "production") {
models.splice(1)
}
for (const model of models) {
for (const product of model.products) {
for (const trigger of model.triggers) {
const spec = trigger({ model: model.id, product })
new honeycomb.Trigger(resourceName(`${spec.id}-${product}-${model.id}`), {
name: spec.title,
description: spec.description,
queryJson: honeycomb.getQuerySpecificationOutput(spec.json).json,
alertType: "on_change",
frequency: 300,
thresholds: [{ ...spec.threshold, exceededLimit: 1 }],
recipients: [
{
id: webhookRecipient.id,
notificationDetails: [
{
variables: [
{ name: "model", value: model.id },
{ name: "product", value: product },
],
},
],
},
],
})
}
}
}
new honeycomb.Trigger("IncreasedModelHttpErrorsZen", {
name: "Increased Model HTTP Errors [Zen]",
description,
queryJson: modelHttpErrorsQuery("zen"),
alertType: "on_change",
frequency: 300,
thresholds: [{ op: ">=", value: 0.8, exceededLimit: 1 }],
recipients: [
// {
// id: webhookRecipient.id,
// notificationDetails: [
// {
// variables: [{ name: "type", value: "model_http_errors" }],
// },
// ],
// },
],
})

7
infra/secret.ts
View file

@ -1,4 +1,11 @@
sst.Linkable.wrap(random.RandomPassword, (resource) => ({
properties: {
value: resource.result,
},
}))
export const SECRET = {
R2AccessKey: new sst.Secret("R2AccessKey", "unknown"),
R2SecretKey: new sst.Secret("R2SecretKey", "unknown"),
HoneycombWebhookSecret: new random.RandomPassword("HoneycombWebhookSecret", { length: 24 }),
}