feat(o11y): adding HTTP 40[1-3] alerts for providers (#835)

reown-com · Nov 19, 2024 · 9823700 · 9823700
1 parent c36e88b
commit 9823700
Showing 1 changed file with 30 additions and 2 deletions.
diff --git a/terraform/monitoring/panels/status/provider.libsonnet b/terraform/monitoring/panels/status/provider.libsonnet
@@ -11,10 +11,38 @@ local targets   = grafana.targets;
       datasource  = ds.prometheus,
     )
     .configure(defaults.configuration.timeseries)
-
     .addTarget(targets.prometheus(
-      datasource  = ds.prometheus,
+      datasource    = ds.prometheus,
       expr          = 'sum by(status_code) (increase(provider_status_code_counter_total{provider="%s"}[$__rate_interval]))' % provider,
       legendFormat  = '__auto',
     ))
+
+    .addTarget(targets.prometheus(
+      datasource    = ds.prometheus,
+      expr          = 'sum by(status_code) (increase(provider_status_code_counter_total{provider="%s", status_code=~"401|402|403"}[$__rate_interval]))' % provider,
+      refId         = 'Provider4xxErrors',
+      hide          = true,
+    ))
+    .setAlert(
+      vars.environment,
+      grafana.alert.new(
+        namespace     = vars.namespace,
+        name          = "%(env)s - Provider 40[1-3] Errors alert" % { env: grafana.utils.strings.capitalize(vars.environment) },
+        message       = '%(env)s - Provider 40[1-3] Errors alert' % { env: grafana.utils.strings.capitalize(vars.environment) },
+        notifications = vars.notifications,
+        noDataState   = 'no_data',
+        period        = '0m',
+        conditions    = [
+          grafana.alertCondition.new(
+            evaluatorParams = [ 0 ],
+            evaluatorType   = 'gt',
+            operatorType    = 'or',
+            queryRefId      = 'Provider4xxErrors',
+            queryTimeStart  = '15m',
+            queryTimeEnd    = 'now',
+            reducerType     = grafana.alert_reducers.Avg
+          ),
+        ],
+      ),
+    )
 }