Reduce ContainerRestarting alert noise

Once fired, leave it firing for 10m. That should help with crashloops where the alert keeps getting resolved and firing again.
utilitywarehouse · Oct 10, 2023 · 26cdc3a · 26cdc3a
1 parent 15966f8
commit 26cdc3a
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 0 deletions.
diff --git a/common/all.yaml.tmpl b/common/all.yaml.tmpl
@@ -143,6 +143,7 @@ groups:
           dashboard: "https://grafana.$ENVIRONMENT.$PROVIDER.uw.systems/d/VAE0wIcik/kubernetes-pod-resources?orgId=1&refresh=1m&from=now-12h&to=now&var-instance=All&var-namespace={{ $labels.namespace }}"
       - alert: SystemPodRestartingOften
         expr: increase(kube_pod_container_status_restarts_total{namespace=~"kube-system|sys-.*"}[10m]) > 3
+        keep_firing_for: 10m
         labels:
           team: infra
         annotations:

diff --git a/common/container.yaml.tmpl b/common/container.yaml.tmpl
@@ -6,6 +6,7 @@ groups:
     rules:
       - alert: ContainerRestartingOften
         expr: increase(kube_pod_container_status_restarts_total[10m]) > 3
+        keep_firing_for: 10m
         labels:
           group: container
         annotations: