spin-alerts.list

    groups:
    - name: pod_cpu_usage_is_high
      rules:
        - alert: CPUISHIGH
          expr: sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name!=""}[5m])) by (pod_name)  > 0.1
          for: 1m
          labels:
            severity: critical
          annotations:
            description: 'POD {{ $labels.pod }} cpu is high'
            summary: 'pod cpu is high'

    - name: node_cpu_greater_than_80
      rules:
        - alert: NODECPUISHIGH
          expr: (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80 
          for: 1m
          labels:
            severity: critical
          annotations:
            description: 'node {{ $labels.node }} cpu is high'
            summary: 'node cpu is greater than 80 precent'
    - name: node_memory_left_lessser_than_10 
      rules:
        - alert: NODEMEMORY
          expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10 
          for: 1m
          labels:
            severity: critical
          annotations:
            description: 'node {{ $labels.node }} memory left is low'
            summary: 'node memory left is lesser than 10 precent'
    - name: spinnaker-service-is-down
      rules:
      - alert: clouddriver-is-down
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            is not responding
          summary: Clouddriver Spinnaker service is down
        expr: absent(up{service="spin-clouddriver"}) == 1
        labels:
          severity: critical
      #- alert: clouddriver-rw-is-down
      #  annotations:
      #    description: Service {{$labels.service}} in namespace {{$labels.namespace}}
      #      is not responding
      #    summary: Clouddriver-rw Spinnaker service is down
      #  expr: absent(up{service="spin-clouddriver-rw"}) == 1
      #  labels:
      #    severity: critical
      #- alert: clouddriver-ro-is-down
      #  annotations:
      #    description: Service {{$labels.service}} in namespace {{$labels.namespace}}
      #      is not responding
      #    summary: Clouddriver-ro Spinnaker service is down
      #  expr: absent(up{service="spin-clouddriver-ro"}) == 1
      #  labels:
      #    severity: critical
      #- alert: clouddriver-caching-is-down
      #  annotations:
      #    description: Service {{$labels.service}} in namespace {{$labels.namespace}}
      #      is not responding
      #    summary: Clouddriver-caching Spinnaker service is down
      #  expr: absent(up{service="spin-clouddriver-caching"}) == 1
      #  labels:
      #    severity: critical
      - alert: gate-is-down
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            is not responding
          summary: Gate Spinnaker services is down
        expr: absent(up{service="spin-gate"}) == 1
        labels:
          severity: critical
      - alert: orca-is-down
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            is not responding
          summary: Orca Spinnaker service is down
        expr: absent(up{job="opsmx_spinnaker_metrics", service="spin-orca"}) == 1
        labels:
          severity: critical
      - alert: igor-is-down
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            is not responding
          summary: Igor Spinnaker service is down
        expr: absent(up{service="spin-igor"}) == 1
        labels:
          severity: critical
      - alert: echo-is-down
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            is not responding
          summary: Echo Spinnaker service is down
        expr: absent(up{service="spin-echo" }) == 1
        labels:
          severity: critical
      #- alert: echo-scheduler-is-down
      #  annotations:
      #    description: Service {{$labels.service}} in namespace {{$labels.namespace}}
      #      is not responding
      #    summary: Echo-Scheduler Spinnaker service is down
      #  expr: absent(up{service="spin-echo-scheduler" }) == 1
      #  labels:
      #    severity: critical
      #- alert: echo-worker-is-down
      #  annotations:
      #    description: Service {{$labels.service}} in namespace {{$labels.namespace}}
      #      is not responding
      #    summary: Echo-worker Spinnaker service is down
      #  expr: absent(up{service="spin-echo-worker" }) == 1
      #  labels:
      #    severity: critical
      - alert: front50-is-down
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            is not responding
          summary: Front50 Spinnaker service is down
        expr: absent(up{service="spin-front50" }) == 1
        labels:
          severity: critical
    - name: latency-too-high
      rules:
      - alert: clouddriver-latency-too-high
        annotations:
          description: "Service {{$labels.service}} is slow ({{$value}} per minute)"
          summary: "latency of clouddriver is too high"
        expr: sum(rate(clouddriver:controller:invocations__count_total{service="spin-clouddriver",statusCode="200"}[2m]))
          by (instance, method)/ sum(rate(clouddriver:controller:invocations__total{service="spin-clouddriver",statusCode="200"}[2m]))
          by (instance, method) > 70000
        for: 5m
        labels:
          severity: warning
      #- alert: clouddriver-ro-latency-too-high
      #  annotations:
      #    description: Service {{$labels.service}} is slow ({{$value}} per minute)
      #    summary: latency of clouddriver-ro is too high
      #  expr: sum(rate(clouddriver_ro:controller:invocations__count_total{service="spin-clouddriver-ro",statusCode="200"}[2m]))
      #    by (instance, method)/ sum(rate(clouddriver_ro:controller:invocations__total{service="spin-clouddriver-ro",statusCode="200"}[2m]))
      #    by (instance, method) > 200
      #  for: 5m
      #  labels:
      #    severity: warning
      #- alert: clouddriver-rw-latency-too-high
      #  annotations:
      #    description: Service {{$labels.service}} is slow ({{$value}} per minute)
      #    summary: latency of clouddriver-rw is too high
      #  expr: sum(rate(clouddriver_rw:controller:invocations__count_total{service="spin-clouddriver-rw",statusCode="200"}[2m]))
      #    by (instance, method)/ sum(rate(clouddriver_rw:controller:invocations__total{service="spin-clouddriver-rw",statusCode="200"}[2m]))
      #    by (instance, method) > 200
      #  for: 5m
      #  labels:
      #    severity: warning
      #- alert: clouddriver-caching-latency-too-high
      #  annotations:
      #    description: Service {{$labels.service}} is slow ({{$value}} per minute)
      #    summary: latency clouddriver-caching is too high
      #  expr: sum(rate(clouddriver_caching:controller:invocations__count_total{service="spin-clouddriver-caching",statusCode="200"}[2m]))
      #    by (instance, method)/ sum(rate(clouddriver_caching:controller:invocations__total{service="spin-clouddriver-caching",statusCode="200"}[2m]))
      #    by (instance, method) > 300
      #  for: 5m
      #  labels:
      #    severity: warning
      - alert: gate-latency-too-high
        annotations:
          description: Service {{$labels.service}} is slow ({{$value}} per minute)
          summary: latency of gate is too high
        expr: sum(rate(gate:controller:invocations__count_total{service="spin-gate",statusCode="200"}[2m]))
          by (instance, method)/ sum(rate(gate:controller:invocations__total{service="spin-gate",statusCode="200"}[2m]))
          by (instance, method) > 5000
        for: 5m
        labels:
          severity: warning
      - alert: orca-latency-too-high
        annotations:
          description: Service {{$labels.service}} is slow ({{$value}} per minute)
          summary: latency of orca is too high
        expr: sum(rate(orca:controller:invocations__count_total{service="spin-orca",statusCode="200"}[2m]))
          by (instance, method)/ sum(rate(orca:controller:invocations__total{service="spin-orca",statusCode="200"}[2m]))
          by (instance, method) > 2000
        for: 5m
        labels:
          severity: warning
      - alert: igor-latency-too-high
        annotations:
          description: Service {{$labels.service}} is slow ({{$value}} per minute)
          summary: latency of igor is too high
        expr: sum(rate(igor:controller:invocations__count_total{service="spin-igor",statusCode="200"}[2m]))
          by (instance, method)/ sum(rate(igor:controller:invocations__total{service="spin-igor",statusCode="200"}[2m]))
          by (instance, method) > 10000
        for: 5m
        labels:
          severity: warning
      - alert: echo-latency-too-high
        annotations:
          description: Service {{$labels.service}} is slow ({{$value}} per minute)
          summary: latency of echo is too high
        expr: sum(rate(echo:controller:invocations__count_total{service="spin-echo",statusCode="200"}[2m]))
          by (instance, method)/ sum(rate(echo:controller:invocations__total{service="spin-echo",statusCode="200"}[2m]))
          by (instance, method) > 6000
        for: 5m
        labels:
          severity: warning
      #- alert: echo_scheduler-latency-too-high
      #  annotations:
      #    description: Service {{$labels.service}} is slow ({{$value}} per minute)
      #    summary: latency of echo-scheduler is too high
      #  expr: sum(rate(echo_scheduler:controller:invocations__count_total{service="spin-echo-scheduler",statusCode="200"}[2m]))
      #    by (instance, method)/ sum(rate(echo_scheduler:controller:invocations__total{service="spin-echo-scheduler",statusCode="200"}[2m]))
      #    by (instance, method) > 300
      #  for: 5m
      #  labels:
      #    severity: warning
      #- alert: echo_worker-latency-too-high
      #  annotations:
      #    description: Service {{$labels.service}} is slow ({{$value}} per minute)
      #    summary: latency of echo-worker is too high
      #  expr: sum(rate(echo_worker:controller:invocations__count_total{service="spin-echo-worker",statusCode="200"}[2m]))
      #    by (instance, method)/ sum(rate(echo_worker:controller:invocations__total{service="spin-echo-worker",statusCode="200"}[2m]))
      #    by (instance, method) > 2000
      #  for: 5m
      #  labels:
      #    severity: warning
      - alert: front50-latency-too-high
        annotations:
          description: "Service {{$labels.service}} is slow ({{$value}} per minute)"
          summary: 'latency of front50 is too high'
        expr: sum(rate(front50:controller:invocations__count_total{service="spin-front50",statusCode="200"}[2m]))
          by (instance, method)/ sum(rate(front50:controller:invocations__total{service="spin-front50",statusCode="200"}[2m]))
          by (instance, method) > 1200
        for: 5m
        labels:
          severity: warning
    - name: jvm-too-high
      rules:
      - alert: clouddriver-rw-pod-may-be-evicted-soon
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            may be evicted soon
          summary: Clouddriver-rw JVM memory too high
        expr: (sum(clouddriver_rw:jvm:memory:used__value) by (instance, area) / sum(clouddriver_rw:jvm:memory:max__value)
          by (instance, area)) > .9
        labels:
          severity: warning
      - alert: clouddriver-ro-pod-may-be-evicted-soon
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            may be evicted soon
          summary: Clouddriver-ro JVM memory too high
        expr: (sum(clouddriver_ro:jvm:memory:used__value) by (instance, area) / sum(clouddriver_ro:jvm:memory:max__value)
          by (instance, area)) > .9
        labels:
          severity: warning
      - alert: clouddriver-caching-pod-may-be-evicted-soon
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            may be evicted soon
          summary: Clouddriver-caching JVM memory too high
        expr: (sum(clouddriver_caching:jvm:memory:used__value) by (instance, area) / sum(clouddriver_caching:jvm:memory:max__value)
          by (instance, area)) > .9
        labels:
          severity: warning
      - alert: gate-pod-may-be-evicted-soon
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            may be evicted soon
          summary: gate JVM memory too high
        expr: (sum(gate:jvm:memory:used__value) by (instance, area) / sum(gate:jvm:memory:max__value)
          by (instance, area)) > .9
        labels:
          severity: warning
      - alert: orca-pod-may-be-evicted-soon
        annotations:
          description: Service {{$labels.service}} may be evicted soon
          summary: orca JVM memory too high
        expr: (sum(orca:jvm:gc:liveDataSize__value) by (instance, area) / sum(orca:jvm:gc:maxDataSize__value)
          by (instance, area)) > .9
        labels:
          severity: warning
      - alert: igor-pod-may-be-evicted-soon
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            may be evicted soon
          summary: igor JVM memory too high
        expr: (sum(igor:jvm:memory:used__value) by (instance, area) / sum(igor:jvm:memory:max__value)
          by (instance, area)) > .9
        labels:
          severity: warning
      - alert: echo-scheduler-pod-may-be-evicted-soon
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            may be evicted soon
          summary: echo-scheduler JVM memory too high
        expr: (sum(echo_scheduler:jvm:memory:used__value) by (instance, area) / sum(echo_scheduler:jvm:memory:max__value)
          by (instance, area)) > .9
        labels:
          severity: warning
      - alert: echo-worker-pod-may-be-evicted-soon
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            may be evicted soon
          summary: echo-worker JVM memory too high
        expr: (sum(echo_worker:jvm:memory:used__value) by (instance, area) / sum(echo_worker:jvm:memory:max__value)
          by (instance, area)) > .9
        labels:
          severity: warning
      - alert: front50-pod-may-be-evicted-soon
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            may be evicted soon
          summary: Front50 JVM memory too high
        expr: (sum(front50:jvm:memory:used__value) by (instance, area) / sum(front50:jvm:memory:max__value)
          by (instance, area)) > .9
        labels:
          severity: warning
    # Clouddriver-Service specific alerts
    - name: Clouddriver-execution-time
      rules:
      - alert: Clouddriver-execution-time-too-high
        annotations:
          description: "Service {{$labels.service}} execution time too high"
          summary: "Clouddriver:executiontime is high"
        expr: sum(rate(clouddriver:executionTime__total[2m])) by (instance, agent) / sum(rate(clouddriver:executionTime__count_total[2m])) by (instance, agent) > 100
        for: 2m
        labels:
          severity: warning
    # Front50-Service specific alerts
    - name: Front50-cache
      rules:
      - alert: "front50:storageServiceSupport:cacheAge__value"
        annotations:
          description: value = {{$value}}
          summary: "front50 cacheAge too high"
        expr: front50:storageServiceSupport:cacheAge__value > 300000
        for: 2m
        labels:
          severity: warning
    # fiat-Service specific alerts
    - name: Fiat-UserRoles-sync
      rules:
      - alert: fiat-userRoles-sync-time-too-high
        annotations:
          description: "Fiat userRoles sync time is high"
          summary: "fiat-userRoles-sync-time-too-high"
        expr: fiat:fiat:userRoles:syncTime__count_total/fiat:fiat:userRoles:syncTime__total > 8
        for: 2m
        labels:
          severity: warning
    # gate-Service specific alerts
    - name: Gate-issues
      rules:
      - alert: "Gate:hystrix:latencyTotal__percentile_50__value"
        annotations:
          description: "Median gate response too slow"
          summary: "50 percentile value is {{ $value }}"
        expr: gate:hystrix:latencyTotal__percentile_50__value > 1600
        for: 5m
        labels:
          severity: critical
      - alert: "gate:hystrix:latencyTotal__percentile_90__value"
        annotations:
          description: "gate response slow"
          summary: "90 percentile value is {{ $value }}"
        expr: gate:hystrix:latencyTotal__percentile_90__value > 2000
        for: 5m
        labels:
          severity: warning
      - alert: "gate:hystrix:errorPercentage__value"
        annotations:
          description: "gate experiencing errors"
          summary: "Error percentage is {{ $value }}"
        expr: gate:hystrix:errorPercentage__value > 0.01
        for: 2m
        labels:
          severity: warning
    # Orca-Service specific alerts
    - name: orca-queue-issue
      rules:
      - alert: orca-queue-depth-high
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            may be evicted soon
          summary: Orca queue depth is high
        expr: (sum(orca:queue:ready:depth) by (instance) ) > 10
        labels:
          severity: warning
      - alert: orca-queue-lag-high
        annotations:
          description: Lag = {{$value}}
          summary: Orca queue lag is high
        expr: sum(rate (orca:controller:invocations__totalTime_total[2m])) by (instance)  /
          sum(rate(orca:controller:invocations__count_total[2m])) by (instance) > 0.5
        labels:
          severity: warning
    - name: igor-needs-attention
      rules:
      - alert: igor-needs-attention
        annotations:
          description: Igor in namespace {{$labels.namespace}} needs human help
          summary: Igor needs attention
        expr: igor:pollingMonitor:itemsOverThreshold > 0
        labels:
          severity: crtical
# okhttp alerts
#  - name: clouddriver-okhttp
#      rules:
#      - alert: clouddriver-okhttp
#        annotations:
#          description: clouddriver needs attention
#          summary: clouddriver needs attention
#        expr: sum(rate(clouddriver:okhttp:requests__count_total{service="spin-clouddriver",statusCode="200"}[2m])) by (instance, method) / 
#          sum(rate(clouddriver:okhttp:requests__total{service="spin-clouddriver",statusCode="200"}[2m])) by (instance, method) > 100
#        labels:
#          severity: warning
#    - name: echo-okhttp
#      rules:
#      - alert: echo-okhttp
#        annotations:
#          description: echo needs attention
#          summary: echo needs attention
#        expr: sum(rate(echo:okhttp:requests__count_total{service="spin-echo",statusCode="200"}[2m])) by (instance, method) /
#          sum(rate(echo:okhttp:requests__total{service="spin-echo",statusCode="200"}[2m])) by (instance, method) > 50
#        labels:
#          severity: warning
#    - name: fiat-okhttp
#      rules:
#      - alert: fiat-okhttp
#        annotations:
#          description: fiat needs attention
#          summary: fiat needs attention
#        expr: sum(rate(fiat:okhttp:requests__count_total{service="spin-fiat",statusCode="200"}[2m])) by (instance, method) /
#          sum(rate(fiat:okhttp:requests__total{service="spin-fiat",statusCode="200"}[2m])) by (instance, method) > 20
#        labels:
#          severity: warning
#    - name: gate-okhttp
#      rules:
#      - alert: gate-okhttp
#        annotations:
#          description: gate needs attention
#          summary: gate needs attention
#        expr: sum(rate(gate:okhttp:requests__count_total{service="spin-gate",statusCode="200"}[2m])) by (instance, method) /
#          sum(rate(gate:okhttp:requests__total{service="spin-gate",statusCode="200"}[2m])) by (instance, method) > 20 
#        labels:
#          severity: warning
#    - name: igor-okhttp
#      rules:
#      - alert: igor-okhttp
#        annotations:
#          description: igor needs attention
#          summary: igor needs attention
#        expr: sum(rate(igor:okhttp:requests__count_total{service="spin-igor",statusCode="200"}[2m])) by (instance, method) /
#          sum(rate(igor:okhttp:requests__total{service="spin-igor",statusCode="200"}[2m])) by (instance, method) > 30
#        labels:
#          severity: warning
#    - name: orca-okhttp
#      rules:
#      - alert: orca-okhttp
#        annotations:
#          description: orca needs attention
#          summary: orca needs attention
#        expr: sum(rate(orca:okhttp:requests__count_total{service="spin-orca",statusCode="200"}[2m])) by (instance, method) /
#          sum(rate(orca:okhttp:requests__total{service="spin-orca",statusCode="200"}[2m])) by (instance, method) > 100
#        labels:
#          severity: warning
#    - name: rosco-okhttp
#      rules:
#      - alert: rosco-okhttp
#        annotations:
#          description: roscoin needs attention
#          summary: rosco needs attention
#        expr: sum(rate(front50:okhttp:requests__count_total{service="spin-front50",statusCode="200"}[2m])) by (instance, method)/ 
#          sum(rate(front50:okhttp:requests__total{service="spin-front50",statusCode="200"}[2m])) by (instance, method) > 50
#        labels:
#          severity: warning