aerospike_rules.yml

groups:
- name: aerospike.rules
  rules:
  - alert: AerospikeExporterAgentDown
    expr: up{job="aerospike"} == 0
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Aerospike Prometheus exporter job {{ $labels.instance }} down"
      description: "{{ $labels.instance }} has been down for more than 30s."

  - alert: AerospikeNodeDown
    expr: aerospike_node_up{job="aerospike"} == 0
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Node {{ $labels.instance }} down"
      description: "{{ $labels.instance }} node is down."

- name: aerospike_aerospike.rules > NAMESPACE
  rules:
  - alert: NamespaceStopWrites
    expr: aerospike_namespace_stop_writes{job="aerospike" } == 1
    for: 30s
    labels:
      severity: critical
    annotations:
      summary: "Stop writes for {{ $labels.instance }}/{{ $labels.ns }}"
      description: "Used disk space for namespace {{ $labels.ns }} in node {{ $labels.instance }} is above stop writes limit. . <a href='localhost:7100' target='_blank'>namespace view </a>"

  - alert: AerospikeAllFlashAverageObjectsPerSprig
    expr:  ( ((aerospike_namespace_master_objects { job="aerospike"  }/4096)/aerospike_namespace_partition_tree_sprigs{ job="aerospike"  } ) and ignoring (index, sindex) ((aerospike_namespace_index_type_mounts_size_limit { job="aerospike"  }) or (aerospike_namespace_sindex_type_mounts_size_limit { job="aerospike"  }) ))> 50
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Average Objects per sprig in {{ $labels.instance  }}/{{ $labels.ns }}"
      description: "Average objects per sprig has been breached for namespace {{ $labels.ns }} in node {{ $labels.instance }}. "

  - alert: AerospikeAverageObjectsPerSprig
    expr:  ( ((aerospike_namespace_master_objects { job="aerospike"  }/4096)/aerospike_namespace_partition_tree_sprigs{ job="aerospike"  } ) unless ignoring (index, sindex) ((aerospike_namespace_index_type_mounts_size_limit { job="aerospike"  }) or (aerospike_namespace_sindex_type_mounts_size_limit { job="aerospike"  }) ))> 5000
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Average Objects per sprig in {{ $labels.instance  }}/{{ $labels.ns }}"
      description: "Average objects per sprig has been breached for namespace {{ $labels.ns }} in node {{ $labels.instance }}. "

  - alert: AerospikeIndexStageSizeWarn
    # Check here: https://docs.aerospike.com/reference/configuration#index-stage-size
    #  <128mb or >4gb -- send warn alert
    expr:  (aerospike_namespace_index_stage_size{job="aerospike" }>4000000000)  
    for: 1m
    labels:
      severity: warn
    annotations:
      summary: "Index stage size configuration is not configured according to documentation in {{ $labels.instance  }}/{{ $labels.ns }}"
      description: "Index stage size configuration is not configured according to documentation in {{ $labels.ns }} in node {{ $labels.instance }}. "

  - alert: AerospikeSIndexStageSizeWarn
    # Check here: https://docs.aerospike.com/reference/configuration#sindex-stage-size
    #  <128mb or >4gb -- send warn alert
    expr:  (aerospike_namespace_sindex_stage_size{job="aerospike" }>4000000000)  
    for: 1m
    labels:
      severity: warn
    annotations:
      summary: "SIndex stage size configuration is not configured according to documentation in {{ $labels.instance }}/{{ $labels.ns }}"
      description: "SIndex stage size configuration is not configured according to documentation in {{ $labels.ns }} in node {{ $labels.instance }}. "

  - alert: AerospikeIndexPressureDirtyMemoryWarn
    # Check here: https://docs.aerospike.com/reference/info#index-pressure
    expr: (((aerospike_namespace_index_pressure_dirty_memory{ job="aerospike"  })/(aerospike_namespace_index_pressure_total_memory{ job="aerospike"  })*100)>10000000) 
    for: 1m
    labels:
      severity: warn
    annotations:
      summary: "Dirty memory ratio against the total memory is above configured limit in node {{ $labels.instance }}"
      description: "Dirty memory ration against the total memory is above configured limit in node {{ $labels.instance }}"
      
  - alert: NamespaceDiskCloseToStopWrites
    expr: (aerospike_namespace_device_available_pct{job="aerospike" } - aerospike_namespace_storage_engine_min_avail_pct{job="aerospike" }) <= 10
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Close to stop writes for {{ $labels.instance }}/{{ $labels.ns }} due to device_available_pct"
      description: "device_available_pct for namespace {{ $labels.ns }} in node {{ $labels.instance }} is close to min-avail-pct (stop writes) limit."

  - alert: NamespaceMemoryCloseToStopWrites
    expr: (aerospike_namespace_stop_writes_pct{job="aerospike" } - (100 - aerospike_namespace_memory_free_pct{job="aerospike" })) <= 10
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Close to stop writes for {{ $labels.instance }}/{{ $labels.ns }} due to memory "
      description: "Free memory for namespace {{ $labels.ns }} in node {{ $labels.instance }} is close to stop writes limit."
  
  - alert: NamespacePmemCloseToStopWrites
    expr: (aerospike_namespace_pmem_available_pct{job="aerospike" } - aerospike_namespace_storage_engine_min_avail_pct{job="aerospike" }) <= 10
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Close to stop writes for {{ $labels.instance }}/{{ $labels.ns }} due to pmem_available_pct"
      description: "pmem_available_pct for namespace {{ $labels.ns }} in node {{ $labels.instance }} is close to min-avail-pct (stop writes) limit."
  
  - alert: NamespaceFreeMemoryCloseToStopWrites
    expr: (aerospike_namespace_stop_writes_sys_memory_pct{job="aerospike" } - scalar(100 - (aerospike_node_stats_system_free_mem_pct{job="aerospike" }))) <= 10
    for: 30s
    labels:
      severity: critical
    annotations:
      summary: "Close to stop writes for {{ $labels.instance }}/{{ $labels.ns }} due to memory"
      description: "Free memory for namespace {{ $labels.ns }} in node {{ $labels.instance }} is close to stop writes limit."

  - alert: ActiveProxies
    expr: (increase(aerospike_namespace_client_proxy_complete{job="aerospike" }[2m]) + increase(aerospike_namespace_client_proxy_timeout{job="aerospike" }[2m]) + increase(aerospike_namespace_client_proxy_error{job="aerospike" }[2m]) + increase(aerospike_namespace_batch_sub_proxy_complete{job="aerospike" }[2m]) + increase(aerospike_namespace_batch_sub_proxy_timeout{job="aerospike" }[2m]) + increase(aerospike_namespace_batch_sub_proxy_error{job="aerospike" }[2m])) > 0
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Node is doing proxy. Proxies can happen during cluster change / migrations or if there are any network issues."
      description: "Active proxies detected for {{ $labels.ns }} on node {{ $labels.instance }}"

  - alert: NamespaceSupervisorFallingBehind
    expr: aerospike_namespace_objects{job="aerospike"}>0 and aerospike_namespace_nsup_cycle_deleted_pct{job="aerospike" } > 1 # (Aerospike 6.3 and later) 
    for: 30s
    labels:
      severity: critical
    annotations:
      summary: "NSUP is falling behind and/or display the length of time the most recent NSUP cycle lasted"
      description: "There seems some lag falling behind and/or display the length of time the most recent NSUP cycle lasted {{ $labels.ns }} in node {{ $labels.instance }}"

  - alert: HwmBreached
    expr: aerospike_namespace_hwm_breached{job="aerospike" } == 1
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "High water mark breached for {{ $labels.instance }}/{{ $labels.ns }}"
      description: "high-water-disk-pct or high-water-memory-pct has been breached for namespace {{ $labels.ns }} in node {{ $labels.instance }}. Eviction may start to recover disk space."

  - alert: LowDeviceAvailWarning
    expr: aerospike_namespace_device_available_pct{job="aerospike" } < 55
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Device available warning for {{ $labels.instance }}/{{ $labels.ns }}"
      description: "Device available has dropped below 55% for namespace {{ $labels.ns }} in node {{ $labels.instance }}. May indicate that defrag is unable to keep up with the current load, and may result in stop writes if it continues to drop."

  - alert: LowDeviceAvailCritical
    expr: aerospike_namespace_device_available_pct{job="aerospike" } < 25
    for: 30s
    labels:
      severity: critical
    annotations:
      summary: "Device available critically low for {{ $labels.instance }}/{{ $labels.ns }}"
      description: "Device available has dropped below 25% for namespace {{ $labels.ns }} in node {{ $labels.instance }}. May indicate that defrag is unable to keep up with the current load, and may result in stop writes if it continues to drop."

  - alert: ClientTimeouts
    expr: rate(aerospike_namespace_client_read_timeout{job="aerospike" }[1m]) > 1 or rate(aerospike_namespace_client_write_timeout{job="aerospike" }[1m]) > 1 or rate(aerospike_namespace_client_tsvc_timeout{job="aerospike" }[1m]) > 1
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: "Client transactions are timing out"
      description: "Client connections timing out at a rate greater than 1/s. Timeouts can occur during network issues or resource contention on the client and/or server nodes."

  - alert: LowMemoryNamespaceWarning
    expr: aerospike_namespace_memory_free_pct{job="aerospike" } < 20
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Memory available warning for {{ $labels.instance }}/{{ $labels.ns }}"
      description: "Memory free has dropped below 20% for namespace {{ $labels.ns }} in node {{ $labels.instance }}. May indicate a need to reduce the object count or increase capacity."

  - alert: LowMemoryNamespaceCritical
    expr: aerospike_namespace_memory_free_pct{job="aerospike" } < 15
    for: 30s
    labels:
      severity: critical
    annotations:
      summary: "Memory available critically low for {{ $labels.instance }}/{{ $labels.ns }}"
      description: "Memory free has dropped below 15% for namespace {{ $labels.ns }} in node {{ $labels.instance }}. May indicate a need to reduce the object count or increase capacity."

  - alert: DeviceWriteQWarning
    expr: aerospike_namespace_storage_engine_device_write_q{job="aerospike" } > 1
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Device write queue high for {{ $labels.instance }}/{{ $labels.ns }}/{{ $labels.device_index }}"
      description: "Device write queue is greater than 1 for namespace {{ $labels.ns }} on device {{ $labels.device_index }} in node {{ $labels.instance }}. May indicate underperforming storage subsystem or hotkeys."

  - alert: ShadowDeviceWriteQWarning
    expr: aerospike_namespace_storage_engine_device_shadow_write_q{job="aerospike" } > 1
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Shadow device write queue high for {{ $labels.instance }}/{{ $labels.ns }}/{{ $labels.device_index }}"
      description: "Shadow device write queue is greater than 1 for namespace {{ $labels.ns }} on device {{ $labels.device_index }} in node {{ $labels.instance }}. May indicate underperforming storage subsystem or hotkeys."

  - alert: DeviceDefragQWarning
    expr: aerospike_namespace_storage_engine_device_defrag_q{job="aerospike" }> 1000
    for: 5m
    labels:
      severity: warn
    annotations:
      summary: "Device defrag queue high for {{ $labels.instance }}/{{ $labels.ns }}/{{ $labels.device_index }}"
      description: "Device defrag queue has been above 1000 for more than 5m for namespace {{ $labels.ns }} on device {{ $labels.device_index }} in node {{ $labels.instance }}. May indicate underperforming storage subsystem or hotkeys."

  - alert: ClockSkewStopWrites
    expr: aerospike_namespace_clock_skew_stop_writes{job="aerospike" } == 1
    for: 30s
    labels:
      severity: critical
    annotations:
      summary: "Clock skew stop writes"
      description: "Clock has skewed for namespace {{ $labels.ns }} in node {{ $labels.instance }}"

  - alert: UnavailablePartitions
    expr: aerospike_namespace_unavailable_partitions{job="aerospike" } > 0
    for: 30s
    labels:
      severity: critical
    annotations:
      summary: "Some partitions are inaccessible, and roster nodes are missing from the cluster."
      description: "Some partitions are not available for namespace {{ $labels.ns }} on node {{ $labels.instance }}. Check for network issues and make sure the cluster forms properly."

  - alert: DeadPartitions
    expr: aerospike_namespace_dead_partitions{job="aerospike" } > 2
    for: 30s
    labels:
      severity: critical
    annotations:
      summary: "There are unavailable partition, but all roster nodes are present in the cluster."
      description: "Some partitions are dead for namespace {{ $labels.ns }} on node {{ $labels.instance }}. Greater than replication-factor number nodes had an unclean shutdown, and there may be data loss. Will require the use of the revive command to make the partitions available again."
  
  - alert: NamespaceDataCloseToStopWrites
    expr: (aerospike_namespace_data_avail_pct{job="aerospike"  } - aerospike_namespace_storage_engine_stop_writes_avail_pct{job="aerospike"  }) <= 10
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Close to stop writes for {{ $labels.instance }}/{{ $labels.ns }} due to data_avail_pct"
      description: "data_avail_pct for namespace {{ $labels.ns }} in node {{ $labels.instance }} is close to stop-writes-avail-pct limit."
  
  - alert: LowDataAvailWarning
    expr: aerospike_namespace_data_avail_pct{job="aerospike"  } < 55
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Device available warning for {{ $labels.instance }}/{{ $labels.ns }}"
      description: "Device available has dropped below 55% for namespace {{ $labels.ns }} in node {{ $labels.instance }}. May indicate that defrag is unable to keep up with the current load, and may result in stop writes if it continues to drop."
  
  - alert: LowDataAvailCritical
    expr: aerospike_namespace_data_avail_pct{job="aerospike"  } < 25
    for: 30s
    labels:
      severity: critical
    annotations:
      summary: "Device available critically low for {{ $labels.instance }}/{{ $labels.ns }}"
      description: "Device available has dropped below 25% for namespace {{ $labels.ns }} in node {{ $labels.instance }}. May indicate that defrag is unable to keep up with the current load, and may result in stop writes if it continues to drop."
  
  - alert: HighDataUseNamespaceWarning
    expr: aerospike_namespace_data_used_pct{job="aerospike" , storage_engine="memory" } > 80
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Data utilization warning for {{ $labels.instance }}/{{ $labels.ns }}"
      description: "Data used has crossed above 80% for namespace {{ $labels.ns }} in node {{ $labels.instance }}. May indicate a need to reduce the object count or increase capacity."
  
  - alert: HighDataUseNamespaceCritical
    expr: aerospike_namespace_data_used_pct{job="aerospike" , storage_engine="memory" } > 85
    for: 30s
    labels:
      severity: critical
    annotations:
      summary: "Data utilization critically high for {{ $labels.instance }}/{{ $labels.ns }}"
      description: "Data used has crossed above 85% for namespace {{ $labels.ns }} in node {{ $labels.instance }}. May indicate a need to reduce the object count or increase capacity."

- name: aerospike_aerospike.rules > NODE
  rules:
  - alert: PrometheusNodeExporterNotPresent
    expr: absent(node_cpu_seconds_total) == 1
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: " Prometheus Node Exporter is not configured "
      description: " Prometheus Node Exporter is not configured in {{ $labels.instance }} "

  - alert: BestPracticesFailure
    expr: aerospike_node_stats_failed_best_practices{job="aerospike" } > 0
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: " Best Practices check failed on {{ $labels.instance }} in cluster {{ $labels.cluster_name }}"
      description: " Best Practices check failed on {{ $labels.instance }} in cluster {{ $labels.cluster_name }}"

  - alert: ClusterSize
    expr: aerospike_node_stats_cluster_size{job="aerospike" } < 3
    for: 30s
    labels:
      severity: critical
    annotations:
      summary: "Cluster size lower than expected"
      description: "Cluster size mismatch for node {{ $labels.instance }}"

  - alert: ClientConnectionsWarning
    expr: aerospike_node_stats_client_connections{job="aerospike" } > 11
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Client connections warning"
      description: "Client connections are greater than 11. Connections will fail if they exceed proto-fd-max."
  - alert: ClientConnectionsCritical
    expr: aerospike_node_stats_client_connections{job="aerospike" } > 10000
    for: 30s
    labels:
      severity: critical
    annotations:
      summary: "Client connections critical"
      description: "Client connections are greater than expected peak of 10000."

  - alert: ClientConnectionChurn
    expr: rate(aerospike_node_stats_client_connections_opened{job="aerospike" }[1m]) > 100 or rate(aerospike_node_stats_client_connections_closed{job="aerospike" }[1m]) > 100
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: "Clients are churning connections at a high rate"
      description: "Client connections are being opened or closed at a rate greater than 100/s. Connection churn can increase latency and client timeouts which in turn cause the client to open more connections."

  - alert: ClockSkewWarning
    expr: aerospike_node_stats_cluster_clock_skew_ms{job="aerospike" } > 2000
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Cluster clock skew warning{"
      description: "Current maximum clock skew between nodes - will trigger stop writes when it exceeds 2000 seconds if nsup-period is non-zero."

  - alert: ClockSkewCritical
    expr: aerospike_node_stats_cluster_clock_skew_ms{job="aerospike" } > 20000
    for: 30s
    labels:
      severity: critical
    annotations:
      summary: "Cluster clock skew critical alert"
      description: "Current maximum clock skew between nodes - will trigger stop writes when it exceeds 20000 if nsup-period is non-zero."

  - alert: LowMemorySystemWarning
    expr: aerospike_node_stats_system_free_mem_pct{job="aerospike" } < 20
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Memory available warning for {{ $labels.instance }}"
      description: "Total memory free has dropped below 20% for node {{ $labels.instance }}."

  - alert: LowMemorySystemCritical
    expr: aerospike_node_stats_system_free_mem_pct{job="aerospike" } < 10
    for: 30s
    labels:
      severity: critical
    annotations:
      summary: "Memory available critically low for {{ $labels.instance }}"
      description: "Total memory free has dropped below 10% for node {{ $labels.instance }}."

  - alert: HeapEfficiencyWarning
    #expr: aerospike_node_stats_heap_efficiency_pct{job="aerospike" } < 60
    expr: (100 - aerospike_node_stats_system_free_mem_pct{job="aerospike" }) > 70 and aerospike_node_stats_heap_efficiency_pct{job="aerospike" } < 60
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Heap efficiency warning for {{ $labels.instance }}"
      description: "Heap efficiency for node for {{ $labels.instance }} has dropped below 60%."

  - alert: RwInProgressWarning
    expr: aerospike_node_stats_rw_in_progress{job="aerospike" }> 100
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Read/write queue too high for {{ $labels.instance }}/{{ $labels.ns }}/{{ $labels.device_index }}"
      description: "Read/write queue is greater than 100 for namespace {{ $labels.ns }} on device {{ $labels.device_index }} in node {{ $labels.instance }}. May indicate underperforming storage subsystem or hotkeys."

- name: aerospike_aerospike.rules > SET
  rules:
  - alert: pre7x_NamespaceSetQuotaWarning
    expr: (((aerospike_sets_device_data_bytes{job="aerospike" } + aerospike_sets_memory_data_bytes{job="aerospike" }) / (aerospike_sets_stop_writes_size{job="aerospike" } != 0)) * 100) > 80
    for: 30s
    labels:
      severity: warn
    annotations:
      description: "Nearing memory quota for {{ $labels.set }} in namespace {{ $labels.ns }} in node {{ $labels.instance }}."
      summary: "One of your nodes is at  % of the quota you have configured on the set."
  
  - alert: pre7x_NamespaceSetQuotaAlertCritical
    expr: (((aerospike_sets_device_data_bytes{job="aerospike" } + aerospike_sets_memory_data_bytes{job="aerospike" }) / (aerospike_sets_stop_writes_size{job="aerospike" } != 0)) * 100) > 99
    for: 30s
    labels:
      severity: critical
    annotations:
      description: "At or Above memory quota for {{ $labels.set }} in namespace {{ $labels.ns }} in node {{ $labels.instance }}."
      summary: "One of your nodes is at % of the quota you have configured on the set."
  
  - alert: NamespaceSetQuotaWarning
    expr: (((aerospike_sets_data_used_bytes{job="aerospike"  } ) / (aerospike_sets_stop_writes_size{job="aerospike"  } != 0)) * 100) > 80
    for: 30s
    labels:
      severity: warn
    annotations:
      description: "Nearing memory quota for {{ $labels.set }} in namespace {{ $labels.ns }} in node {{ $labels.instance }}."
      summary: "One of your nodes is at % of the quota you have configured on the set."
    
  - alert: NamespaceSetQuotaAlertCritical
    expr: (((aerospike_sets_data_used_bytes{job="aerospike"  } ) / (aerospike_sets_stop_writes_size{job="aerospike"  } != 0)) * 100) > 99
    for: 30s
    labels:
      severity: critical
    annotations:
      description: "At or Above memory quota for {{ $labels.set }} in namespace {{ $labels.ns }} in node {{ $labels.instance }}."
      summary: "One of your nodes is at % of the quota you have configured on the set."
            
- name: aerospike_aerospike.rules > LATENCIES
  rules:
  - alert: ReadLatencyP95Warning
    expr: histogram_quantile(0.95, (aerospike_latencies_read_ms_bucket{job="aerospike" })) > 2
    for: 2m
    labels:
      severity: warn
    annotations:
      summary: "Read latency breached for {{ $labels.ns }} on {{ $labels.instance }}"
      description: "95th percentile read latency breached 2ms for namespace {{ $labels.ns }} on host {{ $labels.instance }}."

  - alert: ReadLatencyP99Warning
    expr: histogram_quantile(0.99, (aerospike_latencies_read_ms_bucket{job="aerospike" })) > 4
    for: 2m
    labels:
      severity: warn
    annotations:
      summary: "Read latency breached for {{ $labels.ns }} on {{ $labels.instance}}"
      description: "99th percentile read latency breached 4ms for namespace {{ $labels.ns }} on host {{ $labels.instance }}."

  - alert: ReadLatencyP999Warning
    expr: histogram_quantile(0.999, (aerospike_latencies_read_ms_bucket{job="aerospike" })) > 16
    for: 2m
    labels:
      severity: warn
    annotations:
      summary: "Read latency breached for {{ $labels.ns }} on {{ $labels.instance }}"
      description: "99.9th percentile read latency breached 16ms for namespace {{ $labels.ns }} on host {{ $labels.instance }}."

  - alert: WriteLatencyP95Warning
    expr: histogram_quantile(0.95, (aerospike_latencies_write_ms_bucket{job="aerospike" })) > 4
    for: 2m
    labels:
      severity: warn
    annotations:
      summary: "Write latency breached for {{ $labels.ns }} on {{ $labels.instance }}"
      description: "95th percentile write latency breached 4ms for namespace {{ $labels.ns }} on host {{ $labels.instance }}."

  - alert: WriteLatencyP99Warning
    expr: histogram_quantile(0.99, (aerospike_latencies_write_ms_bucket{job="aerospike" })) > 16
    for: 2m
    labels:
      severity: warn
    annotations:
      summary: "Write latency breached for {{ $labels.ns }} on {{ $labels.instance }}"
      description: "99th percentile write latency breached 16ms for namespace {{ $labels.ns }} on host {{ $labels.instance }}."

  - alert: WriteLatencyP999Warning
    expr: histogram_quantile(0.999, (aerospike_latencies_write_ms_bucket{job="aerospike" })) > 64
    for: 2m
    labels:
      severity: warn
    annotations:
      summary: "Write latency breached for {{ $labels.ns }} on {{ $labels.instance }}"
      description: "99.9th percentile write latency breached 64ms for namespace {{ $labels.ns }} on host {{ $labels.instance }}."


- name: aerospike_aerospike.rules > XDR
  rules:

  - alert: XDRTimelag
    expr: aerospike_xdr_lag{job="aerospike" } > 5
    for: 2m
    labels:
      severity: warn
    annotations:
      summary: "XDR lag for namespace {{ $labels.ns }} exceeding 5 second(s) from node {{ $labels.instance }} to DC {{ $labels.dc }}"
      description: "XDR lag may be due to network connectivity issues, inability for the source to keep up with incoming writes, or write failures at the destination."
  - alert: XDRAbandonedRecords
    expr: rate(aerospike_xdr_abandoned{job="aerospike" }[1m]) > 0
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "Abandoned records detected for XDR on node {{ $labels.instance }} to DC {{ $labels.dc }}"
      description: "Records abandoned at a destination cluster may indicate a configuration mismatch for the namespace between source and destination."
  - alert: XDRRetryNoNode
    expr: rate(aerospike_xdr_retry_no_node{job="aerospike" }[1m]) > 0
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "XDR retries occuring on node {{ $labels.instance }} to DC {{ $labels.dc }} due to unknown master node destination"
      description: "XDR cannot determine which destination node is the master."

  - alert: XDRRetryConnReset
    expr: rate(aerospike_xdr_retry_conn_reset{job="aerospike" }[1m]) > 2
    for: 2m
    labels:
      severity: warn
    annotations:
      summary: "Rate of XDR connection resets greater than 2/s from {{ $labels.instance }} to DC {{ $labels.dc }} "
      description: "XDR retries occuring due to due to timeouts, network problems, or destination node restarts."

  - alert: XDRRetryDest
    expr: rate(aerospike_xdr_retry_dest{job="aerospike" }[1m]) > 5
    for: 2m
    labels:
      severity: warn
    annotations:
      summary: "Increase in XDR write retries is greater than 5/s from {{ $labels.instance }} to DC {{ $labels.dc }}"
      description: "XDR retries due to errors returned by the destination node, u.e. key busy or device overload."

  - alert: XDRLatencyWarning
    expr: aerospike_xdr_latency_ms{job="aerospike" } > 100
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "XDR latency above 100ms from {{ $labels.instance }} to DC {{ $labels.dc }}"
      description: "Network latency between XDR source and destination over the last 30s is higher than expected."

  - alert: XDRLap
    expr: aerospike_xdr_lap_us{job="aerospike" } > 75000
    for: 30s
    labels:
      severity: warn
    annotations:
      summary: "XDR lap time greater than 75000 microseconds from {{ $labels.instance }} to DC {{ $labels.dc }}"
      description: "The XDR processing cycle time (lap_us) is approaching the configured period-ms value."

  - alert: XDRRecoveries
    expr: increase(aerospike_xdr_recoveries{job="aerospike" }[1m]) > 0
    for:  2m
    labels:
      severity: critical
    annotations:
      summary: "XDR recoveries increasing on {{ $labels.instance }} to DC {{ $labels.dc }}"
      description: "XDR recoveries happen during reind or may indicate that the in-memory transaction queue is full (the transaction-queue-limit may be too small)."