---
conf:
  prometheus:
    rules:
      elasticsearch:
        groups:
        - name: elasticsearch.alerting_rules
          rules:
          - alert: prom_exporter_elasticsearch_unavailable
            expr: avg_over_time(up{job="elasticsearch-exporter"}[5m]) == 0
            for: 5m
            labels:
              severity: warning
            annotations:
              description: Elasticsearch exporter is not collecting metrics or is not available for past 10 minutes
              title: Elasticsearch exporter is not collecting metrics or is not available
          - alert: es_high_process_open_files_count
            expr: sum(elasticsearch_process_open_files_count) by (host) > 64000
            for: 10m
            labels:
              severity: warning
            annotations:
              description: 'Elasticsearch at {{ $labels.host }} has more than 64000 process open file count.'
              summary: 'Elasticsearch has a very high process open file count.'
          - alert: es_high_process_cpu_percent
            expr: elasticsearch_process_cpu_percent > 95
            for: 10m
            labels:
              severity: warning
            annotations:
              description: 'Elasticsearch at {{ $labels.instance }} has high process cpu percent of {{ $value }}.'
              summary: 'Elasticsearch process cpu usage is more than 95 percent.'
          - alert: es_fs_usage_high
            expr: (100 * (elasticsearch_filesystem_data_size_bytes - elasticsearch_filesystem_data_free_bytes) / elasticsearch_filesystem_data_size_bytes) > 80
            for: 10m
            labels:
              severity: warning
            annotations:
              description: 'Elasticsearch at {{ $labels.instance }} has filesystem usage of {{ $value }}.'
              summary: 'Elasticsearch filesystem usage is high.'
          - alert: es_unassigned_shards
            expr: elasticsearch_cluster_health_unassigned_shards > 0
            for: 10m
            labels:
              severity: warning
            annotations:
              description: 'Elasticsearch has {{ $value }} unassigned shards.'
              summary: 'Elasticsearch has unassigned shards and hence a unhealthy cluster state.'
          - alert: es_cluster_health_timed_out
            expr: elasticsearch_cluster_health_timed_out > 0
            for: 10m
            labels:
              severity: warning
            annotations:
              description: 'Elasticsearch cluster health status call timedout {{ $value }} times.'
              summary: 'Elasticsearch cluster health status calls are timing out.'
          - alert: es_cluster_health_status_alert
            expr: (sum(elasticsearch_cluster_health_status{color="green"})*2)+sum(elasticsearch_cluster_health_status{color="yellow"}) < 2
            for: 10m
            labels:
              severity: warning
            annotations:
              description: 'Elasticsearch cluster health status is {{ $value }}, not 2 (green). One or more shards or replicas are unallocated.'
              summary: 'Elasticsearch cluster health status is not green.'
          - alert: es_cluster_health_too_few_nodes_running
            expr: elasticsearch_cluster_health_number_of_nodes < 3
            for: 10m
            labels:
              severity: warning
            annotations:
              description: 'There are only {{$value}} < 3 ElasticSearch nodes running'
              summary: 'ElasticSearch running on less than 3 nodes'
          - alert: es_cluster_health_too_few_data_nodes_running
            expr: elasticsearch_cluster_health_number_of_data_nodes < 3
            for: 10m
            labels:
              severity: warning
            annotations:
              description: 'There are only {{$value}} < 3 ElasticSearch data nodes running'
              summary: 'ElasticSearch running on less than 3 data nodes'
          - alert: es_cluster_health_too_few_data_nodes_running
            expr: elasticsearch_cluster_health_number_of_data_nodes < 3
            for: 10m
            labels:
              severity: warning
            annotations:
              description: 'There are only {{$value}} < 3 ElasticSearch data nodes running'
              summary: 'ElasticSearch running on less than 3 data nodes'
      fluentd:
        groups:
        - name: fluentd.alerting_rules
          rules:
          - alert: prom_exporter_fluentd_unavailable
            expr: avg_over_time(up{job="fluentd-daemonset-exporter"}[5m]) == 0
            for: 5m
            labels:
              severity: warning
            annotations:
              description: Fluentd exporter is not collecting metrics or is not available for past 10 minutes
              title: Fluentd exporter is not collecting metrics or is not available
...