Merge "Add the ability to enable or disable rollups / indexes"

elk_metrics_6x/createElasticIndexes.yml

@@ -28,6 +28,43 @@
     - role: elastic_retention
 
   post_tasks:
+    - name: Create beat indexes
+      uri:
+        url: http://127.0.0.1:9200/{{ item.name }}
+        method: PUT
+        body: "{{ item.index_options | to_json }}"
+        status_code: 200,400
+        body_format: json
+      register: elk_indexes
+      until: elk_indexes is success
+      retries: 3
+      delay: 30
+      with_items: |-
+        {% set beat_indexes = [] %}
+        {% for key, value in elastic_beat_retention_policy_hosts.items() %}
+        {%   if ((value.hosts | length) > 0) and (value.make_index | default(false) | bool) %}
+        {%
+               set _index = {
+                 'name': key,
+                 'index_options': {
+                   'settings': {
+                     'index': {
+                       'codec': 'best_compression',
+                       'mapping': {
+                         'total_fields': {
+                           'limit': '10000'
+                         }
+                       },
+                       'refresh_interval': elastic_refresh_interval
+                     }
+                   }
+                 }
+               }
+        %}
+        {%     set _ = beat_indexes.append(_index) %}
+        {%   endif %}
+        {% endfor %}
+        {{ beat_indexes }}
     - name: Create basic indexes
       uri:
         url: http://127.0.0.1:9200/{{ item.name }}
@@ -40,15 +77,6 @@
       retries: 3
       delay: 30
       with_items:
-        - name: "osprofiler-notifications"
-          index_options:
-            settings:
-              index:
-                codec: "best_compression"
-                mapping:
-                  total_fields:
-                    limit: "10000"
-                refresh_interval: "{{ elastic_refresh_interval }}"
         - name: "_all/_settings?preserve_existing=true"
           index_options:
             index.queries.cache.enabled: "true"
@@ -120,8 +148,101 @@
       delay: 30
       vars:
         index_option:
-          template: ".monitoring-*"
+          template: ".monitoring*"
           order: 1
           settings:
             number_of_replicas: "{{ elasticsearch_number_of_replicas | int }}"
             number_of_shards: "{{ ((elasticsearch_number_of_replicas | int) * 2) + 1 }}"
+
+    - name: Create custom skydive index template
+      uri:
+        url: http://127.0.0.1:9200/_template/skydive
+        method: PUT
+        body: "{{ index_option | to_json }}"
+        status_code: 200
+        body_format: json
+      register: create_basicIndexTemplate
+      until: create_basicIndexTemplate is success
+      retries: 3
+      delay: 30
+      vars:
+        index_option:
+          template: "skydive*"
+          order: 1
+          settings:
+            number_of_replicas: "{{ elasticsearch_number_of_replicas | int }}"
+            number_of_shards: "{{ ((elasticsearch_number_of_replicas | int) * 2) + 1 }}"
+
+
+- name: Create/Setup known indexes in Kibana
+  hosts: kibana
+  become: true
+  vars_files:
+    - vars/variables.yml
+
+  environment: "{{ deployment_environment_variables | default({}) }}"
+
+  roles:
+    - role: elastic_retention
+
+  post_tasks:
+    - name: Create kibana indexe patterns
+      uri:
+        url: "http://127.0.0.1:5601/api/saved_objects/index-pattern/{{ item.name }}"
+        method: POST
+        body: "{{ item.index_options | to_json }}"
+        status_code: 200,409
+        body_format: json
+        headers:
+          Content-Type: "application/json"
+          kbn-xsrf: "{{ inventory_hostname | to_uuid }}"
+      with_items: |-
+        {% set beat_indexes = [] %}
+        {% for key, value in elastic_beat_retention_policy_hosts.items() %}
+        {%   if (value.hosts | length) > 0 %}
+        {%
+               set _index = {
+                 'name': key,
+                 'index_options': {
+                   'attributes': {
+                     'title': (key ~ '*')
+                   }
+                 }
+               }
+        %}
+        {%     if value.timeFieldName is defined %}
+        {%       set _ = _index.index_options.attributes.__setitem__('timeFieldName', (value.timeFieldName | string)) %}
+        {%     endif %}
+        {%     set _ = beat_indexes.append(_index) %}
+        {%   endif %}
+        {% endfor %}
+        {% set _ = beat_indexes.append({'name': 'default', 'index_options': {'attributes': {'title': '*'}}}) %}
+        {{ beat_indexes }}
+      register: kibana_indexes
+      until: kibana_indexes is success
+      retries: 6
+      delay: 30
+      run_once: true
+
+    - name: Create basic indexes
+      uri:
+        url: "http://127.0.0.1:5601/api/kibana/settings/defaultIndex"
+        method: POST
+        body: "{{ item.index_options | to_json }}"
+        status_code: 200
+        body_format: json
+        headers:
+          Content-Type: "application/json"
+          kbn-xsrf: "{{ inventory_hostname | to_uuid }}"
+      with_items:
+        - name: "default"
+          index_options:
+            value: "default"
+      register: kibana_indexes
+      until: kibana_indexes is success
+      retries: 6
+      delay: 30
+      run_once: true
+
+  tags:
+    - server-install

elk_metrics_6x/installAPMserver.yml

@@ -26,7 +26,8 @@
     - role: elastic_apm_server
     - role: elastic_rollup
       index_name: apm
-
+      when:
+        - elastic_create_rollup | bool
   tags:
     - apm-server
 
@@ -44,6 +45,7 @@
   roles:
     - role: elastic_rollup
       index_name: apm
-
+      when:
+        - elastic_create_rollup | bool
   tags:
     - apm-server

elk_metrics_6x/installAuditbeat.yml

@@ -42,7 +42,8 @@
   roles:
     - role: elastic_rollup
       index_name: auditbeat
-
+      when:
+        - elastic_create_rollup | bool
   tags:
     - auditbeat
 

elk_metrics_6x/installFilebeat.yml

@@ -42,7 +42,8 @@
   roles:
     - role: elastic_rollup
       index_name: filebeat
-
+      when:
+        - elastic_create_rollup | bool
   tags:
     - filebeat
 

elk_metrics_6x/installHeartbeat.yml

@@ -56,7 +56,8 @@
   roles:
     - role: elastic_rollup
       index_name: heartbeat
-
+      when:
+        - elastic_create_rollup | bool
   tags:
     - heartbeat
 

elk_metrics_6x/installJournalbeat.yml

@@ -96,6 +96,7 @@
   roles:
     - role: elastic_rollup
       index_name: journalbeat
-
+      when:
+        - elastic_create_rollup | bool
   tags:
     - journalbeat

elk_metrics_6x/installKibana.yml

@@ -22,48 +22,5 @@
   roles:
     - role: elastic_kibana
 
-  post_tasks:
-    - name: Create basic indexes
-      uri:
-        url: "http://127.0.0.1:5601/api/saved_objects/index-pattern/{{ item.name }}"
-        method: POST
-        body: "{{ item.index_options | to_json }}"
-        status_code: 200,409
-        body_format: json
-        headers:
-          Content-Type: "application/json"
-          kbn-xsrf: "{{ inventory_hostname | to_uuid }}"
-      with_items:
-        - name: "*"
-          index_options:
-            attributes:
-              title: "*"
-              timeFieldName: "@timestamp"
-      register: kibana_indexes
-      until: kibana_indexes is success
-      retries: 6
-      delay: 30
-      run_once: true
-
-    - name: Create basic indexes
-      uri:
-        url: "http://127.0.0.1:5601/api/kibana/settings/defaultIndex"
-        method: POST
-        body: "{{ item.index_options | to_json }}"
-        status_code: 200
-        body_format: json
-        headers:
-          Content-Type: "application/json"
-          kbn-xsrf: "{{ inventory_hostname | to_uuid }}"
-      with_items:
-        - name: "*"
-          index_options:
-            value: "*"
-      register: kibana_indexes
-      until: kibana_indexes is success
-      retries: 6
-      delay: 30
-      run_once: true
-
   tags:
     - server-install

elk_metrics_6x/installMetricbeat.yml

@@ -42,7 +42,8 @@
   roles:
     - role: elastic_rollup
       index_name: metricbeat
-
+      when:
+        - elastic_create_rollup | bool
   tags:
     - metricbeat
 

elk_metrics_6x/installPacketbeat.yml

@@ -42,7 +42,8 @@
   roles:
     - role: elastic_rollup
       index_name: packetbeat
-
+      when:
+        - elastic_create_rollup | bool
   tags:
     - packetbeat
 

elk_metrics_6x/roles/elastic_journalbeat/tasks/systemd.general-overrides.conf.j2

@@ -1 +0,0 @@
-../../../templates/systemd.general-overrides.conf.j2

elk_metrics_6x/roles/elastic_retention/defaults/main.yml

@@ -70,35 +70,45 @@ elastic_index_retention_algorithm: default
 
 elastic_beat_retention_policy_hosts:
   logstash:
+    make_index: true
     weight: 1
     hosts: "{{ groups['elastic-logstash'] | default([]) }}"
   apm:
+    make_index: true
+    timeFieldName: '@timestamp'
     weight: 1
     hosts: "{{ groups['apm-server'] | default([]) }}"
   auditbeat:
+    timeFieldName: '@timestamp'
     weight: 10
     hosts: "{{ groups['hosts'] | default([]) }}"
   filebeat:
+    timeFieldName: '@timestamp'
     weight: 10
     hosts: "{{ groups['hosts'] | default([]) }}"
   syslog:
+    make_index: true
     weight: 1
     hosts: "{{ groups['hosts'] | default([]) }}"
   heartbeat:
+    timeFieldName: '@timestamp'
     weight: 1
     hosts: "{{ groups['kibana'][:3] | default([]) }}"
   journalbeat:
+    timeFieldName: '@timestamp'
     weight: 3
-    hosts: "{{ groups['all'] | default([]) }}"
+    hosts: "{{ groups['hosts'] | default([]) }}"
   metricbeat:
+    timeFieldName: '@timestamp'
     weight: 2
     hosts: "{{ groups['all'] | default([]) }}"
   packetbeat:
+    timeFieldName: '@timestamp'
     weight: 1
     hosts: "{{ groups['hosts'] | default([]) }}"
   skydive:
     weight: 1
-    hosts: "{{ groups['skydive_analyzers'] | default([]) }}"
+    hosts: "{{ (((groups['skydive_analyzers'] | default([])) | length) > 0) | ternary((groups['hosts'] | default([])), []) }}"
 
 # Refresh the elasticsearch retention policy local facts.
 elastic_retention_refresh: false

elk_metrics_6x/vars/variables.yml

@@ -11,6 +11,7 @@ q_storage: "{{ (ansible_processor_count | int) * (ansible_processor_threads_per_
 apm_port: 8200
 elastic_port: 9200
 elastic_hap_port: 9201
+elastic_create_rollup: false
 logstash_beat_input_port: 5044
 logstash_syslog_input_port: 5140
 logstash_syslog_input_mode: udp