Allow beat processors to be defined through configuration data

Processors for each of the beats are currently defined per template and
can be configured directly via template or once deployed. This change
allows processors for all or a subset of beats to be configured through
role data or via an overlay inventory.

Change-Id: I71fc96611082555b43fd0f505219c42c890464ef

elk_metrics_7x/roles/elastic_auditbeat/defaults/main.yml

@@ -25,3 +25,6 @@ ilm_policy: "{{ (auditbeat_ilm_policy | default(default_ilm_policy)) | default({
 ilm_policy_template: "{{ auditbeat_ilm_policy_template | default('') }}"
 ilm_policy_filename: "{{ (auditbeat_ilm_policy_filename | default(default_ilm_policy_filename)) | default('') }}"
 ilm_policy_file_location: "{{ (auditbeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}"
+
+# beat processors. Empty dictionary sets `add_host_metadata: ~`
+processors: {}

elk_metrics_7x/roles/elastic_auditbeat/templates/auditbeat.yml.j2

@@ -266,6 +266,7 @@ auditbeat.modules:
 #max_procs:
 
 #================================ Processors ===================================
+{{ elk_macros.beat_processors(processors) }}
 
 # Processors are used to reduce the number of fields in the exported event or to
 # enhance the event with external metadata. This section defines a list of
@@ -375,8 +376,6 @@ auditbeat.modules:
 #    max_depth: 1
 #    target: ""
 #    overwrite_keys: false
-processors:
-- add_host_metadata: ~
 
 #============================= Elastic Cloud ==================================
 

elk_metrics_7x/roles/elastic_filebeat/defaults/main.yml

@@ -293,3 +293,6 @@ ilm_policy: "{{ (filebeat_ilm_policy | default(default_ilm_policy)) | default({}
 ilm_policy_template: "{{ filebeat_ilm_policy_template | default('') }}"
 ilm_policy_filename: "{{ (filebeat_ilm_policy_filename | default(default_ilm_policy_filename)) | default('') }}"
 ilm_policy_file_location: "{{ (filebeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}"
+
+# beat processors. Empty dictionary sets `add_host_metadata: ~`
+processors: {}

elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2

@@ -765,6 +765,7 @@ filebeat.inputs:
 #max_procs:
 
 #================================ Processors ===================================
+{{ elk_macros.beat_processors(processors) }}
 
 # Processors are used to reduce the number of fields in the exported event or to
 # enhance the event with external metadata. This section defines a list of
@@ -852,8 +853,8 @@ filebeat.inputs:
 #
 # The following example enriches each event with host metadata.
 #
-processors:
-- add_host_metadata: ~
+#processors:
+#- add_host_metadata: ~
 #   netinfo.enabled: false
 #
 # The following example enriches each event with process metadata using

elk_metrics_7x/roles/elastic_heartbeat/defaults/main.yml

@@ -25,3 +25,6 @@ ilm_policy: "{{ (heartbeat_ilm_policy | default(default_ilm_policy)) | default({
 ilm_policy_template: "{{ heartbeat_ilm_policy_template | default('') }}"
 ilm_policy_filename: "{{ (heartbeat_ilm_policy_filename | default(heartbeat_ilm_policy_filename)) | default('') }}"
 ilm_policy_file_location: "{{ (heartbeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}"
+
+# beat processors. Empty dictionary sets `add_host_metadata: ~`
+processors: {}

elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2

@@ -407,6 +407,7 @@ heartbeat.scheduler:
 #max_procs:
 
 #================================ Processors ===================================
+{{ elk_macros.beat_processors(processors) }}
 
 # Processors are used to reduce the number of fields in the exported event or to
 # enhance the event with external metadata. This section defines a list of
@@ -516,8 +517,6 @@ heartbeat.scheduler:
 #    max_depth: 1
 #    target: ""
 #    overwrite_keys: false
-processors:
-- add_host_metadata: ~
 
 #============================= Elastic Cloud ==================================
 

elk_metrics_7x/roles/elastic_journalbeat/defaults/main.yml

@@ -25,3 +25,6 @@ ilm_policy: "{{ (journalbeat_ilm_policy | default(default_ilm_policy)) | default
 ilm_policy_template: "{{ journalbeat_ilm_policy_template | default('') }}"
 ilm_policy_filename: "{{ (journalbeat_ilm_policy_filename | default(default_ilm_policy_filename)) | default('') }}"
 ilm_policy_file_location: "{{ (journalbeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}"
+
+# beat processors. Empty dictionary sets `add_host_metadata: ~`
+processors: {}

elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2

@@ -162,6 +162,7 @@ tags:
 #max_procs:
 
 #================================ Processors ===================================
+{{ elk_macros.beat_processors(processors) }}
 
 # Processors are used to reduce the number of fields in the exported event or to
 # enhance the event with external metadata. This section defines a list of
@@ -272,8 +273,6 @@ tags:
 #    target: ""
 #    overwrite_keys: false
 
-processors:
-- add_host_metadata: 
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using journalbeat with the Elastic Cloud (https://cloud.elastic.co/).

elk_metrics_7x/roles/elastic_metricbeat/defaults/main.yml

@@ -36,3 +36,6 @@ ilm_policy: "{{ (metricbeat_ilm_policy | default(default_ilm_policy)) | default(
 ilm_policy_template: "{{ metricbeat_ilm_policy_template | default('') }}"
 ilm_policy_filename: "{{ (metricbeat_ilm_policy_filename | default(default_ilm_policy_filename)) | default('') }}"
 ilm_policy_file_location: "{{ (metricbeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}"
+
+# beat processors. Empty dictionary sets `add_host_metadata: ~`
+processors: {}

elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2

@@ -880,6 +880,7 @@ metricbeat.modules:
 #max_procs:
 
 #================================ Processors ===================================
+{{ elk_macros.beat_processors(processors) }}
 
 # Processors are used to reduce the number of fields in the exported event or to
 # enhance the event with external metadata. This section defines a list of
@@ -989,8 +990,6 @@ metricbeat.modules:
 #    max_depth: 1
 #    target: ""
 #    overwrite_keys: false
-processors:
-- add_host_metadata: ~
 
 #============================= Elastic Cloud ==================================
 

elk_metrics_7x/roles/elastic_packetbeat/defaults/main.yml

@@ -25,3 +25,6 @@ ilm_policy: "{{ (packetbeat_ilm_policy | default(default_ilm_policy)) | default(
 ilm_policy_template: "{{ packetbeat_ilm_policy_template | default('') }}"
 ilm_policy_filename: "{{ (packetbeat_ilm_policy_filename | default(default_ilm_policy_filename)) | default('') }}"
 ilm_policy_file_location: "{{ (packetbeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}"
+
+# beat processors. Empty dictionary sets `add_host_metadata: ~`
+processors: {}

elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2

@@ -599,6 +599,7 @@ packetbeat.ignore_outgoing: false
 #max_procs:
 
 #================================ Processors ===================================
+{{ elk_macros.beat_processors(processors) }}
 
 # Processors are used to reduce the number of fields in the exported event or to
 # enhance the event with external metadata. This section defines a list of
@@ -708,8 +709,6 @@ packetbeat.ignore_outgoing: false
 #    max_depth: 1
 #    target: ""
 #    overwrite_keys: false
-processors:
-- add_host_metadata: ~
 
 #============================= Elastic Cloud ==================================
 

elk_metrics_7x/templates/_macros.j2

@@ -524,3 +524,14 @@ xpack.monitoring.elasticsearch:
   #metrics.period: 10s
   #state.period: 1m
 {%- endmacro %}
+
+{% macro beat_processors(processors) -%}
+# Processors are used to reduce the number of fields in the exported event or to
+# enhance the event with external metadata.
+processors:
+{% if processors is defined and processors is iterable and processors | length > 0 %}
+{{ processors | to_yaml }}
+{% else %}
+- add_host_metadata: ~
+{%- endif %}
+{%- endmacro %}