Module config updates in ELK 7x Metricbeat template

The template for metricbeat has been updated to allow modules to be
turned off from the ansible directly. This can prevent metricbeat from
regularly throwing errors for plugins that can't connect to the required
endpoints when the aren't configured.

Change-Id: Ie03688fe5e647db2374f62b13353ce36b32fcb96

elk_metrics_7x/roles/elastic_data_hosts/vars/data-node-variables.yml

@@ -16,7 +16,7 @@
 # storage node count is equal to the cluster size
 storage_node_count: "{{ groups['elastic-logstash'] | length }}"
 
-# the elasticserch cluster elects one master from all those which are marked as master-eligible
+# the elasticsearch cluster elects one master from all those which are marked as master-eligible
 # 1 node cluster can only have one master
 # 2 node clusters have 1 master-eligable nodes to avoid split-brain
 # 3 node clusters have 3 master-eligable nodes
@@ -137,6 +137,7 @@ zen_nodes: >-
   {{
     (groups['elastic-logstash'] | union(groups['kibana'])) | map('extract', hostvars, 'ansible_host') | list | shuffle(seed=inventory_hostname)
   }}
+
 elasticserch_interface_speed: |-
   {% set default_interface_fact = hostvars[inventory_hostname]['ansible_' + (elastic_data_interface | replace('-', '_'))] %}
   {% set speeds = [] %}

elk_metrics_7x/roles/elastic_metricbeat/defaults/main.yml

@@ -23,6 +23,9 @@ metricbeat_service_state: restarted
 # via either ceph-restapi or mgr prometheus module
 ceph_metricbeat_group: cinder_volume
 
+# Include default kibana dashboards
+setup_kibana_dashboards: true
+
 metricbeat_ilm_policy_name: "metricbeat-{{ beat_version }}"
 metricbeat_ilm_policy_template: "metricbeat-{{ beat_version }}"
 metricbeat_ilm_policy_filename: "metricbeat-ilm-policy.json"

elk_metrics_7x/roles/elastic_metricbeat/tasks/main.yml

@@ -202,17 +202,21 @@
       {% endif %}
       {{ ceph_detect }}
 
-
+    # Set to false for now, to be updated with existence checks at a later date
+    couchdb_enabled: false
     docker_enabled: "{{ docker.stat.exists | bool }}"
+    envoyproxy_enabled: false
     etcd_enabled: "{{ etcd.stat.exists | bool }}"
     haproxy_enabled: "{{ haproxy.stat.exists | bool }}"
     kvm_enabled: "{{ kvm.stat.exists | bool }}"
     memcached_enabled: "{{ memcached.stat.exists | bool }}"
     mysql_enabled: "{{ mysql.stat.exists | bool }}"
+    nats_enabled: false
     nginx_enabled: "{{ nginx.stat.exists | bool }}"
     rabbitmq_enabled: "{{ rabbitmq.stat.exists | bool }}"
     uwsgi_enabled: "{{ uwsgi.stat.exists | bool }}"
     uwsgi_sockets: "{{ uwsgi_find_sockets }}"
+    traefik_enabled: false
 
 # Apache 2 stats enablement
 - name: Enable apache2

elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2

@@ -176,11 +176,12 @@ metricbeat.modules:
 #  enabled: true
 #
 #------------------------------- couchdb Module ------------------------------
+{% if couchdb_enabled | bool %}
 - module: couchdb
   metricsets: ["server"]
   period: 10s
   hosts: ["localhost:5984"]
-
+{% endif %}
 #------------------------------- Docker Module -------------------------------
 {% if docker_enabled | default(false) | bool %}
 - module: docker
@@ -245,10 +246,12 @@ metricbeat.modules:
   #xpack.enabled: false
 {% endif %}
 #----------------------------- envoyproxy Module -----------------------------
+{% if envoyproxy_enabled | bool %}
 - module: envoyproxy
   metricsets: ["server"]
   period: 10s
   hosts: ["localhost:9901"]
+{% endif %}
 
 #-------------------------------- Etcd Module --------------------------------
 {% if etcd_enabled | default(false) | bool %}
@@ -570,6 +573,7 @@ metricbeat.modules:
   #raw: false
 {% endif %}
 #-------------------------------- Nats Module --------------------------------
+{% if nats_enabled | bool %}
 - module: nats
   metricsets: ["connections", "routes", "stats", "subscriptions"]
   period: 10s
@@ -578,7 +582,7 @@ metricbeat.modules:
   #connections.metrics_path: "/connz"
   #routes.metrics_path: "/routez"
   #subscriptions.metrics_path: "/subsz"
-
+{% endif %}
 #-------------------------------- Nginx Module -------------------------------
 {% if nginx_enabled | default(false) | bool %}
 - module: nginx
@@ -707,10 +711,12 @@ metricbeat.modules:
   #password: foobared
 
 #------------------------------- traefik Module ------------------------------
+{% if traefik_enabled | bool %}
 - module: traefik
   metricsets: ["health"]
   period: 10s
   hosts: ["localhost:8080"]
+{% endif %}
 
 #-------------------------------- uwsgi Module -------------------------------
 {% if uwsgi_enabled | default(false) | bool %}
@@ -1457,9 +1463,13 @@ processors:
 #keystore.path: "${path.config}/beats.keystore"
 
 #============================== Dashboards =====================================
+{% if (groups['kibana'] | length) > 0 and (setup_kibana_dashboards | default(true)) %}
 {{ elk_macros.setup_dashboards('metricbeat') }}
+{% endif %}
+
 
 #============================== Template =====================================
+
 {{ elk_macros.setup_template('metricbeat', inventory_hostname, data_nodes, elasticsearch_beat_settings) }}
 
 #============================== Setup ILM =====================================
@@ -1487,7 +1497,7 @@ setup.ilm.policy_file: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }}
 {% endif %}
 
 #============================== Kibana =====================================
-{% if (groups['kibana'] | length) > 0 %}
+{% if (groups['kibana'] | length) > 0 and (setup_kibana_dashboards | default(true)) %}
 {{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }}
 {% endif %}