From 9dffb8c18498562e1fe88e5ec0cc8f3e1dae6e42 Mon Sep 17 00:00:00 2001 From: Duncan Martin Walker Date: Wed, 22 Jan 2020 10:05:57 +0000 Subject: [PATCH] Module config updates in ELK 7x Metricbeat template The template for metricbeat has been updated to allow modules to be turned off from the ansible directly. This can prevent metricbeat from regularly throwing errors for plugins that can't connect to the required endpoints when the aren't configured. Change-Id: Ie03688fe5e647db2374f62b13353ce36b32fcb96 --- .../vars/data-node-variables.yml | 3 ++- .../roles/elastic_metricbeat/defaults/main.yml | 3 +++ .../roles/elastic_metricbeat/tasks/main.yml | 6 +++++- .../templates/metricbeat.yml.j2 | 16 +++++++++++++--- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/elk_metrics_7x/roles/elastic_data_hosts/vars/data-node-variables.yml b/elk_metrics_7x/roles/elastic_data_hosts/vars/data-node-variables.yml index aad576af..a6e8b385 100644 --- a/elk_metrics_7x/roles/elastic_data_hosts/vars/data-node-variables.yml +++ b/elk_metrics_7x/roles/elastic_data_hosts/vars/data-node-variables.yml @@ -16,7 +16,7 @@ # storage node count is equal to the cluster size storage_node_count: "{{ groups['elastic-logstash'] | length }}" -# the elasticserch cluster elects one master from all those which are marked as master-eligible +# the elasticsearch cluster elects one master from all those which are marked as master-eligible # 1 node cluster can only have one master # 2 node clusters have 1 master-eligable nodes to avoid split-brain # 3 node clusters have 3 master-eligable nodes @@ -137,6 +137,7 @@ zen_nodes: >- {{ (groups['elastic-logstash'] | union(groups['kibana'])) | map('extract', hostvars, 'ansible_host') | list | shuffle(seed=inventory_hostname) }} + elasticserch_interface_speed: |- {% set default_interface_fact = hostvars[inventory_hostname]['ansible_' + (elastic_data_interface | replace('-', '_'))] %} {% set speeds = [] %} diff --git a/elk_metrics_7x/roles/elastic_metricbeat/defaults/main.yml b/elk_metrics_7x/roles/elastic_metricbeat/defaults/main.yml index a2ae4065..f376e217 100644 --- a/elk_metrics_7x/roles/elastic_metricbeat/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_metricbeat/defaults/main.yml @@ -23,6 +23,9 @@ metricbeat_service_state: restarted # via either ceph-restapi or mgr prometheus module ceph_metricbeat_group: cinder_volume +# Include default kibana dashboards +setup_kibana_dashboards: true + metricbeat_ilm_policy_name: "metricbeat-{{ beat_version }}" metricbeat_ilm_policy_template: "metricbeat-{{ beat_version }}" metricbeat_ilm_policy_filename: "metricbeat-ilm-policy.json" diff --git a/elk_metrics_7x/roles/elastic_metricbeat/tasks/main.yml b/elk_metrics_7x/roles/elastic_metricbeat/tasks/main.yml index 3481ec75..3b4cdf5f 100644 --- a/elk_metrics_7x/roles/elastic_metricbeat/tasks/main.yml +++ b/elk_metrics_7x/roles/elastic_metricbeat/tasks/main.yml @@ -202,17 +202,21 @@ {% endif %} {{ ceph_detect }} - + # Set to false for now, to be updated with existence checks at a later date + couchdb_enabled: false docker_enabled: "{{ docker.stat.exists | bool }}" + envoyproxy_enabled: false etcd_enabled: "{{ etcd.stat.exists | bool }}" haproxy_enabled: "{{ haproxy.stat.exists | bool }}" kvm_enabled: "{{ kvm.stat.exists | bool }}" memcached_enabled: "{{ memcached.stat.exists | bool }}" mysql_enabled: "{{ mysql.stat.exists | bool }}" + nats_enabled: false nginx_enabled: "{{ nginx.stat.exists | bool }}" rabbitmq_enabled: "{{ rabbitmq.stat.exists | bool }}" uwsgi_enabled: "{{ uwsgi.stat.exists | bool }}" uwsgi_sockets: "{{ uwsgi_find_sockets }}" + traefik_enabled: false # Apache 2 stats enablement - name: Enable apache2 diff --git a/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 b/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 index 52e959e9..2e9777e0 100644 --- a/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 @@ -176,11 +176,12 @@ metricbeat.modules: # enabled: true # #------------------------------- couchdb Module ------------------------------ +{% if couchdb_enabled | bool %} - module: couchdb metricsets: ["server"] period: 10s hosts: ["localhost:5984"] - +{% endif %} #------------------------------- Docker Module ------------------------------- {% if docker_enabled | default(false) | bool %} - module: docker @@ -245,10 +246,12 @@ metricbeat.modules: #xpack.enabled: false {% endif %} #----------------------------- envoyproxy Module ----------------------------- +{% if envoyproxy_enabled | bool %} - module: envoyproxy metricsets: ["server"] period: 10s hosts: ["localhost:9901"] +{% endif %} #-------------------------------- Etcd Module -------------------------------- {% if etcd_enabled | default(false) | bool %} @@ -570,6 +573,7 @@ metricbeat.modules: #raw: false {% endif %} #-------------------------------- Nats Module -------------------------------- +{% if nats_enabled | bool %} - module: nats metricsets: ["connections", "routes", "stats", "subscriptions"] period: 10s @@ -578,7 +582,7 @@ metricbeat.modules: #connections.metrics_path: "/connz" #routes.metrics_path: "/routez" #subscriptions.metrics_path: "/subsz" - +{% endif %} #-------------------------------- Nginx Module ------------------------------- {% if nginx_enabled | default(false) | bool %} - module: nginx @@ -707,10 +711,12 @@ metricbeat.modules: #password: foobared #------------------------------- traefik Module ------------------------------ +{% if traefik_enabled | bool %} - module: traefik metricsets: ["health"] period: 10s hosts: ["localhost:8080"] +{% endif %} #-------------------------------- uwsgi Module ------------------------------- {% if uwsgi_enabled | default(false) | bool %} @@ -1457,9 +1463,13 @@ processors: #keystore.path: "${path.config}/beats.keystore" #============================== Dashboards ===================================== +{% if (groups['kibana'] | length) > 0 and (setup_kibana_dashboards | default(true)) %} {{ elk_macros.setup_dashboards('metricbeat') }} +{% endif %} + #============================== Template ===================================== + {{ elk_macros.setup_template('metricbeat', inventory_hostname, data_nodes, elasticsearch_beat_settings) }} #============================== Setup ILM ===================================== @@ -1487,7 +1497,7 @@ setup.ilm.policy_file: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }} {% endif %} #============================== Kibana ===================================== -{% if (groups['kibana'] | length) > 0 %} +{% if (groups['kibana'] | length) > 0 and (setup_kibana_dashboards | default(true)) %} {{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }} {% endif %}