From fb7aed401bb4c6f1ed4445850b9fb53935948363 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Fri, 15 Jun 2018 16:45:49 -0500 Subject: [PATCH] Break out setup and always tag facts This change breaks out the basic install and setup playbooks into different playbooks. This is done to ensure an operator can easily rerun any part of the playbook as they upgrade, change, or modify a setup over the lifetime of the deployment. Change-Id: I66c53fcb21880c950ea3fee202e7d2224dfdff3a Signed-off-by: Kevin Carter --- .../common_task_data_node_hosts.yml | 14 ++++++ elk_metrics_6x/installAPMserver.yml | 24 ++-------- elk_metrics_6x/installAuditbeat.yml | 25 ++--------- elk_metrics_6x/installCurator.yml | 3 ++ elk_metrics_6x/installElastic.yml | 3 ++ elk_metrics_6x/installFilebeat.yml | 25 ++--------- elk_metrics_6x/installHeartbeat.yml | 25 ++--------- elk_metrics_6x/installJournalbeat.yml | 25 ++--------- elk_metrics_6x/installKibana.yml | 3 ++ elk_metrics_6x/installLogstash.yml | 3 ++ elk_metrics_6x/installMetricbeat.yml | 26 ++--------- elk_metrics_6x/installPacketbeat.yml | 25 ++--------- elk_metrics_6x/setupAPMserver.yml | 29 ++++++++++++ elk_metrics_6x/setupAuditbeat.yml | 29 ++++++++++++ elk_metrics_6x/setupFilebeat.yml | 29 ++++++++++++ elk_metrics_6x/setupHeartbeat.yml | 29 ++++++++++++ elk_metrics_6x/setupJournalbeat.yml | 45 +++++++++++++++++++ elk_metrics_6x/setupMetricbeat.yml | 29 ++++++++++++ elk_metrics_6x/setupPacketbeat.yml | 29 ++++++++++++ 19 files changed, 266 insertions(+), 154 deletions(-) create mode 100644 elk_metrics_6x/setupAPMserver.yml create mode 100644 elk_metrics_6x/setupAuditbeat.yml create mode 100644 elk_metrics_6x/setupFilebeat.yml create mode 100644 elk_metrics_6x/setupHeartbeat.yml create mode 100644 elk_metrics_6x/setupJournalbeat.yml create mode 100644 elk_metrics_6x/setupMetricbeat.yml create mode 100644 elk_metrics_6x/setupPacketbeat.yml diff --git a/elk_metrics_6x/common_task_data_node_hosts.yml b/elk_metrics_6x/common_task_data_node_hosts.yml index ffd65a3a..5b4eedc2 100644 --- a/elk_metrics_6x/common_task_data_node_hosts.yml +++ b/elk_metrics_6x/common_task_data_node_hosts.yml @@ -4,15 +4,21 @@ - name: Node count fact set_fact: storage_node_count: "{{ groups['elastic-logstash'] | length }}" + tags: + - always - name: Master node pre-count fact set_fact: _master_node_count: "{{ ((storage_node_count | int) > 1) | ternary((((storage_node_count | int) // 2) | int), 1) }}" + tags: + - always # if the master node count is even, add one to it otherwise use the provided value - name: Master node count fact set_fact: master_node_count: "{{ ((_master_node_count | int) % 2 != 0) | ternary((_master_node_count | int), ((_master_node_count | int) + 1)) }}" + tags: + - always - name: Data nodes fact set_fact: @@ -30,10 +36,14 @@ {% set _ = nodes.insert(loop.index, (hostvars[host]['ansible_host'] | string)) %} {% endfor %} {{ nodes }} + tags: + - always - name: Data node count fact set_fact: data_node_count: "{{ data_nodes | length }}" + tags: + - always # if the master node count is even, add one to it otherwise use the provided value # set the data nodes to be all master and alternate through the remaining nodes @@ -41,6 +51,8 @@ set_fact: master_node: "{{ (inventory_hostname in master_nodes) | ternary(true, false) }}" data_node: "{{ (inventory_hostname in data_nodes) | ternary(true, false) }}" + tags: + - always # Set a data node facts. The data nodes, in the case of elasticsearch are also # ingest nodes. @@ -68,3 +80,5 @@ {% set data_hosts = nodes | shuffle(seed=inventory_hostname) %} {% endif %} {{ data_hosts }} + tags: + - always diff --git a/elk_metrics_6x/installAPMserver.yml b/elk_metrics_6x/installAPMserver.yml index bd834336..688332d3 100644 --- a/elk_metrics_6x/installAPMserver.yml +++ b/elk_metrics_6x/installAPMserver.yml @@ -43,25 +43,7 @@ enabled: true state: restarted -- name: Load apm-server Dashboards - hosts: apm-server[0] - become: true - vars_files: - - vars/variables.yml - tasks: - - name: Load templates - shell: >- - apm-server setup - {{ item }} - -E 'apm-server.host=localhost:8200' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -e -v - with_items: - - "--template" - - "--dashboards" - register: templates - until: templates is success - retries: 3 - delay: 2 tags: - - beat-setup + - server-install + +- import_playbook: setupAPMserver.yml diff --git a/elk_metrics_6x/installAuditbeat.yml b/elk_metrics_6x/installAuditbeat.yml index bdf46117..c17cb9fb 100644 --- a/elk_metrics_6x/installAuditbeat.yml +++ b/elk_metrics_6x/installAuditbeat.yml @@ -52,26 +52,7 @@ enabled: "{{ not inventory_hostname in groups['kibana'] | default([]) }}" state: restarted - -- name: Load Auditbeat Dashboards - hosts: hosts[0] - become: true - vars_files: - - vars/variables.yml - tasks: - - name: Load templates - shell: >- - auditbeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -e -v - with_items: - - "--template" - - "--dashboards" - register: templates - until: templates is success - retries: 3 - delay: 2 tags: - - beat-setup + - beat-install + +- import_playbook: setupAuditbeat.yml diff --git a/elk_metrics_6x/installCurator.yml b/elk_metrics_6x/installCurator.yml index b1f5853a..2f52fd55 100644 --- a/elk_metrics_6x/installCurator.yml +++ b/elk_metrics_6x/installCurator.yml @@ -113,3 +113,6 @@ name: "curator.timer" enabled: true state: restarted + + tags: + - beat-install diff --git a/elk_metrics_6x/installElastic.yml b/elk_metrics_6x/installElastic.yml index 3fd06886..a8797c84 100644 --- a/elk_metrics_6x/installElastic.yml +++ b/elk_metrics_6x/installElastic.yml @@ -114,3 +114,6 @@ state: restarted tags: - config + + tags: + - server-install diff --git a/elk_metrics_6x/installFilebeat.yml b/elk_metrics_6x/installFilebeat.yml index abcb5ced..fb92d50e 100644 --- a/elk_metrics_6x/installFilebeat.yml +++ b/elk_metrics_6x/installFilebeat.yml @@ -148,26 +148,7 @@ enabled: true state: restarted - -- name: Load Filebeat Dashboards - hosts: hosts[0] - become: true - vars_files: - - vars/variables.yml - tasks: - - name: Load templates - shell: >- - filebeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -e -v - with_items: - - "--template" - - "--dashboards" - register: templates - until: templates is success - retries: 3 - delay: 2 tags: - - beat-setup + - beat-install + +- import_playbook: setupFilebeat.yml diff --git a/elk_metrics_6x/installHeartbeat.yml b/elk_metrics_6x/installHeartbeat.yml index 0005dfbe..d88b9fa9 100644 --- a/elk_metrics_6x/installHeartbeat.yml +++ b/elk_metrics_6x/installHeartbeat.yml @@ -41,26 +41,7 @@ enabled: true state: restarted - -- name: Load Heartbeat Dashboards - hosts: utility_all[0] - become: true - vars_files: - - vars/variables.yml - tasks: - - name: Load templates - shell: >- - heartbeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -e -v - with_items: - - "--template" - - "--dashboards" - register: templates - until: templates is success - retries: 3 - delay: 2 tags: - - beat-setup + - beat-install + +- import_playbook: setupHeartbeat.yml diff --git a/elk_metrics_6x/installJournalbeat.yml b/elk_metrics_6x/installJournalbeat.yml index d8a0d35b..83f89e89 100644 --- a/elk_metrics_6x/installJournalbeat.yml +++ b/elk_metrics_6x/installJournalbeat.yml @@ -119,26 +119,7 @@ state: restarted daemon_reload: yes + tags: + - beat-install -## NOTE(cloudnull): This task is broken at this point due to missing -## configuration. Once the following issue -## [ https://github.com/mheese/journalbeat/issues/136 ] is -## resolved this should be uncommented. -# - name: Load Journalbeat Dashboards -# hosts: hosts[0] -# become: true -# vars_files: -# - vars/variables.yml -# tasks: -# - name: Load templates -# shell: >- -# /usr/local/bin/journalbeat -setup -# -E 'output.logstash.enabled=false' -# -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' -# -e -v -# register: templates -# until: templates is success -# retries: 3 -# delay: 2 -# tags: -# - beat-setup +- import_playbook: setupJournalbeat.yml diff --git a/elk_metrics_6x/installKibana.yml b/elk_metrics_6x/installKibana.yml index 50685caa..bf9cab0d 100644 --- a/elk_metrics_6x/installKibana.yml +++ b/elk_metrics_6x/installKibana.yml @@ -56,3 +56,6 @@ name: "kibana" enabled: true state: restarted + + tags: + - server-install diff --git a/elk_metrics_6x/installLogstash.yml b/elk_metrics_6x/installLogstash.yml index b2fc34de..3f962f0d 100644 --- a/elk_metrics_6x/installLogstash.yml +++ b/elk_metrics_6x/installLogstash.yml @@ -157,3 +157,6 @@ name: "logstash" enabled: true state: restarted + + tags: + - server-install diff --git a/elk_metrics_6x/installMetricbeat.yml b/elk_metrics_6x/installMetricbeat.yml index c81fbae6..ab80ede9 100644 --- a/elk_metrics_6x/installMetricbeat.yml +++ b/elk_metrics_6x/installMetricbeat.yml @@ -156,27 +156,7 @@ name: "metricbeat" enabled: true state: restarted - - -- name: Load Metricsbeat Dashboards - hosts: all[0] - become: true - vars_files: - - vars/variables.yml - tasks: - - name: Load templates - shell: >- - metricbeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -e -v - with_items: - - "--template" - - "--dashboards" - register: templates - until: templates is success - retries: 3 - delay: 2 tags: - - beat-setup + - beat-install + +- import_playbook: setupMetricbeat.yml diff --git a/elk_metrics_6x/installPacketbeat.yml b/elk_metrics_6x/installPacketbeat.yml index 2b3418d8..f6d038be 100644 --- a/elk_metrics_6x/installPacketbeat.yml +++ b/elk_metrics_6x/installPacketbeat.yml @@ -44,26 +44,7 @@ enabled: true state: restarted - -- name: Load Packetbeat Dashboards - hosts: hosts[0] - become: true - vars_files: - - vars/variables.yml - tasks: - - name: Load templates - shell: >- - packetbeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -e -v - with_items: - - "--template" - - "--dashboards" - register: templates - until: templates is success - retries: 3 - delay: 2 tags: - - beat-setup + - beat-install + +- import_playbook: setupPacketbeat.yml diff --git a/elk_metrics_6x/setupAPMserver.yml b/elk_metrics_6x/setupAPMserver.yml new file mode 100644 index 00000000..433d4b9f --- /dev/null +++ b/elk_metrics_6x/setupAPMserver.yml @@ -0,0 +1,29 @@ +--- + +- name: Load apm-server Dashboards + hosts: apm-server[0] + gather_facts: false + vars_files: + - vars/variables.yml + + pre_tasks: + - include_tasks: common_task_data_node_hosts.yml + + tasks: + - name: Load templates + shell: >- + apm-server setup + {{ item }} + -E 'apm-server.host=localhost:8200' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -e -v + with_items: + - "--template" + - "--dashboards" + register: templates + until: templates is success + retries: 3 + delay: 2 + + tags: + - beat-setup diff --git a/elk_metrics_6x/setupAuditbeat.yml b/elk_metrics_6x/setupAuditbeat.yml new file mode 100644 index 00000000..75e6a0bb --- /dev/null +++ b/elk_metrics_6x/setupAuditbeat.yml @@ -0,0 +1,29 @@ +--- + +- name: Load Auditbeat Dashboards + hosts: hosts[0] + gather_facts: false + vars_files: + - vars/variables.yml + + pre_tasks: + - include_tasks: common_task_data_node_hosts.yml + + tasks: + - name: Load templates + shell: >- + auditbeat setup + {{ item }} + -E 'output.logstash.enabled=false' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -e -v + with_items: + - "--template" + - "--dashboards" + register: templates + until: templates is success + retries: 3 + delay: 2 + + tags: + - beat-setup diff --git a/elk_metrics_6x/setupFilebeat.yml b/elk_metrics_6x/setupFilebeat.yml new file mode 100644 index 00000000..56644f94 --- /dev/null +++ b/elk_metrics_6x/setupFilebeat.yml @@ -0,0 +1,29 @@ +--- + +- name: Load Filebeat Dashboards + hosts: hosts[0] + gather_facts: false + vars_files: + - vars/variables.yml + + pre_tasks: + - include_tasks: common_task_data_node_hosts.yml + + tasks: + - name: Load templates + shell: >- + filebeat setup + {{ item }} + -E 'output.logstash.enabled=false' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -e -v + with_items: + - "--template" + - "--dashboards" + register: templates + until: templates is success + retries: 3 + delay: 2 + + tags: + - beat-setup diff --git a/elk_metrics_6x/setupHeartbeat.yml b/elk_metrics_6x/setupHeartbeat.yml new file mode 100644 index 00000000..0319f03b --- /dev/null +++ b/elk_metrics_6x/setupHeartbeat.yml @@ -0,0 +1,29 @@ +--- + +- name: Load Heartbeat Dashboards + hosts: utility_all[0] + gather_facts: false + vars_files: + - vars/variables.yml + + pre_tasks: + - include_tasks: common_task_data_node_hosts.yml + + tasks: + - name: Load templates + shell: >- + heartbeat setup + {{ item }} + -E 'output.logstash.enabled=false' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -e -v + with_items: + - "--template" + - "--dashboards" + register: templates + until: templates is success + retries: 3 + delay: 2 + + tags: + - beat-setup diff --git a/elk_metrics_6x/setupJournalbeat.yml b/elk_metrics_6x/setupJournalbeat.yml new file mode 100644 index 00000000..192233d3 --- /dev/null +++ b/elk_metrics_6x/setupJournalbeat.yml @@ -0,0 +1,45 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Load Journalbeat Dashboards + hosts: hosts[0] + gather_facts: false + vars_files: + - vars/variables.yml + + pre_tasks: + # NOTE(cloudnull): This task is broken at this point due to missing + # configuration. Once the following issue + # [ https://github.com/mheese/journalbeat/issues/136 ] is + # resolved, this should be removed. + - name: exit playbook + meta: end_play + + - include_tasks: common_task_data_node_hosts.yml + + tasks: + - name: Load templates + shell: >- + /usr/local/bin/journalbeat -setup + -E 'output.logstash.enabled=false' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -e -v + register: templates + until: templates is success + retries: 3 + delay: 2 + + tags: + - beat-setup diff --git a/elk_metrics_6x/setupMetricbeat.yml b/elk_metrics_6x/setupMetricbeat.yml new file mode 100644 index 00000000..c9fb9401 --- /dev/null +++ b/elk_metrics_6x/setupMetricbeat.yml @@ -0,0 +1,29 @@ +--- + +- name: Load Metricsbeat Dashboards + hosts: all[0] + gather_facts: false + vars_files: + - vars/variables.yml + + pre_tasks: + - include_tasks: common_task_data_node_hosts.yml + + tasks: + - name: Load templates + shell: >- + metricbeat setup + {{ item }} + -E 'output.logstash.enabled=false' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -e -v + with_items: + - "--template" + - "--dashboards" + register: templates + until: templates is success + retries: 3 + delay: 2 + + tags: + - beat-setup diff --git a/elk_metrics_6x/setupPacketbeat.yml b/elk_metrics_6x/setupPacketbeat.yml new file mode 100644 index 00000000..3991858a --- /dev/null +++ b/elk_metrics_6x/setupPacketbeat.yml @@ -0,0 +1,29 @@ +--- + +- name: Load Packetbeat Dashboards + hosts: hosts[0] + gather_facts: false + vars_files: + - vars/variables.yml + + pre_tasks: + - include_tasks: common_task_data_node_hosts.yml + + tasks: + - name: Load templates + shell: >- + packetbeat setup + {{ item }} + -E 'output.logstash.enabled=false' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -e -v + with_items: + - "--template" + - "--dashboards" + register: templates + until: templates is success + retries: 3 + delay: 2 + + tags: + - beat-setup