From e08c58dd151e2ffa9bf80bf1fae11814e78cb206 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Tue, 27 Nov 2018 13:42:42 -0600 Subject: [PATCH] modify fact gathering to use local facts This change makes the retention gathering operation faster by storing the retention values as "local facts". The local facts are then referenced in templates loading from the local fact file instead of running repetitive queries which are slow making very large deployments cumbersome. To make the retention policy fact gathering process smarter it will now automatically refresh if undefined or should the elasticsearch cluster size change. This will ensure we're improving speed of execution while also catering to the needs of deployers to grow, or shrink, elasticsearch cluster sizes. Documentation has been added regarding the new option and why it may be of use to depoyers. Change-Id: I3936ee94461ac39fb8bc78dc2c873e6067552461 Signed-off-by: Kevin Carter --- elk_metrics_6x/README.rst | 16 +++ .../roles/elastic_curator/tasks/main.yml | 7 ++ .../templates/curator-actions-age.yml.j2 | 4 +- .../templates/curator-actions-size.yml.j2 | 4 +- .../roles/elastic_retention/defaults/main.yml | 3 + .../roles/elastic_retention/tasks/main.yml | 110 +++++++++++++----- .../roles/elastic_rollup/tasks/main.yml | 3 +- 7 files changed, 112 insertions(+), 35 deletions(-) diff --git a/elk_metrics_6x/README.rst b/elk_metrics_6x/README.rst index ecba8352..cc0e40f1 100644 --- a/elk_metrics_6x/README.rst +++ b/elk_metrics_6x/README.rst @@ -570,6 +570,22 @@ state variable, `elk_package_state`, to latest. ansible-playbook site.yml $USER_VARS -e 'elk_package_state="latest"' +Forcing the Elasticsearch cluster retention policy to refresh +------------------------------------------------------------- + +To force the cluster retention policy to refresh set `elastic_retention_refresh`, to +"yes". When setting `elastic_retention_refresh` to "yes" the retention policy will forcibly +be refresh across all hosts. This option should only be used when the Elasticsearch storage +array is modified on an existing cluster. Should the Elasticseach cluster size change +(nodes added or removed) the retention policy will automatically be refreshed on playbook +execution. + +.. code-block:: bash + + cd /opt/openstack-ansible-ops/elk_metrics_6x + ansible-playbook site.yml $USER_VARS -e 'elastic_retention_refresh="yes"' + + Trouble shooting ---------------- diff --git a/elk_metrics_6x/roles/elastic_curator/tasks/main.yml b/elk_metrics_6x/roles/elastic_curator/tasks/main.yml index c40c492d..9aa6681c 100644 --- a/elk_metrics_6x/roles/elastic_curator/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_curator/tasks/main.yml @@ -30,6 +30,13 @@ when: ansible_service_mgr != 'systemd' +- name: Refresh local facts + setup: + filter: ansible_local + gather_subset: "!all" + tags: + - always + - name: Ensure virtualenv is installed package: name: "{{ curator_distro_packages }}" diff --git a/elk_metrics_6x/roles/elastic_curator/templates/curator-actions-age.yml.j2 b/elk_metrics_6x/roles/elastic_curator/templates/curator-actions-age.yml.j2 index 2c7b337b..9f2ed25e 100644 --- a/elk_metrics_6x/roles/elastic_curator/templates/curator-actions-age.yml.j2 +++ b/elk_metrics_6x/roles/elastic_curator/templates/curator-actions-age.yml.j2 @@ -15,10 +15,10 @@ {% set action_items = [] -%} {# Delete index loop #} -{% for key in elastic_beat_retention_policy_keys -%} +{% for key in (ansible_local['elastic']['retention']['elastic_beat_retention_policy_keys'] | from_yaml) -%} {% set delete_indices = {} -%} {# Total retention size in days #} -{% set _index_retention = hostvars[inventory_hostname]['elastic_' + key + '_retention'] -%} +{% set _index_retention = ansible_local['elastic']['retention']['elastic_' + key + '_retention'] -%} {% set index_retention = ((_index_retention | int) > 0) | ternary(_index_retention, 1) | int %} {% set _ = delete_indices.update( { diff --git a/elk_metrics_6x/roles/elastic_curator/templates/curator-actions-size.yml.j2 b/elk_metrics_6x/roles/elastic_curator/templates/curator-actions-size.yml.j2 index b0f5f488..88e1c3c4 100644 --- a/elk_metrics_6x/roles/elastic_curator/templates/curator-actions-size.yml.j2 +++ b/elk_metrics_6x/roles/elastic_curator/templates/curator-actions-size.yml.j2 @@ -15,10 +15,10 @@ {% set action_items = [] -%} {# Delete index loop #} -{% for key in elastic_beat_retention_policy_keys -%} +{% for key in (ansible_local['elastic']['retention']['elastic_beat_retention_policy_keys'] | from_yaml) -%} {% set delete_indices = {} -%} {# Total retention size in gigabytes #} -{% set _index_size = ((hostvars[inventory_hostname]['elastic_' + key + '_size'] | int) // 1024) -%} +{% set _index_size = ((ansible_local['elastic']['retention']['elastic_' + key + '_size'] | int) // 1024) -%} {% set index_size = ((_index_size | int) > 0) | ternary(_index_size, 1) | int %} {% set _ = delete_indices.update( { diff --git a/elk_metrics_6x/roles/elastic_retention/defaults/main.yml b/elk_metrics_6x/roles/elastic_retention/defaults/main.yml index 0019cfab..1bb40529 100644 --- a/elk_metrics_6x/roles/elastic_retention/defaults/main.yml +++ b/elk_metrics_6x/roles/elastic_retention/defaults/main.yml @@ -94,3 +94,6 @@ elastic_beat_retention_policy_hosts: packetbeat: weight: 1 hosts: "{{ groups['hosts'] | default([]) }}" + +# Refresh the elasticsearch retention policy local facts. +elastic_retention_refresh: false \ No newline at end of file diff --git a/elk_metrics_6x/roles/elastic_retention/tasks/main.yml b/elk_metrics_6x/roles/elastic_retention/tasks/main.yml index 1a147563..9e096773 100644 --- a/elk_metrics_6x/roles/elastic_retention/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_retention/tasks/main.yml @@ -13,40 +13,90 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Query es storage - uri: - url: "http://{{ coordination_nodes[0] }}/_nodes/{{ (data_nodes | map('extract', hostvars, 'ansible_host') | list) | join(',') }}/stats/fs" - method: GET - register: elk_data - environment: - no_proxy: "{{ coordination_nodes[0].split(':')[0] }}" - until: - - elk_data is success and elk_data['json'] is defined - retries: 5 - delay: 30 - run_once: true +- name: Ensure local facts directory exists + file: + dest: "/etc/ansible/facts.d" + state: directory + group: "root" + owner: "root" + mode: "0755" + recurse: no -- name: Set retention keys fact - set_fact: - es_storage_json: "{{ elk_data['json'] }}" +- name: Initialize local facts + ini_file: + dest: "/etc/ansible/facts.d/elastic.fact" + section: "retention" + option: cacheable + value: true -- name: Load retention algo variables - include_vars: "calculate_index_retention_{{ elastic_index_retention_algorithm }}.yml" +- name: Refresh local facts + setup: + filter: ansible_local + gather_subset: "!all" tags: - always -- name: Set retention facts (mb size) - set_fact: "elastic_{{ item.key }}_size={{ item.value }}" - when: - - hostvars[inventory_hostname]["elastic_" ~ item.key ~ "_size"] is undefined - with_dict: "{{ es_storage_per_index }}" +- name: Retention storage block + block: + - name: Query es storage + uri: + url: "http://{{ coordination_nodes[0] }}/_nodes/{{ (data_nodes | map('extract', hostvars, 'ansible_host') | list) | join(',') }}/stats/fs" + method: GET + register: elk_data + environment: + no_proxy: "{{ coordination_nodes[0].split(':')[0] }}" + until: + - elk_data is success and elk_data['json'] is defined + retries: 5 + delay: 30 + run_once: true -- name: Set retention facts (days) - set_fact: "elastic_{{ item.key }}_retention={{ item.value }}" - when: - - hostvars[inventory_hostname]["elastic_" ~ item.key ~ "_retention"] is undefined - with_dict: "{{ es_days_per_index }}" + - name: Set retention keys fact + set_fact: + es_storage_json: "{{ elk_data['json'] }}" -- name: Set retention keys fact - set_fact: - elastic_beat_retention_policy_keys: "{{ elastic_beat_retention_policy_hosts.keys() | list }}" + - name: Load retention algo variables + include_vars: "calculate_index_retention_{{ elastic_index_retention_algorithm }}.yml" + tags: + - always + + - name: Set storage fact + ini_file: + dest: "/etc/ansible/facts.d/elastic.fact" + section: "retention" + option: "cluster_nodes" + value: "{{ groups['elastic-logstash'] | length }}" + + - name: Set retention policy fact + ini_file: + dest: "/etc/ansible/facts.d/elastic.fact" + section: "retention" + option: "elastic_beat_retention_policy_keys" + value: "{{ elastic_beat_retention_policy_hosts.keys() | list }}" + + - name: Set size fact + ini_file: + dest: "/etc/ansible/facts.d/elastic.fact" + section: "retention" + option: "elastic_{{ item.key }}_size" + value: "{{ item.value }}" + with_dict: "{{ es_storage_per_index }}" + + - name: Set retention fact + ini_file: + dest: "/etc/ansible/facts.d/elastic.fact" + section: "retention" + option: "elastic_{{ item.key }}_retention" + value: "{{ item.value }}" + with_dict: "{{ es_days_per_index }}" + + - name: Refresh local facts + setup: + filter: ansible_local + gather_subset: "!all" + tags: + - always + when: + - (ansible_local['elastic']['retention']['cluster_nodes'] is undefined) or + ((groups['elastic-logstash'] | length) != (ansible_local['elastic']['retention']['cluster_nodes'] | int)) or + (elastic_retention_refresh | bool) diff --git a/elk_metrics_6x/roles/elastic_rollup/tasks/main.yml b/elk_metrics_6x/roles/elastic_rollup/tasks/main.yml index 77cdf542..96caefdd 100644 --- a/elk_metrics_6x/roles/elastic_rollup/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_rollup/tasks/main.yml @@ -33,6 +33,7 @@ OK with the old rollup job being purged, add the following option `elastic_allow_rollup_purge=yes` to the command line and rerun the playbook. + run_once: true when: - check_rollup['json']['jobs'] | length > 0 - not elastic_allow_rollup_purge | bool @@ -44,7 +45,7 @@ min_days_until_rollup: |- {% set index_retention = [] %} {% for item in ansible_play_hosts %} - {% set _ = index_retention.append(hostvars[item]['elastic_' + index_name + '_retention'] | int) %} + {% set _ = index_retention.append(ansible_local['elastic']['retention']['elastic_' + index_name + '_retention'] | int) %} {% endfor %} {{ index_retention | min }} run_once: true