Improvements to ILM

Elasticsearch 7 introduced Index Lifecycle Management, my aim is to eventually
replace curator with this.

This update allows for a default ILM policy to be configured for all beats.
The default policy is quite basic: an index will be written to for 15 days or
until the index reaches 30GB in size, then it will remain in the hot stage
(no performance degradation) for another 15 days, after which it will be
deleted.

This still may lead to a situation where the disk space on log nodes could be
filled.

ILM policies can be configured for each beat by overriding the defaults
that are in the role for each beat.

ILM is set up for beats when undergoing an update (elk_package_state="latest").
During ILM set up elasticsearch creates an ILM ready template for the beat and
uses the ILM policy we provision to nodes for the initial ILM policy.
Subsequent ILM policy updates use ES APIs. New ILM policy files are not
provisioned to nodes outside of using the beat upgrade flags, so the policy
file on the node may fall out of step.

Change-Id: I2c5c3abd4bb65075f2377227cbbfe31b68b0dc38
This commit is contained in:
Georgina Shippey 2019-10-03 15:13:46 +01:00
parent 8e64de671a
commit f89dd344c3
22 changed files with 238 additions and 5 deletions

View File

@ -282,3 +282,14 @@ filebeat_prospectors:
- /var/log/elasticsearch/*.log
tags:
- beats
filebeat_ilm_policy_name: "filebeat-{{ beat_version }}"
filebeat_ilm_policy_template: "filebeat-{{ beat_version }}"
filebeat_ilm_policy_filename: "filebeat-ilm-policy.json"
filebeat_ilm_policy_file_location: "/etc/filebeat"
ilm_policy_name: "{{ filebeat_ilm_policy_name | default('') }}"
ilm_policy: "{{ (filebeat_ilm_policy | default(default_ilm_policy)) | default({}) }}"
ilm_policy_template: "{{ filebeat_ilm_policy_template | default('') }}"
ilm_policy_filename: "{{ (filebeat_ilm_policy_filename | default(default_ilm_policy_filename)) | default('') }}"
ilm_policy_file_location: "{{ (filebeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}"

View File

@ -82,6 +82,10 @@
notify:
- Enable and restart filebeat
- include_role:
name: elastic_ilm
when: ilm_policy.keys() | length > 0
- name: Run the beat setup role
include_role:
name: elastic_beat_setup

View File

@ -1368,7 +1368,12 @@ processors:
# Configure rollover index pattern.
#setup.ilm.pattern: "{now/d}-000001"
{% if ilm_policy_name is defined %}
setup.ilm.policy_name: "{{ ilm_policy_name }}"
{% endif %}
{% if ilm_policy_file_location is defined %}
setup.ilm.policy_file: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }}"
{% endif %}
#============================== Kibana ======================================
{% if (groups['kibana'] | length) > 0 %}

View File

@ -14,3 +14,14 @@
# limitations under the License.
heartbeat_service_state: restarted
heartbeat_ilm_policy_name: "heartbeat-{{ beat_version }}"
heartbeat_ilm_policy_template: "heartbeat-{{ beat_version}}"
heartbeat_ilm_policy_filename: "heartbeat-ilm-policy.json"
heartbeat_ilm_policy_file_location: "/etc/heartbeat"
ilm_policy_name: "{{ heartbeat_ilm_policy_name | default('') }}"
ilm_policy: "{{ (heartbeat_ilm_policy | default(default_ilm_policy)) | default({}) }}"
ilm_policy_template: "{{ heartbeat_ilm_policy_template | default('') }}"
ilm_policy_filename: "{{ (heartbeat_ilm_policy_filename | default(heartbeat_ilm_policy_filename)) | default('') }}"
ilm_policy_file_location: "{{ (heartbeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}"

View File

@ -88,6 +88,10 @@
notify:
- Enable and restart heartbeat
- include_role:
name: elastic_ilm
when: ilm_policy.keys() | length > 0
- name: Run the beat setup role
include_role:
name: elastic_beat_setup

View File

@ -1013,7 +1013,12 @@ processors:
# Configure rollover index pattern.
#setup.ilm.pattern: "{now/d}-000001"
{% if ilm_policy_name is defined %}
setup.ilm.policy_name: "{{ ilm_policy_name }}"
{% endif %}
{% if ilm_policy_file_location is defined %}
setup.ilm.policy_file: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }}"
{% endif %}
#============================== Kibana =====================================
{% if (groups['kibana'] | length) > 0 %}
{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }}

View File

@ -0,0 +1,18 @@
beat_version: "7.3.2"
default_ilm_rollover_max_size: "30G"
default_ilm_rollover_max_age: "15d"
default_ilm_delete_min_age: "30d"
default_ilm_policy:
policy:
phases:
hot:
actions:
rollover:
max_size: "{{ default_ilm_rollover_max_size }}"
max_age: "{{ default_ilm_rollover_max_age }}"
delete:
min_age: "{{ default_ilm_delete_min_age }}"
actions:
delete: {}
default_ilm_policy_filename: "default-ilm-policy.json"
default_ilm_policy_file_location: "/tmp"

View File

@ -0,0 +1,32 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
galaxy_info:
author: OpenStack
description: Elastic v7.x elastic_ilm role
company: Rackspace
license: Apache2
min_ansible_version: 2.5
platforms:
- name: Ubuntu
versions:
- trusty
- xenial
- bionic
categories:
- cloud
- development
- elasticsearch
- elastic-stack

View File

@ -0,0 +1,8 @@
---
- name: Template out policy to hosts for initial set up situations
template:
src: "ilm-policy.json.j2"
dest: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }}"
mode: 0750
owner: root
group: root

View File

@ -0,0 +1,43 @@
---
- name: Check if policy exists
uri:
url: "http://127.0.0.1:{{ elastic_port }}/_ilm/policy/{{ ilm_policy_name }}"
method: GET
status_code: 200,404
body_format: json
register: check_policy
when: ilm_policy_name is defined and ilm_policy is defined
- name: Update ILM policy when not installing beat for first time
uri:
url: "http://127.0.0.1:{{ elastic_port }}/_ilm/policy/{{ ilm_policy_name }}"
method: PUT
body: "{{ ilm_policy }}"
status_code: 200
body_format: json
when: check_policy.status == 200 and ilm_policy_name is defined and ilm_policy is defined and (elk_package_state | default('present')) != "latest"
- name: Get template
uri:
url: "http://127.0.0.1:{{ elastic_port }}/_template/{{ ilm_policy_template }}/"
method: GET
status_code: 200,404
body_format: json
register: template
when: ilm_policy_template is defined and ilm_policy_name is defined
- set_fact:
updated_template: "{{ template.json | combine({template.json.keys()[0]|string:{'settings': {'index': {'lifecycle': {'name': 'ilm_policy_name' }}}}}, recursive=True) }}"
when: template.status == 200 and ilm_policy_template is defined and ilm_policy_name is defined
- name: Update template
uri:
url: "http://127.0.0.1:{{ elastic_port }}/_template/{{ ilm_policy_template }}/"
body: "{{ updated_template[template.json.keys()[0]|string] }}"
method: PUT
body_format: json
status_code: 200
headers:
Content-Type: "application/json"
kbn-xsrf: "{{ inventory_hostname | to_uuid }}"
when: template.status == 200 and ilm_policy_template is defined and ilm_policy_name is defined

View File

@ -0,0 +1,12 @@
---
- include_tasks: "elastic_ilm_policy_template.yml"
when: ilm_policy_file_location is defined and ilm_policy_filename is defined and ilm_policy is defined
- block:
- name: Update policy and template when not upgrading
include_tasks: "elastic_ilm_update_policy.yml"
delegate_to: "{{ groups['elastic-logstash'][0] }}"
environment:
no_proxy: "{{ hostvars[groups['elastic-logstash'][0]]['ansible_host'] }},127.0.0.1"
run_once: True
when: (elk_package_state | default('present')) != "latest"

View File

@ -0,0 +1 @@
{{ ilm_policy | to_json }}

View File

@ -14,3 +14,14 @@
# limitations under the License.
journalbeat_service_state: restarted
journalbeat_ilm_policy_name: "journalbeat-{{ beat_version }}"
journalbeat_ilm_policy_template: "journalbeat-{{ beat_version }}"
journalbeat_ilm_policy_filename: "journalbeat-ilm-policy.json"
journalbeat_ilm_policy_file_location: "/etc/journalbeat"
ilm_policy_name: "{{ journalbeat_ilm_policy_name | default('') }}"
ilm_policy: "{{ (journalbeat_ilm_policy | default(default_ilm_policy)) | default({}) }}"
ilm_policy_template: "{{ journalbeat_ilm_policy_template | default('') }}"
ilm_policy_filename: "{{ (journalbeat_ilm_policy_filename | default(default_ilm_policy_filename)) | default('') }}"
ilm_policy_file_location: "{{ (journalbeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}"

View File

@ -88,6 +88,10 @@
notify:
- Enable and restart journalbeat
- include_role:
name: elastic_ilm
when: ilm_policy.keys() | length > 0
- name: Run the beat setup role
include_role:
name: elastic_beat_setup

View File

@ -768,7 +768,12 @@ processors:
# Configure rollover index pattern.
#setup.ilm.pattern: "{now/d}-000001"
{% if ilm_policy_name is defined %}
setup.ilm.policy_name: "{{ ilm_policy_name }}"
{% endif %}
{% if ilm_policy_file_location is defined %}
setup.ilm.policy_file: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }}"
{% endif %}
#============================== Kibana =====================================
{% if (groups['kibana'] | length) > 0 %}

View File

@ -22,3 +22,14 @@ metricbeat_service_state: restarted
# Inventory group to configure metricbeat ceph monitoring
# via either ceph-restapi or mgr prometheus module
ceph_metricbeat_group: cinder_volume
metricbeat_ilm_policy_name: "metricbeat-{{ beat_version }}"
metricbeat_ilm_policy_template: "metricbeat-{{ beat_version }}"
metricbeat_ilm_policy_filename: "metricbeat-ilm-policy.json"
metricbeat_ilm_policy_file_location: "/etc/metricbeat"
ilm_policy_name: "{{ metricbeat_ilm_policy_name | default('') }}"
ilm_policy: "{{ (metricbeat_ilm_policy | default(default_ilm_policy)) | default({}) }}"
ilm_policy_template: "{{ metricbeat_ilm_policy_template | default('') }}"
ilm_policy_filename: "{{ (metricbeat_ilm_policy_filename | default(default_ilm_policy_filename)) | default('') }}"
ilm_policy_file_location: "{{ (metricbeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}"

View File

@ -305,6 +305,10 @@
notify:
- Enable and restart metricbeat
- include_role:
name: elastic_ilm
when: ilm_policy.keys() | length > 0
- name: Run the beat setup role
include_role:
name: elastic_beat_setup

View File

@ -1479,7 +1479,12 @@ processors:
# Configure rollover index pattern.
#setup.ilm.pattern: "{now/d}-000001"
{% if ilm_policy_name is defined %}
setup.ilm.policy_name: "{{ ilm_policy_name }}"
{% endif %}
{% if ilm_policy_file_location is defined %}
setup.ilm.policy_file: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }}"
{% endif %}
#============================== Kibana =====================================
{% if (groups['kibana'] | length) > 0 %}

View File

@ -14,3 +14,14 @@
# limitations under the License.
packetbeat_service_state: restarted
packetbeat_ilm_policy_name: "packetbeat-{{ beat_version }}"
packetbeat_ilm_policy_template: "packetbeat-{{ beat_version }}"
packetbeat_ilm_policy_filename: "packetbeat-ilm-policy.json"
packetbeat_ilm_policy_file_location: "/etc/packetbeat"
ilm_policy_name: "{{ packetbeat_ilm_policy_name | default('') }}"
ilm_policy: "{{ (packetbeat_ilm_policy | default(default_ilm_policy)) | default({}) }}"
ilm_policy_template: "{{ packetbeat_ilm_policy_template | default('') }}"
ilm_policy_filename: "{{ (packetbeat_ilm_policy_filename | default(default_ilm_policy_filename)) | default('') }}"
ilm_policy_file_location: "{{ (packetbeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}"

View File

@ -73,6 +73,10 @@
notify:
- Enable and restart packetbeat
- include_role:
name: elastic_ilm
when: ilm_policy.keys() | length > 0
- name: Run the beat setup role
include_role:
name: elastic_beat_setup

View File

@ -1204,7 +1204,12 @@ processors:
# Configure rollover index pattern.
#setup.ilm.pattern: "{now/d}-000001"
{% if ilm_policy_name is defined %}
setup.ilm.policy_name: "{{ ilm_policy_name }}"
{% endif %}
{% if ilm_policy_file_location is defined %}
setup.ilm.policy_file: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }}"
{% endif %}
#============================== Kibana =====================================
{% if (groups['kibana'] | length) > 0 %}

View File

@ -411,3 +411,22 @@ elastic_beats:
hosts: "{{ (groups['nova_compute'] | default([])) | union((groups['utility_all'] | default([]))) | union((groups['memcached_all'] | default([]))) }}"
skydive:
hosts: "{{ (((groups['skydive_analyzers'] | default([])) | length) > 0) | ternary((groups['hosts'] | default([])), []) }}"
beat_version: "7.3.2"
default_ilm_rollover_max_size: "30G"
default_ilm_rollover_max_age: "15d"
default_ilm_delete_min_age: "30d"
default_ilm_policy:
policy:
phases:
hot:
actions:
rollover:
max_size: "{{ default_ilm_rollover_max_size }}"
max_age: "{{ default_ilm_rollover_max_age }}"
delete:
min_age: "{{ default_ilm_delete_min_age }}"
actions:
delete: {}
default_ilm_policy_filename: "default-ilm-policy.json"
default_ilm_policy_file_location: "/tmp"