add ilm policies to handle the life of the es indices

two policies are added to handle short and long term indices.

life of the indices can be configured using the 'elasticsearch.life'
field at browbeat-config.yml file
shortterm will have: 125 days
longterm will have 2 years

the policy and the policy based templates can be created using the
'es-template' install playbook.

Change-Id: I0f4a4a9acc03092fd582ae4ff50f688850def953
This commit is contained in:
Masco 2023-08-22 18:23:55 +05:30
parent 173c9a1afa
commit 8632866417
28 changed files with 4484 additions and 4405 deletions

View File

@ -534,7 +534,9 @@ es_ip: 1.1.1.1
es_local_port: 9200
es_listen_external: true
elastic5: false
elastic7:
elastic7: true
shortterm_age: '125d'
longterm_age: '750d'
## Filebeat options ##
filebeat_url: https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.6.0-x86_64.rpm

View File

@ -23,7 +23,47 @@
ignore_errors: true
when: elastic5
- name: Upload templates
- name: create ilm policy and templates for esv7
block:
- name: create shortterm ilm
template:
src: "{{ browbeat_path }}/elastic/v7ilm_policy/template-browbeat-rally.json.j2"
dest: "{{ browbeat_path }}/elastic/v7ilm_policy/browbeat-rally-shortterm.json"
vars:
age: "{{ shortterm_age }}"
- name: create longterm ilm
template:
src: "{{ browbeat_path }}/elastic/v7ilm_policy/template-browbeat-rally.json.j2"
dest: "{{ browbeat_path }}/elastic/v7ilm_policy/browbeat-rally-longterm.json"
vars:
age: "{{ longterm_age }}"
- name: create shortterm template
template:
src: "{{ browbeat_path }}/elastic/v7templates/template-browbeat-rally.json.j2"
dest: "{{ browbeat_path }}/elastic/v7templates/browbeat-rally-shortterm.json"
vars:
term: "shortterm"
- name: create longterm template
template:
src: "{{ browbeat_path }}/elastic/v7templates/template-browbeat-rally.json.j2"
dest: "{{ browbeat_path }}/elastic/v7templates/browbeat-rally-longterm.json"
vars:
term: "longterm"
- name: Upload ilm policy
uri:
url: http://{{ es_ip }}:{{es_local_port }}/_ilm/policy/{{ item | basename | regex_replace('\.json','') }}
method: PUT
body: "{{ lookup('file', item) }}"
body_format: json
with_fileglob:
- "{{ browbeat_path }}/elastic/v7ilm_policy/browbeat*"
ignore_errors: true
- name: Upload templates
uri:
url: http://{{ es_ip }}:{{ es_local_port }}/_template/{{ item | basename | regex_replace('\.json','') }}
method: PUT

View File

@ -23,6 +23,7 @@ elasticsearch:
enabled: true
host: 1.1.1.1
port: 9200
life: shortterm
regather: true
metadata_files:
- name: hardware-metadata

View File

@ -37,6 +37,8 @@ elasticsearch:
enabled: false
host: 1.1.1.1
port: 9200
#allowed values: shortterm, longterm
life: shortterm
regather: false
metadata_files:
- name: hardware-metadata

View File

@ -46,9 +46,10 @@ class Elastic(object):
'port': self.config['elasticsearch']['port']}],
send_get_body_as='POST'
)
life = self.config['elasticsearch']['life']
self.workload = workload
today = datetime.datetime.today()
self.index = "{}-{}-{}".format(tool, workload,
self.index = "{}-{}-{}-{}".format(tool, workload, life,
today.strftime('%Y.%m'))
def __del__(self):

View File

@ -75,6 +75,9 @@ mapping:
port:
type: int
required: True
life:
type: str
required: True
regather:
type: bool
required: True

View File

@ -17,6 +17,7 @@ elasticsearch:
enabled: false
host: 1.1.1.1
port: 9200
life: shortterm
regather: false
metadata_files:
- name: hardware-metadata

View File

@ -18,6 +18,7 @@ elasticsearch:
enabled: false
host: 1.1.1.1
port: 9200
life: shortterm
regather: false
metadata_files:
- name: hardware-metadata

View File

@ -18,6 +18,7 @@ elasticsearch:
enabled: false
host: 1.1.1.1
port: 9200
life: shortterm
regather: false
metadata_files:
- name: hardware-metadata

View File

@ -17,6 +17,7 @@ elasticsearch:
enabled: false
host: 1.1.1.1
port: 9200
life: shortterm
regather: false
metadata_files:
- name: hardware-metadata

View File

@ -17,6 +17,7 @@ elasticsearch:
enabled: false
host: 1.1.1.1
port: 9200
life: shortterm
regather: false
metadata_files:
- name: hardware-metadata

View File

@ -20,6 +20,7 @@ elasticsearch:
enabled: false
host: 1.1.1.1
port: 9200
life: shortterm
regather: false
metadata_files:
- name: hardware-metadata

View File

@ -18,6 +18,7 @@ elasticsearch:
enabled: false
host: 1.1.1.1
port: 9200
life: shortterm
regather: false
metadata_files:
- name: hardware-metadata

View File

@ -17,6 +17,7 @@ elasticsearch:
enabled: false
host: 1.1.1.1
port: 9200
life: shortterm
regather: false
metadata_files:
- name: hardware-metadata

View File

@ -24,6 +24,7 @@ elasticsearch:
enabled: false
host: 1.1.1.1
port: 9200
life: shortterm
regather: false
metadata_files:
- name: hardware-metadata

View File

@ -24,6 +24,7 @@ elasticsearch:
enabled: false
host: 1.1.1.1
port: 9200
life: shortterm
regather: false
metadata_files:
- name: hardware-metadata

View File

@ -24,6 +24,7 @@ elasticsearch:
enabled: false
host: 1.1.1.1
port: 9200
life: shortterm
regather: false
metadata_files:
- name: hardware-metadata

View File

@ -24,6 +24,7 @@ elasticsearch:
enabled: false
host: 1.1.1.1
port: 9200
life: shortterm
regather: false
metadata_files:
- name: hardware-metadata

View File

@ -21,6 +21,7 @@ elasticsearch:
enabled: false
host: 1.1.1.1
port: 9200
life: shortterm
regather: false
metadata_files:
- name: hardware-metadata

View File

@ -21,6 +21,7 @@ elasticsearch:
enabled: false
host: 1.1.1.1
port: 9200
life: shortterm
regather: false
metadata_files:
- name: hardware-metadata

View File

@ -20,6 +20,7 @@ elasticsearch:
enabled: false
host: 1.1.1.1
port: 9200
life: shortterm
regather: false
metadata_files:
- name: hardware-metadata

View File

@ -22,6 +22,7 @@ elasticsearch:
enabled: false
host: 1.1.1.1
port: 9200
life: shortterm
regather: false
metadata_files:
- name: hardware-metadata

View File

@ -20,6 +20,7 @@ elasticsearch:
enabled: false
host: 1.1.1.1
port: 9200
life: shortterm
regather: false
metadata_files:
- name: hardware-metadata

View File

@ -21,6 +21,7 @@ elasticsearch:
enabled: false
host: 1.1.1.1
port: 9200
life: shortterm
regather: false
metadata_files:
- name: hardware-metadata

View File

@ -0,0 +1,12 @@
{
"policy": {
"phases": {
"delete": {
"min_age": "{{ age }}",
"actions": {
"delete": {}
}
}
}
}
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -17,6 +17,7 @@ elasticsearch:
enabled: false
host: browbeat.test.com
port: 9200
life: shortterm
regather: false
metadata_files:
- name: hardware-metadata