add ilm policies to handle the life of the es indices
two policies are added to handle short and long term indices. life of the indices can be configured using the 'elasticsearch.life' field at browbeat-config.yml file shortterm will have: 125 days longterm will have 2 years the policy and the policy based templates can be created using the 'es-template' install playbook. Change-Id: I0f4a4a9acc03092fd582ae4ff50f688850def953
This commit is contained in:
parent
173c9a1afa
commit
8632866417
@ -534,7 +534,9 @@ es_ip: 1.1.1.1
|
|||||||
es_local_port: 9200
|
es_local_port: 9200
|
||||||
es_listen_external: true
|
es_listen_external: true
|
||||||
elastic5: false
|
elastic5: false
|
||||||
elastic7:
|
elastic7: true
|
||||||
|
shortterm_age: '125d'
|
||||||
|
longterm_age: '750d'
|
||||||
|
|
||||||
## Filebeat options ##
|
## Filebeat options ##
|
||||||
filebeat_url: https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.6.0-x86_64.rpm
|
filebeat_url: https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.6.0-x86_64.rpm
|
||||||
|
@ -23,14 +23,54 @@
|
|||||||
ignore_errors: true
|
ignore_errors: true
|
||||||
when: elastic5
|
when: elastic5
|
||||||
|
|
||||||
- name: Upload templates
|
- name: create ilm policy and templates for esv7
|
||||||
uri:
|
block:
|
||||||
url: http://{{ es_ip }}:{{ es_local_port }}/_template/{{ item | basename | regex_replace('\.json','') }}
|
- name: create shortterm ilm
|
||||||
method: PUT
|
template:
|
||||||
body: "{{ lookup('file', item) }}"
|
src: "{{ browbeat_path }}/elastic/v7ilm_policy/template-browbeat-rally.json.j2"
|
||||||
body_format: json
|
dest: "{{ browbeat_path }}/elastic/v7ilm_policy/browbeat-rally-shortterm.json"
|
||||||
with_fileglob:
|
vars:
|
||||||
- "{{ browbeat_path }}/elastic/v7templates/browbeat*"
|
age: "{{ shortterm_age }}"
|
||||||
ignore_errors: true
|
|
||||||
|
- name: create longterm ilm
|
||||||
|
template:
|
||||||
|
src: "{{ browbeat_path }}/elastic/v7ilm_policy/template-browbeat-rally.json.j2"
|
||||||
|
dest: "{{ browbeat_path }}/elastic/v7ilm_policy/browbeat-rally-longterm.json"
|
||||||
|
vars:
|
||||||
|
age: "{{ longterm_age }}"
|
||||||
|
|
||||||
|
- name: create shortterm template
|
||||||
|
template:
|
||||||
|
src: "{{ browbeat_path }}/elastic/v7templates/template-browbeat-rally.json.j2"
|
||||||
|
dest: "{{ browbeat_path }}/elastic/v7templates/browbeat-rally-shortterm.json"
|
||||||
|
vars:
|
||||||
|
term: "shortterm"
|
||||||
|
|
||||||
|
- name: create longterm template
|
||||||
|
template:
|
||||||
|
src: "{{ browbeat_path }}/elastic/v7templates/template-browbeat-rally.json.j2"
|
||||||
|
dest: "{{ browbeat_path }}/elastic/v7templates/browbeat-rally-longterm.json"
|
||||||
|
vars:
|
||||||
|
term: "longterm"
|
||||||
|
|
||||||
|
- name: Upload ilm policy
|
||||||
|
uri:
|
||||||
|
url: http://{{ es_ip }}:{{es_local_port }}/_ilm/policy/{{ item | basename | regex_replace('\.json','') }}
|
||||||
|
method: PUT
|
||||||
|
body: "{{ lookup('file', item) }}"
|
||||||
|
body_format: json
|
||||||
|
with_fileglob:
|
||||||
|
- "{{ browbeat_path }}/elastic/v7ilm_policy/browbeat*"
|
||||||
|
ignore_errors: true
|
||||||
|
|
||||||
|
- name: Upload templates
|
||||||
|
uri:
|
||||||
|
url: http://{{ es_ip }}:{{ es_local_port }}/_template/{{ item | basename | regex_replace('\.json','') }}
|
||||||
|
method: PUT
|
||||||
|
body: "{{ lookup('file', item) }}"
|
||||||
|
body_format: json
|
||||||
|
with_fileglob:
|
||||||
|
- "{{ browbeat_path }}/elastic/v7templates/browbeat*"
|
||||||
|
ignore_errors: true
|
||||||
when: elastic7
|
when: elastic7
|
||||||
|
|
||||||
|
@ -23,6 +23,7 @@ elasticsearch:
|
|||||||
enabled: true
|
enabled: true
|
||||||
host: 1.1.1.1
|
host: 1.1.1.1
|
||||||
port: 9200
|
port: 9200
|
||||||
|
life: shortterm
|
||||||
regather: true
|
regather: true
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
@ -37,6 +37,8 @@ elasticsearch:
|
|||||||
enabled: false
|
enabled: false
|
||||||
host: 1.1.1.1
|
host: 1.1.1.1
|
||||||
port: 9200
|
port: 9200
|
||||||
|
#allowed values: shortterm, longterm
|
||||||
|
life: shortterm
|
||||||
regather: false
|
regather: false
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
@ -46,10 +46,11 @@ class Elastic(object):
|
|||||||
'port': self.config['elasticsearch']['port']}],
|
'port': self.config['elasticsearch']['port']}],
|
||||||
send_get_body_as='POST'
|
send_get_body_as='POST'
|
||||||
)
|
)
|
||||||
|
life = self.config['elasticsearch']['life']
|
||||||
self.workload = workload
|
self.workload = workload
|
||||||
today = datetime.datetime.today()
|
today = datetime.datetime.today()
|
||||||
self.index = "{}-{}-{}".format(tool, workload,
|
self.index = "{}-{}-{}-{}".format(tool, workload, life,
|
||||||
today.strftime('%Y.%m'))
|
today.strftime('%Y.%m'))
|
||||||
|
|
||||||
def __del__(self):
|
def __del__(self):
|
||||||
self.flush_cache()
|
self.flush_cache()
|
||||||
|
@ -75,6 +75,9 @@ mapping:
|
|||||||
port:
|
port:
|
||||||
type: int
|
type: int
|
||||||
required: True
|
required: True
|
||||||
|
life:
|
||||||
|
type: str
|
||||||
|
required: True
|
||||||
regather:
|
regather:
|
||||||
type: bool
|
type: bool
|
||||||
required: True
|
required: True
|
||||||
|
@ -17,6 +17,7 @@ elasticsearch:
|
|||||||
enabled: false
|
enabled: false
|
||||||
host: 1.1.1.1
|
host: 1.1.1.1
|
||||||
port: 9200
|
port: 9200
|
||||||
|
life: shortterm
|
||||||
regather: false
|
regather: false
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
@ -18,6 +18,7 @@ elasticsearch:
|
|||||||
enabled: false
|
enabled: false
|
||||||
host: 1.1.1.1
|
host: 1.1.1.1
|
||||||
port: 9200
|
port: 9200
|
||||||
|
life: shortterm
|
||||||
regather: false
|
regather: false
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
@ -18,6 +18,7 @@ elasticsearch:
|
|||||||
enabled: false
|
enabled: false
|
||||||
host: 1.1.1.1
|
host: 1.1.1.1
|
||||||
port: 9200
|
port: 9200
|
||||||
|
life: shortterm
|
||||||
regather: false
|
regather: false
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
@ -17,6 +17,7 @@ elasticsearch:
|
|||||||
enabled: false
|
enabled: false
|
||||||
host: 1.1.1.1
|
host: 1.1.1.1
|
||||||
port: 9200
|
port: 9200
|
||||||
|
life: shortterm
|
||||||
regather: false
|
regather: false
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
@ -17,6 +17,7 @@ elasticsearch:
|
|||||||
enabled: false
|
enabled: false
|
||||||
host: 1.1.1.1
|
host: 1.1.1.1
|
||||||
port: 9200
|
port: 9200
|
||||||
|
life: shortterm
|
||||||
regather: false
|
regather: false
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
@ -20,6 +20,7 @@ elasticsearch:
|
|||||||
enabled: false
|
enabled: false
|
||||||
host: 1.1.1.1
|
host: 1.1.1.1
|
||||||
port: 9200
|
port: 9200
|
||||||
|
life: shortterm
|
||||||
regather: false
|
regather: false
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
@ -18,6 +18,7 @@ elasticsearch:
|
|||||||
enabled: false
|
enabled: false
|
||||||
host: 1.1.1.1
|
host: 1.1.1.1
|
||||||
port: 9200
|
port: 9200
|
||||||
|
life: shortterm
|
||||||
regather: false
|
regather: false
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
@ -17,6 +17,7 @@ elasticsearch:
|
|||||||
enabled: false
|
enabled: false
|
||||||
host: 1.1.1.1
|
host: 1.1.1.1
|
||||||
port: 9200
|
port: 9200
|
||||||
|
life: shortterm
|
||||||
regather: false
|
regather: false
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
@ -24,6 +24,7 @@ elasticsearch:
|
|||||||
enabled: false
|
enabled: false
|
||||||
host: 1.1.1.1
|
host: 1.1.1.1
|
||||||
port: 9200
|
port: 9200
|
||||||
|
life: shortterm
|
||||||
regather: false
|
regather: false
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
@ -24,6 +24,7 @@ elasticsearch:
|
|||||||
enabled: false
|
enabled: false
|
||||||
host: 1.1.1.1
|
host: 1.1.1.1
|
||||||
port: 9200
|
port: 9200
|
||||||
|
life: shortterm
|
||||||
regather: false
|
regather: false
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
@ -24,6 +24,7 @@ elasticsearch:
|
|||||||
enabled: false
|
enabled: false
|
||||||
host: 1.1.1.1
|
host: 1.1.1.1
|
||||||
port: 9200
|
port: 9200
|
||||||
|
life: shortterm
|
||||||
regather: false
|
regather: false
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
@ -24,6 +24,7 @@ elasticsearch:
|
|||||||
enabled: false
|
enabled: false
|
||||||
host: 1.1.1.1
|
host: 1.1.1.1
|
||||||
port: 9200
|
port: 9200
|
||||||
|
life: shortterm
|
||||||
regather: false
|
regather: false
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
@ -21,6 +21,7 @@ elasticsearch:
|
|||||||
enabled: false
|
enabled: false
|
||||||
host: 1.1.1.1
|
host: 1.1.1.1
|
||||||
port: 9200
|
port: 9200
|
||||||
|
life: shortterm
|
||||||
regather: false
|
regather: false
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
@ -21,6 +21,7 @@ elasticsearch:
|
|||||||
enabled: false
|
enabled: false
|
||||||
host: 1.1.1.1
|
host: 1.1.1.1
|
||||||
port: 9200
|
port: 9200
|
||||||
|
life: shortterm
|
||||||
regather: false
|
regather: false
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
@ -20,6 +20,7 @@ elasticsearch:
|
|||||||
enabled: false
|
enabled: false
|
||||||
host: 1.1.1.1
|
host: 1.1.1.1
|
||||||
port: 9200
|
port: 9200
|
||||||
|
life: shortterm
|
||||||
regather: false
|
regather: false
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
@ -22,6 +22,7 @@ elasticsearch:
|
|||||||
enabled: false
|
enabled: false
|
||||||
host: 1.1.1.1
|
host: 1.1.1.1
|
||||||
port: 9200
|
port: 9200
|
||||||
|
life: shortterm
|
||||||
regather: false
|
regather: false
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
@ -20,6 +20,7 @@ elasticsearch:
|
|||||||
enabled: false
|
enabled: false
|
||||||
host: 1.1.1.1
|
host: 1.1.1.1
|
||||||
port: 9200
|
port: 9200
|
||||||
|
life: shortterm
|
||||||
regather: false
|
regather: false
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
@ -21,6 +21,7 @@ elasticsearch:
|
|||||||
enabled: false
|
enabled: false
|
||||||
host: 1.1.1.1
|
host: 1.1.1.1
|
||||||
port: 9200
|
port: 9200
|
||||||
|
life: shortterm
|
||||||
regather: false
|
regather: false
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
12
elastic/v7ilm_policy/template-browbeat-rally.json.j2
Normal file
12
elastic/v7ilm_policy/template-browbeat-rally.json.j2
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"policy": {
|
||||||
|
"phases": {
|
||||||
|
"delete": {
|
||||||
|
"min_age": "{{ age }}",
|
||||||
|
"actions": {
|
||||||
|
"delete": {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
File diff suppressed because it is too large
Load Diff
4392
elastic/v7templates/template-browbeat-rally.json.j2
Normal file
4392
elastic/v7templates/template-browbeat-rally.json.j2
Normal file
File diff suppressed because it is too large
Load Diff
@ -17,6 +17,7 @@ elasticsearch:
|
|||||||
enabled: false
|
enabled: false
|
||||||
host: browbeat.test.com
|
host: browbeat.test.com
|
||||||
port: 9200
|
port: 9200
|
||||||
|
life: shortterm
|
||||||
regather: false
|
regather: false
|
||||||
metadata_files:
|
metadata_files:
|
||||||
- name: hardware-metadata
|
- name: hardware-metadata
|
||||||
|
Loading…
Reference in New Issue
Block a user