add ilm policies to handle the life of the es indices

two policies are added to handle short and long term indices.

life of the indices can be configured using the 'elasticsearch.life'
field at browbeat-config.yml file
shortterm will have: 125 days
longterm will have 2 years

the policy and the policy based templates can be created using the
'es-template' install playbook.

Change-Id: I0f4a4a9acc03092fd582ae4ff50f688850def953

ansible/install/group_vars/all.yml

@@ -534,7 +534,9 @@ es_ip: 1.1.1.1
 es_local_port: 9200
 es_listen_external: true
 elastic5: false
-elastic7:
+elastic7: true
+shortterm_age: '125d'
+longterm_age: '750d'
 
 ## Filebeat options ##
 filebeat_url: https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.6.0-x86_64.rpm

ansible/install/roles/es-template/tasks/main.yml

@@ -23,14 +23,54 @@
   ignore_errors: true
   when: elastic5
 
-- name: Upload templates
-  uri:
-    url: http://{{ es_ip }}:{{ es_local_port }}/_template/{{ item | basename | regex_replace('\.json','') }}
-    method: PUT
-    body: "{{ lookup('file', item) }}"
-    body_format: json
-  with_fileglob:
-  - "{{ browbeat_path }}/elastic/v7templates/browbeat*"
-  ignore_errors: true
+- name: create ilm policy and templates for esv7
+  block:
+    - name: create shortterm ilm
+      template:
+        src: "{{ browbeat_path }}/elastic/v7ilm_policy/template-browbeat-rally.json.j2"
+        dest: "{{ browbeat_path }}/elastic/v7ilm_policy/browbeat-rally-shortterm.json"
+      vars:
+        age: "{{ shortterm_age }}"
+
+    - name: create longterm ilm
+      template:
+        src: "{{ browbeat_path }}/elastic/v7ilm_policy/template-browbeat-rally.json.j2"
+        dest: "{{ browbeat_path }}/elastic/v7ilm_policy/browbeat-rally-longterm.json"
+      vars:
+        age: "{{ longterm_age }}"
+
+    - name: create shortterm template
+      template:
+        src: "{{ browbeat_path }}/elastic/v7templates/template-browbeat-rally.json.j2"
+        dest: "{{ browbeat_path }}/elastic/v7templates/browbeat-rally-shortterm.json"
+      vars:
+        term: "shortterm"
+
+    - name: create longterm template
+      template:
+        src: "{{ browbeat_path }}/elastic/v7templates/template-browbeat-rally.json.j2"
+        dest: "{{ browbeat_path }}/elastic/v7templates/browbeat-rally-longterm.json"
+      vars:
+        term: "longterm"
+
+    - name: Upload ilm policy
+      uri:
+        url: http://{{ es_ip }}:{{es_local_port }}/_ilm/policy/{{ item | basename | regex_replace('\.json','') }}
+        method: PUT
+        body: "{{ lookup('file', item) }}"
+        body_format: json
+      with_fileglob:
+      - "{{ browbeat_path }}/elastic/v7ilm_policy/browbeat*"
+      ignore_errors: true
+
+    - name: Upload templates
+      uri:
+        url: http://{{ es_ip }}:{{ es_local_port }}/_template/{{ item | basename | regex_replace('\.json','') }}
+        method: PUT
+        body: "{{ lookup('file', item) }}"
+        body_format: json
+      with_fileglob:
+      - "{{ browbeat_path }}/elastic/v7templates/browbeat*"
+      ignore_errors: true
   when: elastic7
 

browbeat-complete.yaml

@@ -23,6 +23,7 @@ elasticsearch:
   enabled: true
   host: 1.1.1.1
   port: 9200
+  life: shortterm
   regather: true
   metadata_files:
     - name: hardware-metadata

browbeat-config.yaml

@@ -37,6 +37,8 @@ elasticsearch:
   enabled: false
   host: 1.1.1.1
   port: 9200
+  #allowed values: shortterm, longterm
+  life: shortterm
   regather: false
   metadata_files:
     - name: hardware-metadata

browbeat/elastic.py

@@ -46,10 +46,11 @@ class Elastic(object):
              'port': self.config['elasticsearch']['port']}],
             send_get_body_as='POST'
         )
+        life = self.config['elasticsearch']['life']
         self.workload = workload
         today = datetime.datetime.today()
-        self.index = "{}-{}-{}".format(tool, workload,
-                                       today.strftime('%Y.%m'))
+        self.index = "{}-{}-{}-{}".format(tool, workload, life,
+                                          today.strftime('%Y.%m'))
 
     def __del__(self):
         self.flush_cache()

browbeat/schema/browbeat.yml

@@ -75,6 +75,9 @@ mapping:
       port:
         type: int
         required: True
+      life:
+        type: str
+        required: True
       regather:
         type: bool
         required: True

conf/browbeat-glance-complete.yaml

@@ -17,6 +17,7 @@ elasticsearch:
   enabled: false
   host: 1.1.1.1
   port: 9200
+  life: shortterm
   regather: false
   metadata_files:
     - name: hardware-metadata

conf/browbeat-keystone-complete.yaml

@@ -18,6 +18,7 @@ elasticsearch:
   enabled: false
   host: 1.1.1.1
   port: 9200
+  life: shortterm
   regather: false
   metadata_files:
     - name: hardware-metadata

conf/browbeat-keystone-minimal.yaml

@@ -18,6 +18,7 @@ elasticsearch:
   enabled: false
   host: 1.1.1.1
   port: 9200
+  life: shortterm
   regather: false
   metadata_files:
     - name: hardware-metadata

conf/browbeat-workloads.yml

@@ -17,6 +17,7 @@ elasticsearch:
   enabled: false
   host: 1.1.1.1
   port: 9200
+  life: shortterm
   regather: false
   metadata_files:
     - name: hardware-metadata

conf/ceilometer-minimal.yaml

@@ -17,6 +17,7 @@ elasticsearch:
   enabled: false
   host: 1.1.1.1
   port: 9200
+  life: shortterm
   regather: false
   metadata_files:
     - name: hardware-metadata

conf/ceilometer-persist-response-timing.yaml

@@ -20,6 +20,7 @@ elasticsearch:
   enabled: false
   host: 1.1.1.1
   port: 9200
+  life: shortterm
   regather: false
   metadata_files:
     - name: hardware-metadata

conf/gnocchi-minimal.yaml

@@ -18,6 +18,7 @@ elasticsearch:
   enabled: false
   host: 1.1.1.1
   port: 9200
+  life: shortterm
   regather: false
   metadata_files:
     - name: hardware-metadata

conf/quickstart.yml

@@ -17,6 +17,7 @@ elasticsearch:
   enabled: false
   host: 1.1.1.1
   port: 9200
+  life: shortterm
   regather: false
   metadata_files:
     - name: hardware-metadata

conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml

@@ -24,6 +24,7 @@ elasticsearch:
   enabled: false
   host: 1.1.1.1
   port: 9200
+  life: shortterm
   regather: false
   metadata_files:
     - name: hardware-metadata

conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml

@@ -24,6 +24,7 @@ elasticsearch:
   enabled: false
   host: 1.1.1.1
   port: 9200
+  life: shortterm
   regather: false
   metadata_files:
     - name: hardware-metadata

conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml

@@ -24,6 +24,7 @@ elasticsearch:
   enabled: false
   host: 1.1.1.1
   port: 9200
+  life: shortterm
   regather: false
   metadata_files:
     - name: hardware-metadata

conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml

@@ -24,6 +24,7 @@ elasticsearch:
   enabled: false
   host: 1.1.1.1
   port: 9200
+  life: shortterm
   regather: false
   metadata_files:
     - name: hardware-metadata

conf/telemetry-nova-persist-instances-with-network-fip.yaml

@@ -21,6 +21,7 @@ elasticsearch:
   enabled: false
   host: 1.1.1.1
   port: 9200
+  life: shortterm
   regather: false
   metadata_files:
     - name: hardware-metadata

conf/telemetry-nova-persist-instances-with-network-volume-fip.yaml

@@ -21,6 +21,7 @@ elasticsearch:
   enabled: false
   host: 1.1.1.1
   port: 9200
+  life: shortterm
   regather: false
   metadata_files:
     - name: hardware-metadata

conf/telemetry-nova-persist-instances-with-network-volume.yaml

@@ -20,6 +20,7 @@ elasticsearch:
   enabled: false
   host: 1.1.1.1
   port: 9200
+  life: shortterm
   regather: false
   metadata_files:
     - name: hardware-metadata

conf/telemetry-nova-persist-instances-with-network.yaml

@@ -22,6 +22,7 @@ elasticsearch:
   enabled: false
   host: 1.1.1.1
   port: 9200
+  life: shortterm
   regather: false
   metadata_files:
     - name: hardware-metadata

conf/telemetry-nova-persist-instances-with-volume.yaml

@@ -20,6 +20,7 @@ elasticsearch:
   enabled: false
   host: 1.1.1.1
   port: 9200
+  life: shortterm
   regather: false
   metadata_files:
     - name: hardware-metadata

conf/telemetry-nova-persist-instances.yaml

@@ -21,6 +21,7 @@ elasticsearch:
   enabled: false
   host: 1.1.1.1
   port: 9200
+  life: shortterm
   regather: false
   metadata_files:
     - name: hardware-metadata

elastic/v7ilm_policy/template-browbeat-rally.json.j2

@@ -0,0 +1,12 @@
+{
+  "policy": {
+    "phases": {
+      "delete": {
+        "min_age": "{{ age }}",
+        "actions": {
+          "delete": {}
+        }
+      }
+    }
+  }
+}

tests/data/valid_browbeat.yml

@@ -17,6 +17,7 @@ elasticsearch:
   enabled: false
   host: browbeat.test.com
   port: 9200
+  life: shortterm
   regather: false
   metadata_files:
     - name: hardware-metadata