ES 7 update

Change-Id: Ia0360b04020188812ed2a4fb36e05e8f7a73a9bc

ansible/install/group_vars/all.yml

@@ -425,6 +425,7 @@ es_ip: 1.1.1.1
 es_local_port: 9200
 es_listen_external: true
 elastic5: false
+elastic7:
 
 ## Filebeat options ##
 filebeat_url: https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.6.0-x86_64.rpm

ansible/install/roles/es-template/tasks/main.yml

@@ -2,19 +2,19 @@
 
 - name: Upload templates
   uri:
-    url: http://{{ es_ip }}:{{ es_local_port }}/_template/{{ item | basename | regex_replace('\.json','') }}*
+    url: http://{{ es_ip }}:{{ es_local_port }}/_template/{{ item | basename | regex_replace('\.json','') }}
     method: PUT
     body: "{{ lookup('file', item) }}"
     body_format: json
   with_fileglob:
   - "{{ browbeat_path }}/elastic/templates/browbeat*"
   ignore_errors: true
-  when: not elastic5
+  when: not (elastic5 or elastic7)
 
 
 - name: Upload templates
   uri:
-    url: http://{{ es_ip }}:{{ es_local_port }}/_template/{{ item | basename | regex_replace('\.json','') }}*
+    url: http://{{ es_ip }}:{{ es_local_port }}/_template/{{ item | basename | regex_replace('\.json','') }}
     method: PUT
     body: "{{ lookup('file', item) }}"
     body_format: json
@@ -22,3 +22,15 @@
   - "{{ browbeat_path }}/elastic/v5templates/browbeat*"
   ignore_errors: true
   when: elastic5
+
+- name: Upload templates
+  uri:
+    url: http://{{ es_ip }}:{{ es_local_port }}/_template/{{ item | basename | regex_replace('\.json','') }}
+    method: PUT
+    body: "{{ lookup('file', item) }}"
+    body_format: json
+  with_fileglob:
+  - "{{ browbeat_path }}/elastic/v7templates/browbeat*"
+  ignore_errors: true
+  when: elastic7
+

browbeat/elastic.py

@@ -48,8 +48,8 @@ class Elastic(object):
         )
         self.workload = workload
         today = datetime.datetime.today()
-        self.index = "{}-{}-{}".format(tool,
+        self.index = "{}-{}-{}".format(tool, workload,
-                                       workload, today.strftime('%Y.%m.%d'))
+                                       today.strftime('%Y.%m'))
 
     def __del__(self):
         self.flush_cache()
@@ -97,7 +97,6 @@ class Elastic(object):
             es_item = {}
             es_item['_id'] = item['_id']
             es_item['_source'] = item['result']
-            es_item['_type'] = item['_type']
             es_item['_index'] = self.index
             output.append(es_item)
         return output
@@ -667,6 +666,7 @@ class Elastic(object):
         body = {"query": {
                 "bool": {"must": [
                     {"term": {"browbeat_uuid": browbeat_id}},
+                    {"term": {"doc_type": 'error'}}
                 ]}
                 }}
         """
@@ -680,7 +680,6 @@ class Elastic(object):
         """
         page = self.es.search(
             index=index,
-            doc_type='error',
             scroll='1m',
             size=1000,
             body=body,
@@ -702,7 +701,6 @@ class Elastic(object):
         self.logger.info("Making query against {}".format(index))
         page = self.es.search(
             index=index,
-            doc_type='result',
             scroll='1m',
             size=1000,
             body=body,
@@ -732,10 +730,10 @@ class Elastic(object):
         result['cloud_name'] = self.config['browbeat']['cloud_name']
         result['browbeat_config'] = self.config
         data['result'] = result
+        data['result']['doc_type'] = _type
         data['test_name'] = test_name
         data['result_dir'] = result_dir
         data['identifier'] = identifier
-        data['_type'] = _type
         data['_id'] = _id
         self.cache.append(data)
         now = datetime.datetime.utcnow()

elastic/templates/browbeat-logstash.json

@@ -1,5 +1,5 @@
 {
-    "template": "logstash-*",
+    "template": "browbeat-logstash-*",
     "settings": {
         "index": {
             "number_of_shards": "5",