openstack/monasca-persister
Code Issues Proposed changes

Parse envelope fields from kafka to prepare the ES index

Browse Source 
- Change the default index prefix name for events.

Story: 2003955
Task: 26880

Depends-On: https://review.openstack.org/#/c/600030/
Change-Id: Idb6913229d3ac4e391eda0d6129eb2590bc7a678
This commit is contained in:
Martin Chacon Piza 2018-09-05 15:07:27 +02:00
parent 5862969201
commit 473dea341f
5 changed files with 113 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
monasca_persister/conf/elasticsearch.py
View File

@ -20,8 +20,8 @@ from monasca_persister.conf import types
elasticsearch_opts = [ elasticsearch_opts = [
cfg.StrOpt( cfg.StrOpt(
'index_name', 'index_name',
help='Index name where events are stored', help='Index prefix name where events are stored',
default='monevents'), default='events'),
cfg.ListOpt( cfg.ListOpt(
'hosts', 'hosts',
help='List of Elasticsearch nodes in format host[:port]', help='List of Elasticsearch nodes in format host[:port]',

11
monasca_persister/repositories/elasticsearch/events_repository.py
View File

@ -43,16 +43,17 @@ class ElasticSearchEventsRepository(abstract_repository.AbstractRepository):
def write_batch(self, data_points): def write_batch(self, data_points):
for data_point in data_points: for data_point in data_points:
(tenant_id, timestamp, event_type, payload) = data_point (project_id, timestamp, event_type, payload, dimensions) = data_point
index = '%s-%s-%s' % (self.conf.index_name, tenant_id, index = '%s-%s-%s' % (self.conf.index_name, project_id,
ElasticSearchEventsRepository._normalize_timestamp(timestamp)) ElasticSearchEventsRepository._normalize_timestamp(timestamp))
body = { body = {
'tenant_id': tenant_id, 'project_id': project_id,
'timestamp': timestamp, '@timestamp': timestamp,
'event_type': event_type, 'event_type': event_type,
'payload': payload 'payload': payload,
'dimensions': dimensions
} }
self.es.create( self.es.create(

11
monasca_persister/repositories/utils.py
View File

@ -104,9 +104,10 @@ def parse_alarm_state_hist_message(message):
def parse_events_message(message): def parse_events_message(message):
decoded_message = json.loads(message.message.value) decoded_message = json.loads(message.message.value)
event_type = decoded_message['event_type'] event_type = decoded_message['event']['event_type']
timestamp = decoded_message['timestamp'] timestamp = decoded_message['event']['timestamp']
payload = decoded_message['payload'] payload = decoded_message['event']['payload']
tenant_id = payload['tenant_id'] project_id = decoded_message['meta']['project_id']
dimensions = decoded_message['event']['dimensions']
return tenant_id, timestamp, event_type, payload return project_id, timestamp, event_type, payload, dimensions

146
monasca_persister/tests/events.json
View File

@ -1,54 +1,96 @@
{ {
"event_1": { "event_1":{
"event_type": "compute.instance.create.start", "meta":{
"timestamp": "2017-06-01 09:15:11.494606", "project_id":"de98fbff448f4f278a56e9929db70b03"
"payload": {
"state_description": "",
"availability_zone": "nova",
"terminated_at": "",
"ephemeral_gb": 0,
"instance_type_id": 1,
"deleted_at": "",
"reservation_id": "r-74ndofdp",
"instance_id": "cb724671-cc36-49cd-9987-d08f2c8356b9",
"display_name": "fred",
"hostname": "fred",
"state": "building",
"progress": "",
"launched_at": "",
"metadata": {
}, },
"node": null, "creation_time":1536050124,
"ramdisk_id": "", "event":{
"access_ip_v6": null, "dimensions":{
"disk_gb": 1, "service":"compute",
"access_ip_v4": null, "topic":"notification.sample",
"kernel_id": "", "hostname":"nova-compute:compute"
"image_name": "cirros", },
"host": null, "priority":"INFO",
"user_id": "92e0ceb0f3d648ddabeae1bfde4071b2", "timestamp":"2017-06-01 09:15:11.494606",
"image_ref_url": "http://d00-19-99-b3-7e-2e.st.est.fujitsu.com:9292/images/e08428a7-aa34-42bd-9e91-6fe15e0ed2ae", "publisher_id":"nova-compute:compute",
"cell_name": "", "event_type":"compute.instance.create.start",
"root_gb": 1, "payload":{
"tenant_id": "de98fbff448f4f278a56e9929db70b03", "nova_object.version":"1.3",
"created_at": "2017-06-01 09:15:10+00:00", "nova_object.name":"InstanceActionPayload",
"memory_mb": 512, "nova_object.data":{
"instance_type": "m1.tiny", "availability_zone":"nova",
"vcpus": 1, "terminated_at":null,
"image_meta": { "ip_addresses":[
"container_format": "bare", {
"min_ram": "0", "nova_object.version":"1.0",
"disk_format": "qcow2", "nova_object.name":"IpPayload",
"architecture": "x86_64", "nova_object.namespace":"nova",
"min_disk": "1", "nova_object.data":{
"base_image_ref": "e08428a7-aa34-42bd-9e91-6fe15e0ed2ae" "port_uuid":"ce531f90-199f-48c0-816c-13e38010b442",
}, "label":"private-network",
"architecture": "x86_64", "mac":"fa:16:3e:4c:2c:30",
"os_type": null, "version":4,
"instance_flavor_id": "1" "meta":{
},
"priority": "INFO", },
"publisher_id": "compute.d00-26-2d-0c-d5-64", "address":"192.168.1.3",
"message_id": "5e60c9f1-1cf9-4f5d-9826-91fa329a79c1" "device_name":"tapce531f90-19"
} }
} }
],
"ramdisk_id":"",
"image_uuid":"155d900f-4e14-4e4c-a73d-069cbf4541e6",
"flavor":{
"nova_object.version":"1.3",
"nova_object.name":"FlavorPayload",
"nova_object.data":{
"memory_mb":512,
"root_gb":1,
"name":"test_flavor",
"ephemeral_gb":0,
"disabled":false,
"vcpus":1,
"extra_specs":{
"hw:watchdog_action":"disabled"
},
"swap":0,
"rxtx_factor":1.0,
"is_public":true,
"flavorid":"a22d5517-147c-4147-a0d1-e698df5cd4e3",
"vcpu_weight":0,
"projects":null
},
"nova_object.namespace":"nova"
},
"deleted_at":null,
"reservation_id":"r-npxv0e40",
"user_id":"fake",
"uuid":"178b0921-8f85-4257-88b6-2e743b5a975c",
"display_description":"some-server",
"state":"active",
"power_state":"running",
"progress":0,
"launched_at":"2018-07-18T10:02:11Z",
"metadata":{
},
"node":"fake-mini",
"kernel_id":"",
"key_name":"my-key",
"host":"compute",
"display_name":"some-server",
"task_state":null,
"architecture":"x86_64",
"locked":false,
"fault":null,
"created_at":"2018-07-18T10:01:11Z",
"host_name":"some-server",
"tenant_id":"12345678123456781234567812345678",
"auto_disk_config":"MANUAL",
"os_type":null
},
"nova_object.namespace":"nova"
}
}
}
}

7
monasca_persister/tests/test_events.py
View File

@ -36,10 +36,13 @@ class TestEvents(base.BaseTestCase):
def test_parse_event(self): def test_parse_event(self):
event = self._load_event('event_1') event = self._load_event('event_1')
tenant_id, timestamp, event_type, payload = utils.parse_events_message(event) project_id, timestamp, event_type, payload, dimensions = utils.parse_events_message(event)
self.assertEqual('de98fbff448f4f278a56e9929db70b03', tenant_id) self.assertEqual('de98fbff448f4f278a56e9929db70b03', project_id)
self.assertEqual('2017-06-01 09:15:11.494606', timestamp) self.assertEqual('2017-06-01 09:15:11.494606', timestamp)
self.assertEqual('compute.instance.create.start', event_type) self.assertEqual('compute.instance.create.start', event_type)
self.assertEqual('compute', dimensions['service'])
self.assertEqual('notification.sample', dimensions['topic'])
self.assertEqual('nova-compute:compute', dimensions['hostname'])
self.assertIsNotNone(payload) self.assertIsNotNone(payload)
self.assertThat(len(payload), matchers.GreaterThan(0)) self.assertThat(len(payload), matchers.GreaterThan(0))