Neutron Network and Port Notifier support

Change-Id: I9e665896ab9e59bbcced256339cfbc3c677ff5da

devstack/settings

@@ -24,7 +24,7 @@ VITRAGE_USE_MOD_WSGI=${VITRAGE_USE_MOD_WSGI:-${ENABLE_HTTPD_MOD_WSGI_SERVICES}}
 # Toggle for deploying Vitrage with/without nagios
 VITRAGE_USE_NAGIOS=$(trueorfalse False VITRAGE_USE_NAGIOS)
 
-VITRAGE_DEFAULT_DATASOURCES=${VITRAGE_DEFAULT_DATASOURCES:-nova.host,nova.instance,nova.zone,nagios,static_physical,aodh,cinder.volume,neutron.network,neutron.port}
+VITRAGE_DEFAULT_DATASOURCES=${VITRAGE_DEFAULT_DATASOURCES:-nova.host,nova.instance,nova.zone,nagios,static_physical,aodh,cinder.volume}
 
 
 # Tell Tempest this project is present

vitrage/datasources/neutron/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2016 - Nokia Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+__author__ = 'stack'

vitrage/datasources/neutron/network/driver.py

@@ -27,7 +27,9 @@ class NetworkDriver(NeutronBase):
 
     @staticmethod
     def get_event_types(conf):
-        return ['network.']
+        return ['network.create.end',
+                'network.update.end',
+                'network.delete.end']
 
     @staticmethod
     def enrich_event(event, event_type):

vitrage/datasources/neutron/network/transformer.py

@@ -12,6 +12,8 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from oslo_log import log as logging
+
 from vitrage.common.constants import DatasourceProperties as DSProps
 from vitrage.common.constants import EntityCategory
 from vitrage.common.constants import EventAction
@@ -24,8 +26,18 @@ from vitrage.datasources.transformer_base import extract_field_value
 import vitrage.graph.utils as graph_utils
 
 
+LOG = logging.getLogger(__name__)
+
+
 class NetworkTransformer(ResourceTransformerBase):
 
+    UPDATE_ID_PROPERTY = {
+        'network.create.end': ('network', 'id'),
+        'network.update.end': ('network', 'id'),
+        'network.delete.end': ('network_id',),
+        None: ('id',)
+    }
+
     # Event types which need to refer them differently
     UPDATE_EVENT_TYPES = {
         'network.delete.end': EventAction.DELETE_ENTITY,
@@ -34,22 +46,41 @@ class NetworkTransformer(ResourceTransformerBase):
     def __init__(self, transformers):
         super(NetworkTransformer, self).__init__(transformers)
 
-    def _create_entity_key(self, entity_event):
-        network_id = 'network_id' if tbase.is_update_event(entity_event) \
-            else 'id'
-        key_fields = self._key_values(NEUTRON_NETWORK_DATASOURCE,
-                                      extract_field_value(entity_event,
-                                                          network_id))
-        return tbase.build_key(key_fields)
-
     def _create_snapshot_entity_vertex(self, entity_event):
-        name = extract_field_value(entity_event, 'name')
-        entity_id = extract_field_value(entity_event, 'id')
-        state = extract_field_value(entity_event, 'status')
 
-        return self._create_vertex(entity_event, name, entity_id, state)
+        name = entity_event['name']
+        entity_id = entity_event['id']
+        state = entity_event['status']
+        update_timestamp = entity_event['updated_at']
 
-    def _create_vertex(self, entity_event, name, entity_id, state):
+        return self._create_vertex(entity_event,
+                                   name,
+                                   entity_id,
+                                   state,
+                                   update_timestamp)
+
+    def _create_update_entity_vertex(self, entity_event):
+
+        event_type = entity_event[DSProps.EVENT_TYPE]
+        name = extract_field_value(entity_event, 'network', 'name')
+        state = extract_field_value(entity_event, 'network', 'status')
+        update_timestamp = \
+            extract_field_value(entity_event, 'network', 'updated_at')
+        entity_id = extract_field_value(entity_event,
+                                        *self.UPDATE_ID_PROPERTY[event_type])
+
+        return self._create_vertex(entity_event,
+                                   name,
+                                   entity_id,
+                                   state,
+                                   update_timestamp)
+
+    def _create_vertex(self,
+                       entity_event,
+                       name,
+                       entity_id,
+                       state,
+                       update_timestamp):
 
         metadata = {
             VProps.NAME: name,
@@ -57,12 +88,6 @@ class NetworkTransformer(ResourceTransformerBase):
 
         sample_timestamp = entity_event[DSProps.SAMPLE_DATE]
 
-        # TODO(Alexey): need to check here that only the UPDATE sync_mode will
-        #               update the UPDATE_TIMESTAMP property
-        update_timestamp = self._format_update_timestamp(
-            extract_field_value(entity_event, DSProps.SAMPLE_DATE),
-            sample_timestamp)
-
         return graph_utils.create_vertex(
             self._create_entity_key(entity_event),
             entity_id=entity_id,
@@ -73,5 +98,10 @@ class NetworkTransformer(ResourceTransformerBase):
             update_timestamp=update_timestamp,
             metadata=metadata)
 
-    def _create_update_entity_vertex(self, entity_event):
+    def _create_entity_key(self, entity_event):
-        pass
+        event_type = entity_event.get(DSProps.EVENT_TYPE, None)
+        network_id = extract_field_value(entity_event,
+                                         *self.UPDATE_ID_PROPERTY[event_type])
+
+        key_fields = self._key_values(NEUTRON_NETWORK_DATASOURCE, network_id)
+        return tbase.build_key(key_fields)

vitrage/datasources/neutron/port/driver.py

@@ -27,7 +27,9 @@ class PortDriver(NeutronBase):
 
     @staticmethod
     def get_event_types(conf):
-        return ['port.']
+        return ['port.create.end',
+                'port.update.end',
+                'port.delete.end']
 
     @staticmethod
     def enrich_event(event, event_type):

vitrage/datasources/neutron/port/transformer.py

@@ -36,6 +36,19 @@ LOG = logging.getLogger(__name__)
 
 class PortTransformer(ResourceTransformerBase):
 
+    UPDATE_ID_PROPERTY = {
+        'port.create.end': ('port', 'id'),
+        'port.update.end': ('port', 'id'),
+        'port.delete.end': ('port_id',),
+        None: ('id',)
+    }
+
+    FIXED_IPS_PROPERTY = {
+        'port.create.end': ('port', 'fixed_ips'),
+        'port.update.end': ('port', 'fixed_ips'),
+        None: ('fixed_ips',)
+    }
+
     # Event types which need to refer them differently
     UPDATE_EVENT_TYPES = {
         'port.delete.end': EventAction.DELETE_ENTITY,
@@ -44,21 +57,47 @@ class PortTransformer(ResourceTransformerBase):
     def __init__(self, transformers):
         super(PortTransformer, self).__init__(transformers)
 
-    def _create_entity_key(self, entity_event):
-        key_fields = self._key_values(NEUTRON_PORT_DATASOURCE,
-                                      extract_field_value(entity_event, 'id'))
-        return tbase.build_key(key_fields)
-
     def _create_snapshot_entity_vertex(self, entity_event):
-        name = extract_field_value(entity_event, 'name')
-        entity_id = extract_field_value(entity_event, 'id')
-        state = extract_field_value(entity_event, 'status')
 
-        return self._create_vertex(entity_event, name if name else None,
+        name = entity_event['name'] if entity_event['name'] else None
-                                   entity_id, state)
+        entity_id = entity_event['id']
+        state = entity_event['status']
+        update_timestamp = entity_event['updated_at']
 
-    def _create_vertex(self, entity_event, name, entity_id, state):
+        return self._create_vertex(entity_event,
-        ip_addresses = [ip['ip_address'] for ip in entity_event['fixed_ips']]
+                                   name,
+                                   entity_id,
+                                   state,
+                                   update_timestamp)
+
+    def _create_update_entity_vertex(self, entity_event):
+
+        event_type = entity_event[DSProps.EVENT_TYPE]
+        name = extract_field_value(entity_event, 'port', 'name')
+        state = extract_field_value(entity_event, 'port', 'status')
+        update_timestamp = \
+            extract_field_value(entity_event, 'port', 'updated_at')
+        entity_id = extract_field_value(entity_event,
+                                        *self.UPDATE_ID_PROPERTY[event_type])
+
+        return self._create_vertex(entity_event,
+                                   name,
+                                   entity_id,
+                                   state,
+                                   update_timestamp)
+
+    def _create_vertex(self,
+                       entity_event,
+                       name,
+                       entity_id,
+                       state,
+                       update_timestamp):
+        event_type = entity_event.get(DSProps.EVENT_TYPE, None)
+        ip_addresses = []
+        if not event_type:
+            fixed_ips = extract_field_value(
+                entity_event, *self.FIXED_IPS_PROPERTY[event_type])
+            ip_addresses = [ip['ip_address'] for ip in fixed_ips]
         metadata = {
             VProps.NAME: name,
             'ip_addresses': tuple(ip_addresses)
@@ -66,12 +105,6 @@ class PortTransformer(ResourceTransformerBase):
 
         sample_timestamp = entity_event[DSProps.SAMPLE_DATE]
 
-        # TODO(Alexey): need to check here that only the UPDATE sync_mode will
-        #               update the UPDATE_TIMESTAMP property
-        update_timestamp = self._format_update_timestamp(
-            extract_field_value(entity_event, DSProps.SAMPLE_DATE),
-            sample_timestamp)
-
         return graph_utils.create_vertex(
             self._create_entity_key(entity_event),
             entity_id=entity_id,
@@ -82,44 +115,42 @@ class PortTransformer(ResourceTransformerBase):
             update_timestamp=update_timestamp,
             metadata=metadata)
 
-    def _create_update_entity_vertex(self, entity_event):
-        pass
-
     def _create_snapshot_neighbors(self, entity_event):
         return self._create_port_neighbors(entity_event,
-                                           'device_owner',
+                                           ('device_owner',),
-                                           'device_id',
+                                           ('device_id',),
-                                           'network_id')
+                                           ('network_id',))
 
     def _create_update_neighbors(self, entity_event):
         return self._create_port_neighbors(entity_event,
-                                           'device_owner',
+                                           ('port', 'device_owner'),
-                                           'server_uuid',
+                                           ('port', 'device_id'),
-                                           'network_id')
+                                           ('port', 'network_id'))
 
     def _create_port_neighbors(self,
                                entity_event,
                                device_owner_property,
                                device_id_property,
-                               net_id_property):
+                               network_id_property):
+        neighbors = [self._create_network_neighbor(entity_event,
+                                                   network_id_property)]
 
-        instance = None
+        device_owner = \
-        net = self._create_net_neighbor(entity_event,
+            extract_field_value(entity_event, *device_owner_property)
-                                        net_id_property)
+        if device_owner == 'compute:nova' or device_owner == 'compute:None':
-
-        if entity_event[device_owner_property] == 'compute:nova':
             instance = self._create_instance_neighbor(
                 entity_event,
                 device_id_property)
+            neighbors.append(instance)
 
-        return [net, instance] if instance else [net]
+        return neighbors
 
     def _create_instance_neighbor(self,
                                   entity_event,
                                   instance_id_property):
         port_vitrage_id = self._create_entity_key(entity_event)
 
-        instance_id = entity_event[instance_id_property]
+        instance_id = extract_field_value(entity_event, *instance_id_property)
 
         sample_timestamp = entity_event[DSProps.SAMPLE_DATE]
 
@@ -137,10 +168,10 @@ class PortTransformer(ResourceTransformerBase):
 
         return Neighbor(instance_vertex, relationship_edge)
 
-    def _create_net_neighbor(self, entity_event, net_id_property):
+    def _create_network_neighbor(self, entity_event, net_id_property):
         port_vitrage_id = self._create_entity_key(entity_event)
 
-        net_id = entity_event[net_id_property]
+        net_id = extract_field_value(entity_event, *net_id_property)
 
         sample_timestamp = entity_event[DSProps.SAMPLE_DATE]
 
@@ -158,3 +189,12 @@ class PortTransformer(ResourceTransformerBase):
             relationship_type=EdgeLabel.CONTAINS)
 
         return Neighbor(net_vertex, relationship_edge)
+
+    def _create_entity_key(self, entity_event):
+        event_type = entity_event.get(DSProps.EVENT_TYPE, None)
+        port_id = extract_field_value(entity_event,
+                                      *self.UPDATE_ID_PROPERTY[event_type])
+
+        key_fields = self._key_values(NEUTRON_PORT_DATASOURCE, port_id)
+
+        return tbase.build_key(key_fields)

vitrage/datasources/nova/instance/driver.py

@@ -47,8 +47,7 @@ class InstanceDriver(NovaDriverBase):
     @staticmethod
     def get_event_types(conf):
         # Add event_types to receive notifications about
-        return ['compute.instance.create.start',
+        return ['compute.instance.create.error',
-                'compute.instance.create.error',
                 'compute.instance.create.end',
                 'compute.instance.delete.start',
                 'compute.instance.delete.end',
@@ -67,8 +66,7 @@ class InstanceDriver(NovaDriverBase):
                 'compute.instance.volume.attach',
                 'compute.instance.volume.detach',
                 'compute.instance.pause.end',
-                'compute.instance.unpause.end'
+                'compute.instance.unpause.end']
-                ]
 
     @staticmethod
     def get_topic(conf):

vitrage/datasources/transformer_base.py

@@ -43,12 +43,14 @@ AVAILABLE = 'available'
 
 
 def extract_field_value(entity_event, *args):
-
+    try:
         value = entity_event
         for key in args:
             value = value[key]
 
         return value
+    except Exception:
+        return None
 
 
 def build_key(key_values):

vitrage/tests/unit/datasources/neutron/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2016 - Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+__author__ = 'stack'

vitrage/tests/unit/datasources/neutron/network/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2016 - Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+__author__ = 'stack'

vitrage/tests/unit/datasources/neutron/network/test_neutron_network_transformer.py

@@ -0,0 +1,15 @@
+# Copyright 2016 - Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+__author__ = 'stack'

vitrage/tests/unit/datasources/neutron/port/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2016 - Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+__author__ = 'stack'

vitrage/tests/unit/datasources/neutron/port/test_neutron_port_transformer.py

@@ -0,0 +1,15 @@
+# Copyright 2016 - Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+__author__ = 'stack'

vitrage_tempest_tests/tests/api/base.py

@@ -11,6 +11,7 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+
 import time
 
 from oslo_log import log as logging
@@ -21,6 +22,8 @@ from vitrage.common.constants import EntityCategory
 from vitrage.common.constants import VertexProperties as VProps
 from vitrage.datasources import AODH_DATASOURCE
 from vitrage.datasources import CINDER_VOLUME_DATASOURCE
+from vitrage.datasources.neutron.network import NEUTRON_NETWORK_DATASOURCE
+from vitrage.datasources.neutron.port import NEUTRON_PORT_DATASOURCE
 from vitrage.datasources import NOVA_HOST_DATASOURCE
 from vitrage.datasources import NOVA_INSTANCE_DATASOURCE
 from vitrage.datasources import NOVA_ZONE_DATASOURCE
@@ -243,6 +246,20 @@ class BaseApiTest(base.BaseTestCase):
                  self.NUM_EDGES_PER_TYPE: kwargs.get('aodh_edges', 0)}
         validation_data.append(props)
 
+        # neutron.network
+        props = {VProps.CATEGORY: EntityCategory.RESOURCE,
+                 VProps.TYPE: NEUTRON_NETWORK_DATASOURCE,
+                 self.NUM_VERTICES_PER_TYPE: kwargs.get('network_entities', 0),
+                 self.NUM_EDGES_PER_TYPE: kwargs.get('network_edges', 0)}
+        validation_data.append(props)
+
+        # neutron.port
+        props = {VProps.CATEGORY: EntityCategory.RESOURCE,
+                 VProps.TYPE: NEUTRON_PORT_DATASOURCE,
+                 self.NUM_VERTICES_PER_TYPE: kwargs.get('port_entities', 0),
+                 self.NUM_EDGES_PER_TYPE: kwargs.get('port_edges', 0)}
+        validation_data.append(props)
+
         return validation_data
 
     def _validate_graph_correctness(self,