From c4598c73fab61ba597b42f8bf0519b2bd370be27 Mon Sep 17 00:00:00 2001 From: Alexey Weyl Date: Wed, 15 Jun 2016 11:16:10 +0300 Subject: [PATCH] Neutron Network and Port Notifier support Change-Id: I9e665896ab9e59bbcced256339cfbc3c677ff5da --- devstack/post_test_hook.sh | 2 +- devstack/settings | 2 +- vitrage/datasources/neutron/__init__.py | 15 +++ vitrage/datasources/neutron/network/driver.py | 4 +- .../neutron/network/transformer.py | 72 +++++++---- vitrage/datasources/neutron/port/driver.py | 4 +- .../datasources/neutron/port/transformer.py | 114 ++++++++++++------ vitrage/datasources/nova/instance/driver.py | 6 +- vitrage/datasources/transformer_base.py | 12 +- .../unit/datasources/neutron/__init__.py | 15 +++ .../datasources/neutron/network/__init__.py | 15 +++ .../test_neutron_network_transformer.py | 15 +++ .../unit/datasources/neutron/port/__init__.py | 15 +++ .../port/test_neutron_port_transformer.py | 15 +++ vitrage_tempest_tests/tests/api/base.py | 17 +++ 15 files changed, 252 insertions(+), 71 deletions(-) create mode 100644 vitrage/tests/unit/datasources/neutron/__init__.py create mode 100644 vitrage/tests/unit/datasources/neutron/network/__init__.py create mode 100644 vitrage/tests/unit/datasources/neutron/network/test_neutron_network_transformer.py create mode 100644 vitrage/tests/unit/datasources/neutron/port/__init__.py create mode 100644 vitrage/tests/unit/datasources/neutron/port/test_neutron_port_transformer.py diff --git a/devstack/post_test_hook.sh b/devstack/post_test_hook.sh index db79869b3..6f8437125 100644 --- a/devstack/post_test_hook.sh +++ b/devstack/post_test_hook.sh @@ -28,7 +28,7 @@ fi (cd $DEVSTACK_PATH/tempest/; sudo pip install -r requirements.txt -r test-requirements.txt) -(cd $DEVSTACK_PATH/tempest/; sudo oslo-config-generator --config-file etc/config-generator.tempest.conf --output-file etc/tempest.conf) +(cd $DEVSTACK_PATH/tempest/; sudo oslo-config-generator --config-file etc/config-generator.tempest.conf --output-file etc/tempest.conf) (cd $DEVSTACK_PATH/; sudo sh -c 'cp -rf vitrage/devstack/files/tempest/tempest.conf /etc/tempest.conf') (cd $DEVSTACK_PATH/; sudo sh -c 'cp -rf vitrage/vitrage_tempest_tests/tests/resources/static_physical/* /etc/vitrage/static_datasources/') (cd $DEVSTACK_PATH/; sudo sh -c 'cp -rf vitrage/vitrage_tempest_tests/tests/resources/templates/api/* /etc/vitrage/templates/') diff --git a/devstack/settings b/devstack/settings index e5630bb0e..782ceaeb6 100644 --- a/devstack/settings +++ b/devstack/settings @@ -24,7 +24,7 @@ VITRAGE_USE_MOD_WSGI=${VITRAGE_USE_MOD_WSGI:-${ENABLE_HTTPD_MOD_WSGI_SERVICES}} # Toggle for deploying Vitrage with/without nagios VITRAGE_USE_NAGIOS=$(trueorfalse False VITRAGE_USE_NAGIOS) -VITRAGE_DEFAULT_DATASOURCES=${VITRAGE_DEFAULT_DATASOURCES:-nova.host,nova.instance,nova.zone,nagios,static_physical,aodh,cinder.volume,neutron.network,neutron.port} +VITRAGE_DEFAULT_DATASOURCES=${VITRAGE_DEFAULT_DATASOURCES:-nova.host,nova.instance,nova.zone,nagios,static_physical,aodh,cinder.volume} # Tell Tempest this project is present diff --git a/vitrage/datasources/neutron/__init__.py b/vitrage/datasources/neutron/__init__.py index e69de29bb..9ae73d02b 100644 --- a/vitrage/datasources/neutron/__init__.py +++ b/vitrage/datasources/neutron/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2016 - Nokia Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__author__ = 'stack' diff --git a/vitrage/datasources/neutron/network/driver.py b/vitrage/datasources/neutron/network/driver.py index 53eaade46..96d03b498 100644 --- a/vitrage/datasources/neutron/network/driver.py +++ b/vitrage/datasources/neutron/network/driver.py @@ -27,7 +27,9 @@ class NetworkDriver(NeutronBase): @staticmethod def get_event_types(conf): - return ['network.'] + return ['network.create.end', + 'network.update.end', + 'network.delete.end'] @staticmethod def enrich_event(event, event_type): diff --git a/vitrage/datasources/neutron/network/transformer.py b/vitrage/datasources/neutron/network/transformer.py index 718b4014b..687dd30b6 100644 --- a/vitrage/datasources/neutron/network/transformer.py +++ b/vitrage/datasources/neutron/network/transformer.py @@ -12,6 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo_log import log as logging + from vitrage.common.constants import DatasourceProperties as DSProps from vitrage.common.constants import EntityCategory from vitrage.common.constants import EventAction @@ -24,8 +26,18 @@ from vitrage.datasources.transformer_base import extract_field_value import vitrage.graph.utils as graph_utils +LOG = logging.getLogger(__name__) + + class NetworkTransformer(ResourceTransformerBase): + UPDATE_ID_PROPERTY = { + 'network.create.end': ('network', 'id'), + 'network.update.end': ('network', 'id'), + 'network.delete.end': ('network_id',), + None: ('id',) + } + # Event types which need to refer them differently UPDATE_EVENT_TYPES = { 'network.delete.end': EventAction.DELETE_ENTITY, @@ -34,22 +46,41 @@ class NetworkTransformer(ResourceTransformerBase): def __init__(self, transformers): super(NetworkTransformer, self).__init__(transformers) - def _create_entity_key(self, entity_event): - network_id = 'network_id' if tbase.is_update_event(entity_event) \ - else 'id' - key_fields = self._key_values(NEUTRON_NETWORK_DATASOURCE, - extract_field_value(entity_event, - network_id)) - return tbase.build_key(key_fields) - def _create_snapshot_entity_vertex(self, entity_event): - name = extract_field_value(entity_event, 'name') - entity_id = extract_field_value(entity_event, 'id') - state = extract_field_value(entity_event, 'status') - return self._create_vertex(entity_event, name, entity_id, state) + name = entity_event['name'] + entity_id = entity_event['id'] + state = entity_event['status'] + update_timestamp = entity_event['updated_at'] - def _create_vertex(self, entity_event, name, entity_id, state): + return self._create_vertex(entity_event, + name, + entity_id, + state, + update_timestamp) + + def _create_update_entity_vertex(self, entity_event): + + event_type = entity_event[DSProps.EVENT_TYPE] + name = extract_field_value(entity_event, 'network', 'name') + state = extract_field_value(entity_event, 'network', 'status') + update_timestamp = \ + extract_field_value(entity_event, 'network', 'updated_at') + entity_id = extract_field_value(entity_event, + *self.UPDATE_ID_PROPERTY[event_type]) + + return self._create_vertex(entity_event, + name, + entity_id, + state, + update_timestamp) + + def _create_vertex(self, + entity_event, + name, + entity_id, + state, + update_timestamp): metadata = { VProps.NAME: name, @@ -57,12 +88,6 @@ class NetworkTransformer(ResourceTransformerBase): sample_timestamp = entity_event[DSProps.SAMPLE_DATE] - # TODO(Alexey): need to check here that only the UPDATE sync_mode will - # update the UPDATE_TIMESTAMP property - update_timestamp = self._format_update_timestamp( - extract_field_value(entity_event, DSProps.SAMPLE_DATE), - sample_timestamp) - return graph_utils.create_vertex( self._create_entity_key(entity_event), entity_id=entity_id, @@ -73,5 +98,10 @@ class NetworkTransformer(ResourceTransformerBase): update_timestamp=update_timestamp, metadata=metadata) - def _create_update_entity_vertex(self, entity_event): - pass + def _create_entity_key(self, entity_event): + event_type = entity_event.get(DSProps.EVENT_TYPE, None) + network_id = extract_field_value(entity_event, + *self.UPDATE_ID_PROPERTY[event_type]) + + key_fields = self._key_values(NEUTRON_NETWORK_DATASOURCE, network_id) + return tbase.build_key(key_fields) diff --git a/vitrage/datasources/neutron/port/driver.py b/vitrage/datasources/neutron/port/driver.py index 913c5124b..b4c998b6a 100644 --- a/vitrage/datasources/neutron/port/driver.py +++ b/vitrage/datasources/neutron/port/driver.py @@ -27,7 +27,9 @@ class PortDriver(NeutronBase): @staticmethod def get_event_types(conf): - return ['port.'] + return ['port.create.end', + 'port.update.end', + 'port.delete.end'] @staticmethod def enrich_event(event, event_type): diff --git a/vitrage/datasources/neutron/port/transformer.py b/vitrage/datasources/neutron/port/transformer.py index 2c511a78a..672bdf4c9 100644 --- a/vitrage/datasources/neutron/port/transformer.py +++ b/vitrage/datasources/neutron/port/transformer.py @@ -36,6 +36,19 @@ LOG = logging.getLogger(__name__) class PortTransformer(ResourceTransformerBase): + UPDATE_ID_PROPERTY = { + 'port.create.end': ('port', 'id'), + 'port.update.end': ('port', 'id'), + 'port.delete.end': ('port_id',), + None: ('id',) + } + + FIXED_IPS_PROPERTY = { + 'port.create.end': ('port', 'fixed_ips'), + 'port.update.end': ('port', 'fixed_ips'), + None: ('fixed_ips',) + } + # Event types which need to refer them differently UPDATE_EVENT_TYPES = { 'port.delete.end': EventAction.DELETE_ENTITY, @@ -44,21 +57,47 @@ class PortTransformer(ResourceTransformerBase): def __init__(self, transformers): super(PortTransformer, self).__init__(transformers) - def _create_entity_key(self, entity_event): - key_fields = self._key_values(NEUTRON_PORT_DATASOURCE, - extract_field_value(entity_event, 'id')) - return tbase.build_key(key_fields) - def _create_snapshot_entity_vertex(self, entity_event): - name = extract_field_value(entity_event, 'name') - entity_id = extract_field_value(entity_event, 'id') - state = extract_field_value(entity_event, 'status') - return self._create_vertex(entity_event, name if name else None, - entity_id, state) + name = entity_event['name'] if entity_event['name'] else None + entity_id = entity_event['id'] + state = entity_event['status'] + update_timestamp = entity_event['updated_at'] - def _create_vertex(self, entity_event, name, entity_id, state): - ip_addresses = [ip['ip_address'] for ip in entity_event['fixed_ips']] + return self._create_vertex(entity_event, + name, + entity_id, + state, + update_timestamp) + + def _create_update_entity_vertex(self, entity_event): + + event_type = entity_event[DSProps.EVENT_TYPE] + name = extract_field_value(entity_event, 'port', 'name') + state = extract_field_value(entity_event, 'port', 'status') + update_timestamp = \ + extract_field_value(entity_event, 'port', 'updated_at') + entity_id = extract_field_value(entity_event, + *self.UPDATE_ID_PROPERTY[event_type]) + + return self._create_vertex(entity_event, + name, + entity_id, + state, + update_timestamp) + + def _create_vertex(self, + entity_event, + name, + entity_id, + state, + update_timestamp): + event_type = entity_event.get(DSProps.EVENT_TYPE, None) + ip_addresses = [] + if not event_type: + fixed_ips = extract_field_value( + entity_event, *self.FIXED_IPS_PROPERTY[event_type]) + ip_addresses = [ip['ip_address'] for ip in fixed_ips] metadata = { VProps.NAME: name, 'ip_addresses': tuple(ip_addresses) @@ -66,12 +105,6 @@ class PortTransformer(ResourceTransformerBase): sample_timestamp = entity_event[DSProps.SAMPLE_DATE] - # TODO(Alexey): need to check here that only the UPDATE sync_mode will - # update the UPDATE_TIMESTAMP property - update_timestamp = self._format_update_timestamp( - extract_field_value(entity_event, DSProps.SAMPLE_DATE), - sample_timestamp) - return graph_utils.create_vertex( self._create_entity_key(entity_event), entity_id=entity_id, @@ -82,44 +115,42 @@ class PortTransformer(ResourceTransformerBase): update_timestamp=update_timestamp, metadata=metadata) - def _create_update_entity_vertex(self, entity_event): - pass - def _create_snapshot_neighbors(self, entity_event): return self._create_port_neighbors(entity_event, - 'device_owner', - 'device_id', - 'network_id') + ('device_owner',), + ('device_id',), + ('network_id',)) def _create_update_neighbors(self, entity_event): return self._create_port_neighbors(entity_event, - 'device_owner', - 'server_uuid', - 'network_id') + ('port', 'device_owner'), + ('port', 'device_id'), + ('port', 'network_id')) def _create_port_neighbors(self, entity_event, device_owner_property, device_id_property, - net_id_property): + network_id_property): + neighbors = [self._create_network_neighbor(entity_event, + network_id_property)] - instance = None - net = self._create_net_neighbor(entity_event, - net_id_property) - - if entity_event[device_owner_property] == 'compute:nova': + device_owner = \ + extract_field_value(entity_event, *device_owner_property) + if device_owner == 'compute:nova' or device_owner == 'compute:None': instance = self._create_instance_neighbor( entity_event, device_id_property) + neighbors.append(instance) - return [net, instance] if instance else [net] + return neighbors def _create_instance_neighbor(self, entity_event, instance_id_property): port_vitrage_id = self._create_entity_key(entity_event) - instance_id = entity_event[instance_id_property] + instance_id = extract_field_value(entity_event, *instance_id_property) sample_timestamp = entity_event[DSProps.SAMPLE_DATE] @@ -137,10 +168,10 @@ class PortTransformer(ResourceTransformerBase): return Neighbor(instance_vertex, relationship_edge) - def _create_net_neighbor(self, entity_event, net_id_property): + def _create_network_neighbor(self, entity_event, net_id_property): port_vitrage_id = self._create_entity_key(entity_event) - net_id = entity_event[net_id_property] + net_id = extract_field_value(entity_event, *net_id_property) sample_timestamp = entity_event[DSProps.SAMPLE_DATE] @@ -158,3 +189,12 @@ class PortTransformer(ResourceTransformerBase): relationship_type=EdgeLabel.CONTAINS) return Neighbor(net_vertex, relationship_edge) + + def _create_entity_key(self, entity_event): + event_type = entity_event.get(DSProps.EVENT_TYPE, None) + port_id = extract_field_value(entity_event, + *self.UPDATE_ID_PROPERTY[event_type]) + + key_fields = self._key_values(NEUTRON_PORT_DATASOURCE, port_id) + + return tbase.build_key(key_fields) diff --git a/vitrage/datasources/nova/instance/driver.py b/vitrage/datasources/nova/instance/driver.py index 3cb2e9562..d3ba918d2 100644 --- a/vitrage/datasources/nova/instance/driver.py +++ b/vitrage/datasources/nova/instance/driver.py @@ -47,8 +47,7 @@ class InstanceDriver(NovaDriverBase): @staticmethod def get_event_types(conf): # Add event_types to receive notifications about - return ['compute.instance.create.start', - 'compute.instance.create.error', + return ['compute.instance.create.error', 'compute.instance.create.end', 'compute.instance.delete.start', 'compute.instance.delete.end', @@ -67,8 +66,7 @@ class InstanceDriver(NovaDriverBase): 'compute.instance.volume.attach', 'compute.instance.volume.detach', 'compute.instance.pause.end', - 'compute.instance.unpause.end' - ] + 'compute.instance.unpause.end'] @staticmethod def get_topic(conf): diff --git a/vitrage/datasources/transformer_base.py b/vitrage/datasources/transformer_base.py index 6fdee7659..1c49494b3 100644 --- a/vitrage/datasources/transformer_base.py +++ b/vitrage/datasources/transformer_base.py @@ -43,12 +43,14 @@ AVAILABLE = 'available' def extract_field_value(entity_event, *args): + try: + value = entity_event + for key in args: + value = value[key] - value = entity_event - for key in args: - value = value[key] - - return value + return value + except Exception: + return None def build_key(key_values): diff --git a/vitrage/tests/unit/datasources/neutron/__init__.py b/vitrage/tests/unit/datasources/neutron/__init__.py new file mode 100644 index 000000000..dd32b852f --- /dev/null +++ b/vitrage/tests/unit/datasources/neutron/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__author__ = 'stack' diff --git a/vitrage/tests/unit/datasources/neutron/network/__init__.py b/vitrage/tests/unit/datasources/neutron/network/__init__.py new file mode 100644 index 000000000..dd32b852f --- /dev/null +++ b/vitrage/tests/unit/datasources/neutron/network/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__author__ = 'stack' diff --git a/vitrage/tests/unit/datasources/neutron/network/test_neutron_network_transformer.py b/vitrage/tests/unit/datasources/neutron/network/test_neutron_network_transformer.py new file mode 100644 index 000000000..dd32b852f --- /dev/null +++ b/vitrage/tests/unit/datasources/neutron/network/test_neutron_network_transformer.py @@ -0,0 +1,15 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__author__ = 'stack' diff --git a/vitrage/tests/unit/datasources/neutron/port/__init__.py b/vitrage/tests/unit/datasources/neutron/port/__init__.py new file mode 100644 index 000000000..dd32b852f --- /dev/null +++ b/vitrage/tests/unit/datasources/neutron/port/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__author__ = 'stack' diff --git a/vitrage/tests/unit/datasources/neutron/port/test_neutron_port_transformer.py b/vitrage/tests/unit/datasources/neutron/port/test_neutron_port_transformer.py new file mode 100644 index 000000000..dd32b852f --- /dev/null +++ b/vitrage/tests/unit/datasources/neutron/port/test_neutron_port_transformer.py @@ -0,0 +1,15 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__author__ = 'stack' diff --git a/vitrage_tempest_tests/tests/api/base.py b/vitrage_tempest_tests/tests/api/base.py index 349255dd5..c2f47c462 100644 --- a/vitrage_tempest_tests/tests/api/base.py +++ b/vitrage_tempest_tests/tests/api/base.py @@ -11,6 +11,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + import time from oslo_log import log as logging @@ -21,6 +22,8 @@ from vitrage.common.constants import EntityCategory from vitrage.common.constants import VertexProperties as VProps from vitrage.datasources import AODH_DATASOURCE from vitrage.datasources import CINDER_VOLUME_DATASOURCE +from vitrage.datasources.neutron.network import NEUTRON_NETWORK_DATASOURCE +from vitrage.datasources.neutron.port import NEUTRON_PORT_DATASOURCE from vitrage.datasources import NOVA_HOST_DATASOURCE from vitrage.datasources import NOVA_INSTANCE_DATASOURCE from vitrage.datasources import NOVA_ZONE_DATASOURCE @@ -243,6 +246,20 @@ class BaseApiTest(base.BaseTestCase): self.NUM_EDGES_PER_TYPE: kwargs.get('aodh_edges', 0)} validation_data.append(props) + # neutron.network + props = {VProps.CATEGORY: EntityCategory.RESOURCE, + VProps.TYPE: NEUTRON_NETWORK_DATASOURCE, + self.NUM_VERTICES_PER_TYPE: kwargs.get('network_entities', 0), + self.NUM_EDGES_PER_TYPE: kwargs.get('network_edges', 0)} + validation_data.append(props) + + # neutron.port + props = {VProps.CATEGORY: EntityCategory.RESOURCE, + VProps.TYPE: NEUTRON_PORT_DATASOURCE, + self.NUM_VERTICES_PER_TYPE: kwargs.get('port_entities', 0), + self.NUM_EDGES_PER_TYPE: kwargs.get('port_edges', 0)} + validation_data.append(props) + return validation_data def _validate_graph_correctness(self,