From 0ca78719f64a6b25cf27cb6ae61edabaa3fd86cd Mon Sep 17 00:00:00 2001 From: Alexey Weyl Date: Thu, 7 Apr 2016 16:20:47 +0300 Subject: [PATCH] topology tempest Change-Id: I17002ef155422b1c2f64db540a6875a0079b6215 --- vitrage/api/controllers/rest.py | 5 +- vitrage/graph/driver/networkx_graph.py | 1 + .../entity_graph/processor/test_processor.py | 9 +- .../tests/api/topology/test_topology.py | 333 ++++++++++++++++-- .../tests/api/topology/utils.py | 16 +- vitrage_tempest_tests/tests/utils.py | 11 +- 6 files changed, 324 insertions(+), 51 deletions(-) diff --git a/vitrage/api/controllers/rest.py b/vitrage/api/controllers/rest.py index c1eca381f..2dfdfde3f 100644 --- a/vitrage/api/controllers/rest.py +++ b/vitrage/api/controllers/rest.py @@ -19,6 +19,7 @@ from oslo_log import log import pecan as pecan from pecan import abort from pecan import rest +from vitrage.common.constants import EntityCategory from vitrage.datasources import OPENSTACK_NODE @@ -28,7 +29,9 @@ LOG = log.getLogger(__name__) class RootRestController(rest.RestController): @staticmethod - def as_tree(graph, root=OPENSTACK_NODE, reverse=False): + def as_tree(graph, + root='%s:%s' % (EntityCategory.RESOURCE, OPENSTACK_NODE), + reverse=False): linked_graph = json_graph.node_link_graph(graph) if reverse: linked_graph = linked_graph.reverse() diff --git a/vitrage/graph/driver/networkx_graph.py b/vitrage/graph/driver/networkx_graph.py index af8a7e63e..3d9ea7691 100644 --- a/vitrage/graph/driver/networkx_graph.py +++ b/vitrage/graph/driver/networkx_graph.py @@ -11,6 +11,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + import copy import json import networkx as nx diff --git a/vitrage/tests/unit/entity_graph/processor/test_processor.py b/vitrage/tests/unit/entity_graph/processor/test_processor.py index fa9a52a0e..921ee5065 100644 --- a/vitrage/tests/unit/entity_graph/processor/test_processor.py +++ b/vitrage/tests/unit/entity_graph/processor/test_processor.py @@ -12,8 +12,6 @@ # License for the specific language governing permissions and limitations # under the License. -import unittest - from oslo_config import cfg from vitrage.common.constants import DatasourceProperties as DSProps @@ -49,8 +47,6 @@ class TestProcessor(TestEntityGraphUnitBase): cls.conf.register_opts(cls.DATASOURCES_OPTS, group='datasources') cls.load_datasources(cls.conf) - # TODO(Alexey): un skip this test when instance transformer update is ready - @unittest.skip('Not ready yet') def test_process_event(self): # check create instance event processor = proc.Processor(self.conf, InitializationStatus()) @@ -61,11 +57,12 @@ class TestProcessor(TestEntityGraphUnitBase): self.NUM_EDGES_AFTER_CREATION) # check update instance even - # TODO(Alexey): Create an event in update event structure - # (update snapshot fields won't work) event[DSProps.SYNC_MODE] = SyncMode.UPDATE event[DSProps.EVENT_TYPE] = 'compute.instance.volume.attach' event['hostname'] = 'new_host' + event['instance_id'] = event['id'] + event['state'] = event['status'] + event['host'] = event['OS-EXT-SRV-ATTR:host'] processor.process_event(event) self._check_graph(processor, self.NUM_VERTICES_AFTER_CREATION, self.NUM_EDGES_AFTER_CREATION) diff --git a/vitrage_tempest_tests/tests/api/topology/test_topology.py b/vitrage_tempest_tests/tests/api/topology/test_topology.py index 17f3a325c..0fb1f665b 100644 --- a/vitrage_tempest_tests/tests/api/topology/test_topology.py +++ b/vitrage_tempest_tests/tests/api/topology/test_topology.py @@ -12,11 +12,26 @@ # License for the specific language governing permissions and limitations # under the License. +import json +import time + from oslo_log import log as logging +from vitrage import clients +from vitrage.common.constants import EntityCategory +from vitrage.common.constants import VertexProperties as VProps +from vitrage.datasources import CINDER_VOLUME_DATASOURCE +from vitrage.datasources import NOVA_HOST_DATASOURCE +from vitrage.datasources import NOVA_INSTANCE_DATASOURCE +from vitrage.datasources import NOVA_ZONE_DATASOURCE +from vitrage.datasources import OPENSTACK_NODE +from vitrage.graph import Edge +from vitrage.graph import NXGraph +from vitrage.graph import Vertex +from vitrage import keystone_client from vitrage_tempest_tests.tests.api.base import BaseVitrageTest -from vitrage_tempest_tests.tests.api.topology.utils \ - import TopologyHelper +import vitrage_tempest_tests.tests.utils as utils +from vitrageclient import client as v_client LOG = logging.getLogger(__name__) @@ -24,39 +39,301 @@ LOG = logging.getLogger(__name__) class BaseTopologyTest(BaseVitrageTest): """Topology test class for Vitrage API tests.""" + NUM_ENTITIES_PER_TYPE = 'num_vertices' + NUM_EDGES_PER_TYPE = 'num_edges_per_type' + @classmethod def setUpClass(cls): super(BaseTopologyTest, cls).setUpClass() - cls.topology_client = TopologyHelper() + cls.conf = utils.get_conf() + cls.vitrage_client = \ + v_client.Client('1', session=keystone_client.get_session(cls.conf)) + cls.nova_client = clients.nova_client(cls.conf) + cls.cinder_client = clients.cinder_client(cls.conf) - def test_compare_graphs(self): + def test_compare_api_and_cli(self): """Wrapper that returns a test graph.""" - api_graph = self.topology_client.get_api_topology('graph') - cli_graph = self.topology_client.show_cli_topology() - if self.topology_client.compare_graphs(api_graph, cli_graph) is False: - LOG.error('The graph tempest_graph is not correct') - else: - LOG.info('The graph tempest_graph is correct') + api_graph = self.vitrage_client.topology.get() + cli_graph = utils.run_vitrage_command('vitrage topology show') + self.assertEqual(True, self._compare_graphs(api_graph, cli_graph)) - def test_get_tree_with_vms(self): - """Wrapper that returns a test tree with created vm's""" - resources = self.topology_client.create_machines(4) - cli_graph = self.topology_client.show_cli_topology() + def test_default_graph(self): + try: + # create entities + self._create_entities(num_instances=3, num_volumes=1) + api_graph = self.vitrage_client.topology.get() + graph = self._create_graph_from_graph_dictionary(api_graph) + entities = self._entities_validation_data( + host_entities=1, host_edges=4, + instance_entities=3, instance_edges=4, + volume_entities=1, volume_edges=1) + self._validate_graph_correctness(graph, 7, 6, entities) + finally: + self._rollback_to_default() - if self.topology_client.validate_graph_correctness( - cli_graph, resources) is False: - LOG.error('The graph ' + self.name + ' is not correct') - else: - LOG.info('The graph ' + self.name + ' is correct') + def test_graph_with_query(self): + try: + # create entities + self._create_entities(num_instances=3, num_volumes=1) + api_graph = self.vitrage_client.topology.get( + query=self._graph_query()) + graph = self._create_graph_from_graph_dictionary(api_graph) + entities = self._entities_validation_data( + host_entities=1, host_edges=4, + instance_entities=3, instance_edges=3) + self._validate_graph_correctness(graph, 6, 5, entities) + finally: + self._rollback_to_default() - def test_get_graph_with_volume(self): - """Wrapper that returns a test graph.""" - resources = self.topology_client.create_volume() - cli_graph = self.topology_client.show_cli_topology() + def test_default_tree(self): + try: + # create entities + self._create_entities(num_instances=3, num_volumes=1) + api_graph = self.vitrage_client.topology.get(graph_type='tree') + graph = self._create_graph_from_tree_dictionary(api_graph) + entities = self._entities_validation_data( + host_entities=1, host_edges=4, + instance_entities=3, instance_edges=3) + self._validate_graph_correctness(graph, 6, 5, entities) + finally: + self._rollback_to_default() - if self.topology_client.validate_graph_correctness( - cli_graph, resources) is False: - LOG.error('The graph ' + self.name + ' is not correct') - else: - LOG.info('The graph ' + self.name + ' is correct') + def test_tree_with_query(self): + try: + # create entities + self._create_entities(num_instances=3, end_sleep=10) + api_graph = self.vitrage_client.topology.get( + graph_type='tree', query=self._tree_query()) + graph = self._create_graph_from_tree_dictionary(api_graph) + entities = self._entities_validation_data( + host_entities=1, host_edges=1) + self._validate_graph_correctness(graph, 3, 2, entities) + finally: + self._rollback_to_default() + + def _rollback_to_default(self): + self._delete_entities(instance=True, volume=True) + api_graph = self.vitrage_client.topology.get() + graph = self._create_graph_from_graph_dictionary(api_graph) + entities = self._entities_validation_data() + self._validate_graph_correctness(graph, 3, 2, entities) + + def _create_entities(self, num_instances=0, num_volumes=0, end_sleep=3): + if num_instances > 0: + resources = self._create_instances(num_instances) + self._wait_for_status(20, + self._check_num_instances, + num_instances=num_instances) + time.sleep(1) + + if num_volumes > 0: + self._create_volume_and_attach('volume-1', 1, + resources[0].__dict__['id'], + '/tmp/vda') + self._wait_for_status(20, + self._check_num_volumes, + num_volumes=1) + + # waiting until all the entities creation were processed by the + # entity graph processor + time.sleep(end_sleep) + + def _delete_entities(self, instance=False, volume=False): + if volume: + self._delete_volumes() + self._wait_for_status(30, + self._check_num_volumes, + num_volumes=0) + + if instance: + self._delete_instances() + self._wait_for_status(20, + self._check_num_instances, + num_instances=0) + + # waiting until all the entities deletion were processed by the + # entity graph processor + time.sleep(5) + + def _entities_validation_data(self, node_entities=1, node_edges=1, + zone_entities=1, zone_edges=2, + host_entities=1, host_edges=1, + instance_entities=0, instance_edges=0, + volume_entities=0, volume_edges=0): + return [ + {VProps.TYPE: OPENSTACK_NODE, + self.NUM_ENTITIES_PER_TYPE: node_entities, + self.NUM_EDGES_PER_TYPE: node_edges}, + {VProps.TYPE: NOVA_ZONE_DATASOURCE, + self.NUM_ENTITIES_PER_TYPE: zone_entities, + self.NUM_EDGES_PER_TYPE: zone_edges}, + {VProps.TYPE: NOVA_HOST_DATASOURCE, + self.NUM_ENTITIES_PER_TYPE: host_entities, + self.NUM_EDGES_PER_TYPE: host_edges}, + {VProps.TYPE: NOVA_INSTANCE_DATASOURCE, + self.NUM_ENTITIES_PER_TYPE: instance_entities, + self.NUM_EDGES_PER_TYPE: instance_edges}, + {VProps.TYPE: CINDER_VOLUME_DATASOURCE, + self.NUM_ENTITIES_PER_TYPE: volume_entities, + self.NUM_EDGES_PER_TYPE: volume_edges} + ] + + def _validate_graph_correctness(self, + graph, + num_entities, + num_edges, + entities): + self.assertIsNot(None, graph) + self.assertIsNot(None, entities) + self.assertEqual(num_entities, graph.num_vertices()) + self.assertEqual(num_edges, graph.num_edges()) + + for entity in entities: + query = { + VProps.CATEGORY: EntityCategory.RESOURCE, + VProps.TYPE: entity[VProps.TYPE], + VProps.IS_DELETED: False, + VProps.IS_PLACEHOLDER: False + } + vertices = graph.get_vertices(vertex_attr_filter=query) + self.assertEqual(entity[self.NUM_ENTITIES_PER_TYPE], len(vertices)) + + num_edges = sum([len(graph.get_edges(vertex.vertex_id)) + for vertex in vertices]) + self.assertEqual(entity[self.NUM_EDGES_PER_TYPE], num_edges) + + def _check_num_instances(self, num_instances=0): + return len(self.nova_client.servers.list()) == num_instances + + def _check_num_volumes(self, num_volumes=0): + return len(self.cinder_client.volumes.list()) == num_volumes + + def _create_volume_and_attach(self, name, size, instance_id, mount_point): + volume = self.cinder_client.volumes.create(display_name=name, + size=size) + time.sleep(3) + self.cinder_client.volumes.attach(volume=volume, + instance_uuid=instance_id, + mountpoint=mount_point) + return volume + + def _create_instances(self, num_machines): + flavors_list = self.nova_client.flavors.list() + images_list = self.nova_client.images.list() + + resources = [self.nova_client.servers.create( + name='%s-%s' % ('vm', index), + flavor=flavors_list[0], + image=images_list[0]) for index in range(num_machines)] + + return resources + + def _delete_instances(self): + instances = self.nova_client.servers.list() + for instance in instances: + try: + self.nova_client.servers.delete(instance) + except Exception: + pass + + def _delete_volumes(self): + volumes = self.cinder_client.volumes.list() + for volume in volumes: + try: + self.cinder_client.volumes.detach(volume) + self.cinder_client.volumes.force_delete(volume) + except Exception: + self.cinder_client.volumes.force_delete(volume) + + @staticmethod + def _wait_for_status(max_waiting, func, **kwargs): + count = 0 + while count < max_waiting: + if func(**kwargs): + return True + count += 1 + time.sleep(2) + LOG.info("wait_for_status - False ") + return False + + @staticmethod + def _compare_graphs(api_graph, cli_graph): + """Compare Graph object to graph form terminal """ + if not api_graph: + LOG.error("The topology graph taken from rest api is empty") + return False + if not cli_graph: + LOG.error("The topology graph taken from terminal is empty") + return False + + parsed_topology = json.loads(cli_graph) + + sorted_cli_graph = sorted(parsed_topology.items()) + sorted_api_graph = sorted(api_graph.items()) + + for item in sorted_cli_graph[4][1]: + item.pop(VProps.UPDATE_TIMESTAMP, None) + + for item in sorted_api_graph[4][1]: + item.pop(VProps.UPDATE_TIMESTAMP, None) + + return sorted_cli_graph == sorted_api_graph + + @staticmethod + def _create_graph_from_graph_dictionary(api_graph): + graph = NXGraph() + + nodes = api_graph['nodes'] + for i in xrange(len(nodes)): + graph.add_vertex(Vertex(str(i), nodes[i])) + + edges = api_graph['links'] + for i in xrange(len(edges)): + graph.add_edge(Edge(str(edges[i]['source']), + str(edges[i]['target']), + edges[i]['relationship_type'])) + + return graph + + def _create_graph_from_tree_dictionary(self, + api_graph, + graph=None, + ancestor=None): + children = [] + graph = NXGraph() if not graph else graph + + if 'children' in api_graph: + children = api_graph.copy()['children'] + del api_graph['children'] + + vertex = Vertex(api_graph[VProps.VITRAGE_ID], api_graph) + graph.add_vertex(vertex) + if ancestor: + graph.add_edge(Edge(ancestor[VProps.VITRAGE_ID], + vertex[VProps.VITRAGE_ID], + 'label')) + + for entity in children: + self._create_graph_from_tree_dictionary(entity, graph, vertex) + + return graph + + @staticmethod + def _graph_query(): + return '{"and": [{"==": {"category": "RESOURCE"}},' \ + '{"==": {"is_deleted": false}},' \ + '{"==": {"is_placeholder": false}},' \ + '{"or": [{"==": {"type": "openstack.node"}},' \ + '{"==": {"type": "nova.instance"}},' \ + '{"==": {"type": "nova.host"}},' \ + '{"==": {"type": "nova.zone"}}]}]}' + + @staticmethod + def _tree_query(): + return '{"and": [{"==": {"category": "RESOURCE"}},' \ + '{"==": {"is_deleted": false}},' \ + '{"==": {"is_placeholder": false}},' \ + '{"or": [{"==": {"type": "openstack.node"}},' \ + '{"==": {"type": "nova.host"}},' \ + '{"==": {"type": "nova.zone"}}]}]}' diff --git a/vitrage_tempest_tests/tests/api/topology/utils.py b/vitrage_tempest_tests/tests/api/topology/utils.py index 84ef63036..74de4c11d 100644 --- a/vitrage_tempest_tests/tests/api/topology/utils.py +++ b/vitrage_tempest_tests/tests/api/topology/utils.py @@ -27,8 +27,8 @@ LOG = logging.getLogger(__name__) class TopologyHelper(BaseVitrageTest): """Topology test class for Vitrage API tests.""" - def setUp(self): - super(TopologyHelper, self).setUp() + def __init__(self): + super(TopologyHelper, self).__init__() self.client = utils.get_client() self.depth = '' self.query = '' @@ -55,18 +55,6 @@ class TopologyHelper(BaseVitrageTest): return utils.run_vitrage_command_with_user( "vitrage topology show", self.conf.service_credentials.user) - def create_machines(self, machine_number): - flavor_id = self.get_flavor_id_from_list() - image_id = self.get_image_id_from_list() - - resources = [] - for r in range(start=0, stop=machine_number, step=1): - self.create_vm_with_exist_image("vm_for_test_" + str(r), - flavor_id, image_id) - resources.append("vm_for_test_" + str(r)) - - return resources - def create_volume(self): flavor_id = self.get_flavor_id_from_list() image_id = self.get_image_id_from_list() diff --git a/vitrage_tempest_tests/tests/utils.py b/vitrage_tempest_tests/tests/utils.py index f3dd9012c..5faaf115b 100644 --- a/vitrage_tempest_tests/tests/utils.py +++ b/vitrage_tempest_tests/tests/utils.py @@ -55,7 +55,14 @@ def get_from_terminal(command): def run_vitrage_command(command): - p = subprocess.Popen(command, + auth_url = '--os-auth-url http://10.41.251.201:5000/v2.0' + user = '--os-user-name admin' + password = '--os-password password' + project_name = '--os-project-name admin' + full_command = '%s %s %s %s %s' % \ + (command, user, password, project_name, auth_url) + + p = subprocess.Popen(full_command, shell=True, executable="/bin/bash", stdout=subprocess.PIPE, @@ -71,7 +78,7 @@ def run_vitrage_command(command): def run_vitrage_command_with_user(command, user): run_vitrage_command( - "cd /home/stack/devstack; . openrc " + + "cd /openstack/devstack; . openrc " + user + " " + user + "; " + command)