topology tempest

Change-Id: I17002ef155422b1c2f64db540a6875a0079b6215

vitrage/api/controllers/rest.py

@@ -19,6 +19,7 @@ from oslo_log import log
 import pecan as pecan
 from pecan import abort
 from pecan import rest
+from vitrage.common.constants import EntityCategory
 
 from vitrage.datasources import OPENSTACK_NODE
 
@@ -28,7 +29,9 @@ LOG = log.getLogger(__name__)
 class RootRestController(rest.RestController):
 
     @staticmethod
-    def as_tree(graph, root=OPENSTACK_NODE, reverse=False):
+    def as_tree(graph,
+                root='%s:%s' % (EntityCategory.RESOURCE, OPENSTACK_NODE),
+                reverse=False):
         linked_graph = json_graph.node_link_graph(graph)
         if reverse:
             linked_graph = linked_graph.reverse()

vitrage/graph/driver/networkx_graph.py

@@ -11,6 +11,7 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+
 import copy
 import json
 import networkx as nx

vitrage/tests/unit/entity_graph/processor/test_processor.py

@@ -12,8 +12,6 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import unittest
-
 from oslo_config import cfg
 
 from vitrage.common.constants import DatasourceProperties as DSProps
@@ -49,8 +47,6 @@ class TestProcessor(TestEntityGraphUnitBase):
         cls.conf.register_opts(cls.DATASOURCES_OPTS, group='datasources')
         cls.load_datasources(cls.conf)
 
-    # TODO(Alexey): un skip this test when instance transformer update is ready
-    @unittest.skip('Not ready yet')
     def test_process_event(self):
         # check create instance event
         processor = proc.Processor(self.conf, InitializationStatus())
@@ -61,11 +57,12 @@ class TestProcessor(TestEntityGraphUnitBase):
                           self.NUM_EDGES_AFTER_CREATION)
 
         # check update instance even
-        # TODO(Alexey): Create an event in update event structure
-        # (update snapshot fields won't work)
         event[DSProps.SYNC_MODE] = SyncMode.UPDATE
         event[DSProps.EVENT_TYPE] = 'compute.instance.volume.attach'
         event['hostname'] = 'new_host'
+        event['instance_id'] = event['id']
+        event['state'] = event['status']
+        event['host'] = event['OS-EXT-SRV-ATTR:host']
         processor.process_event(event)
         self._check_graph(processor, self.NUM_VERTICES_AFTER_CREATION,
                           self.NUM_EDGES_AFTER_CREATION)

vitrage_tempest_tests/tests/api/topology/test_topology.py

@@ -12,11 +12,26 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import json
+import time
+
 from oslo_log import log as logging
 
+from vitrage import clients
+from vitrage.common.constants import EntityCategory
+from vitrage.common.constants import VertexProperties as VProps
+from vitrage.datasources import CINDER_VOLUME_DATASOURCE
+from vitrage.datasources import NOVA_HOST_DATASOURCE
+from vitrage.datasources import NOVA_INSTANCE_DATASOURCE
+from vitrage.datasources import NOVA_ZONE_DATASOURCE
+from vitrage.datasources import OPENSTACK_NODE
+from vitrage.graph import Edge
+from vitrage.graph import NXGraph
+from vitrage.graph import Vertex
+from vitrage import keystone_client
 from vitrage_tempest_tests.tests.api.base import BaseVitrageTest
-from vitrage_tempest_tests.tests.api.topology.utils \
+import vitrage_tempest_tests.tests.utils as utils
-    import TopologyHelper
+from vitrageclient import client as v_client
 
 LOG = logging.getLogger(__name__)
 
@@ -24,39 +39,301 @@ LOG = logging.getLogger(__name__)
 class BaseTopologyTest(BaseVitrageTest):
     """Topology test class for Vitrage API tests."""
 
+    NUM_ENTITIES_PER_TYPE = 'num_vertices'
+    NUM_EDGES_PER_TYPE = 'num_edges_per_type'
+
     @classmethod
     def setUpClass(cls):
         super(BaseTopologyTest, cls).setUpClass()
-        cls.topology_client = TopologyHelper()
+        cls.conf = utils.get_conf()
+        cls.vitrage_client = \
+            v_client.Client('1', session=keystone_client.get_session(cls.conf))
+        cls.nova_client = clients.nova_client(cls.conf)
+        cls.cinder_client = clients.cinder_client(cls.conf)
 
-    def test_compare_graphs(self):
+    def test_compare_api_and_cli(self):
         """Wrapper that returns a test graph."""
-        api_graph = self.topology_client.get_api_topology('graph')
-        cli_graph = self.topology_client.show_cli_topology()
 
-        if self.topology_client.compare_graphs(api_graph, cli_graph) is False:
+        api_graph = self.vitrage_client.topology.get()
-            LOG.error('The graph tempest_graph is not correct')
+        cli_graph = utils.run_vitrage_command('vitrage topology show')
-        else:
+        self.assertEqual(True, self._compare_graphs(api_graph, cli_graph))
-            LOG.info('The graph tempest_graph is correct')
 
-    def test_get_tree_with_vms(self):
+    def test_default_graph(self):
-        """Wrapper that returns a test tree with created vm's"""
+        try:
-        resources = self.topology_client.create_machines(4)
+            # create entities
-        cli_graph = self.topology_client.show_cli_topology()
+            self._create_entities(num_instances=3, num_volumes=1)
+            api_graph = self.vitrage_client.topology.get()
+            graph = self._create_graph_from_graph_dictionary(api_graph)
+            entities = self._entities_validation_data(
+                host_entities=1, host_edges=4,
+                instance_entities=3, instance_edges=4,
+                volume_entities=1, volume_edges=1)
+            self._validate_graph_correctness(graph, 7, 6, entities)
+        finally:
+            self._rollback_to_default()
 
-        if self.topology_client.validate_graph_correctness(
+    def test_graph_with_query(self):
-                cli_graph, resources) is False:
+        try:
-            LOG.error('The graph ' + self.name + ' is not correct')
+            # create entities
-        else:
+            self._create_entities(num_instances=3, num_volumes=1)
-            LOG.info('The graph ' + self.name + ' is correct')
+            api_graph = self.vitrage_client.topology.get(
+                query=self._graph_query())
+            graph = self._create_graph_from_graph_dictionary(api_graph)
+            entities = self._entities_validation_data(
+                host_entities=1, host_edges=4,
+                instance_entities=3, instance_edges=3)
+            self._validate_graph_correctness(graph, 6, 5, entities)
+        finally:
+            self._rollback_to_default()
 
-    def test_get_graph_with_volume(self):
+    def test_default_tree(self):
-        """Wrapper that returns a test graph."""
+        try:
-        resources = self.topology_client.create_volume()
+            # create entities
-        cli_graph = self.topology_client.show_cli_topology()
+            self._create_entities(num_instances=3, num_volumes=1)
+            api_graph = self.vitrage_client.topology.get(graph_type='tree')
+            graph = self._create_graph_from_tree_dictionary(api_graph)
+            entities = self._entities_validation_data(
+                host_entities=1, host_edges=4,
+                instance_entities=3, instance_edges=3)
+            self._validate_graph_correctness(graph, 6, 5, entities)
+        finally:
+            self._rollback_to_default()
 
-        if self.topology_client.validate_graph_correctness(
+    def test_tree_with_query(self):
-                cli_graph, resources) is False:
+        try:
-            LOG.error('The graph ' + self.name + ' is not correct')
+            # create entities
-        else:
+            self._create_entities(num_instances=3, end_sleep=10)
-            LOG.info('The graph ' + self.name + ' is correct')
+            api_graph = self.vitrage_client.topology.get(
+                graph_type='tree', query=self._tree_query())
+            graph = self._create_graph_from_tree_dictionary(api_graph)
+            entities = self._entities_validation_data(
+                host_entities=1, host_edges=1)
+            self._validate_graph_correctness(graph, 3, 2, entities)
+        finally:
+            self._rollback_to_default()
+
+    def _rollback_to_default(self):
+        self._delete_entities(instance=True, volume=True)
+        api_graph = self.vitrage_client.topology.get()
+        graph = self._create_graph_from_graph_dictionary(api_graph)
+        entities = self._entities_validation_data()
+        self._validate_graph_correctness(graph, 3, 2, entities)
+
+    def _create_entities(self, num_instances=0, num_volumes=0, end_sleep=3):
+        if num_instances > 0:
+            resources = self._create_instances(num_instances)
+            self._wait_for_status(20,
+                                  self._check_num_instances,
+                                  num_instances=num_instances)
+            time.sleep(1)
+
+        if num_volumes > 0:
+            self._create_volume_and_attach('volume-1', 1,
+                                           resources[0].__dict__['id'],
+                                           '/tmp/vda')
+            self._wait_for_status(20,
+                                  self._check_num_volumes,
+                                  num_volumes=1)
+
+        # waiting until all the entities creation were processed by the
+        # entity graph processor
+        time.sleep(end_sleep)
+
+    def _delete_entities(self, instance=False, volume=False):
+        if volume:
+            self._delete_volumes()
+            self._wait_for_status(30,
+                                  self._check_num_volumes,
+                                  num_volumes=0)
+
+        if instance:
+            self._delete_instances()
+            self._wait_for_status(20,
+                                  self._check_num_instances,
+                                  num_instances=0)
+
+        # waiting until all the entities deletion were processed by the
+        # entity graph processor
+        time.sleep(5)
+
+    def _entities_validation_data(self, node_entities=1, node_edges=1,
+                                  zone_entities=1, zone_edges=2,
+                                  host_entities=1, host_edges=1,
+                                  instance_entities=0, instance_edges=0,
+                                  volume_entities=0, volume_edges=0):
+        return [
+            {VProps.TYPE: OPENSTACK_NODE,
+             self.NUM_ENTITIES_PER_TYPE: node_entities,
+             self.NUM_EDGES_PER_TYPE: node_edges},
+            {VProps.TYPE: NOVA_ZONE_DATASOURCE,
+             self.NUM_ENTITIES_PER_TYPE: zone_entities,
+             self.NUM_EDGES_PER_TYPE: zone_edges},
+            {VProps.TYPE: NOVA_HOST_DATASOURCE,
+             self.NUM_ENTITIES_PER_TYPE: host_entities,
+             self.NUM_EDGES_PER_TYPE: host_edges},
+            {VProps.TYPE: NOVA_INSTANCE_DATASOURCE,
+             self.NUM_ENTITIES_PER_TYPE: instance_entities,
+             self.NUM_EDGES_PER_TYPE: instance_edges},
+            {VProps.TYPE: CINDER_VOLUME_DATASOURCE,
+             self.NUM_ENTITIES_PER_TYPE: volume_entities,
+             self.NUM_EDGES_PER_TYPE: volume_edges}
+        ]
+
+    def _validate_graph_correctness(self,
+                                    graph,
+                                    num_entities,
+                                    num_edges,
+                                    entities):
+        self.assertIsNot(None, graph)
+        self.assertIsNot(None, entities)
+        self.assertEqual(num_entities, graph.num_vertices())
+        self.assertEqual(num_edges, graph.num_edges())
+
+        for entity in entities:
+            query = {
+                VProps.CATEGORY: EntityCategory.RESOURCE,
+                VProps.TYPE: entity[VProps.TYPE],
+                VProps.IS_DELETED: False,
+                VProps.IS_PLACEHOLDER: False
+            }
+            vertices = graph.get_vertices(vertex_attr_filter=query)
+            self.assertEqual(entity[self.NUM_ENTITIES_PER_TYPE], len(vertices))
+
+            num_edges = sum([len(graph.get_edges(vertex.vertex_id))
+                             for vertex in vertices])
+            self.assertEqual(entity[self.NUM_EDGES_PER_TYPE], num_edges)
+
+    def _check_num_instances(self, num_instances=0):
+        return len(self.nova_client.servers.list()) == num_instances
+
+    def _check_num_volumes(self, num_volumes=0):
+        return len(self.cinder_client.volumes.list()) == num_volumes
+
+    def _create_volume_and_attach(self, name, size, instance_id, mount_point):
+        volume = self.cinder_client.volumes.create(display_name=name,
+                                                   size=size)
+        time.sleep(3)
+        self.cinder_client.volumes.attach(volume=volume,
+                                          instance_uuid=instance_id,
+                                          mountpoint=mount_point)
+        return volume
+
+    def _create_instances(self, num_machines):
+        flavors_list = self.nova_client.flavors.list()
+        images_list = self.nova_client.images.list()
+
+        resources = [self.nova_client.servers.create(
+            name='%s-%s' % ('vm', index),
+            flavor=flavors_list[0],
+            image=images_list[0]) for index in range(num_machines)]
+
+        return resources
+
+    def _delete_instances(self):
+        instances = self.nova_client.servers.list()
+        for instance in instances:
+            try:
+                self.nova_client.servers.delete(instance)
+            except Exception:
+                pass
+
+    def _delete_volumes(self):
+        volumes = self.cinder_client.volumes.list()
+        for volume in volumes:
+            try:
+                self.cinder_client.volumes.detach(volume)
+                self.cinder_client.volumes.force_delete(volume)
+            except Exception:
+                self.cinder_client.volumes.force_delete(volume)
+
+    @staticmethod
+    def _wait_for_status(max_waiting, func, **kwargs):
+        count = 0
+        while count < max_waiting:
+            if func(**kwargs):
+                return True
+            count += 1
+            time.sleep(2)
+        LOG.info("wait_for_status - False ")
+        return False
+
+    @staticmethod
+    def _compare_graphs(api_graph, cli_graph):
+        """Compare Graph object to graph form terminal """
+        if not api_graph:
+            LOG.error("The topology graph taken from rest api is empty")
+            return False
+        if not cli_graph:
+            LOG.error("The topology graph taken from terminal is empty")
+            return False
+
+        parsed_topology = json.loads(cli_graph)
+
+        sorted_cli_graph = sorted(parsed_topology.items())
+        sorted_api_graph = sorted(api_graph.items())
+
+        for item in sorted_cli_graph[4][1]:
+            item.pop(VProps.UPDATE_TIMESTAMP, None)
+
+        for item in sorted_api_graph[4][1]:
+            item.pop(VProps.UPDATE_TIMESTAMP, None)
+
+        return sorted_cli_graph == sorted_api_graph
+
+    @staticmethod
+    def _create_graph_from_graph_dictionary(api_graph):
+        graph = NXGraph()
+
+        nodes = api_graph['nodes']
+        for i in xrange(len(nodes)):
+            graph.add_vertex(Vertex(str(i), nodes[i]))
+
+        edges = api_graph['links']
+        for i in xrange(len(edges)):
+            graph.add_edge(Edge(str(edges[i]['source']),
+                                str(edges[i]['target']),
+                                edges[i]['relationship_type']))
+
+        return graph
+
+    def _create_graph_from_tree_dictionary(self,
+                                           api_graph,
+                                           graph=None,
+                                           ancestor=None):
+        children = []
+        graph = NXGraph() if not graph else graph
+
+        if 'children' in api_graph:
+            children = api_graph.copy()['children']
+            del api_graph['children']
+
+        vertex = Vertex(api_graph[VProps.VITRAGE_ID], api_graph)
+        graph.add_vertex(vertex)
+        if ancestor:
+            graph.add_edge(Edge(ancestor[VProps.VITRAGE_ID],
+                                vertex[VProps.VITRAGE_ID],
+                                'label'))
+
+        for entity in children:
+            self._create_graph_from_tree_dictionary(entity, graph, vertex)
+
+        return graph
+
+    @staticmethod
+    def _graph_query():
+        return '{"and": [{"==": {"category": "RESOURCE"}},' \
+               '{"==": {"is_deleted": false}},' \
+               '{"==": {"is_placeholder": false}},' \
+               '{"or": [{"==": {"type": "openstack.node"}},' \
+               '{"==": {"type": "nova.instance"}},' \
+               '{"==": {"type": "nova.host"}},' \
+               '{"==": {"type": "nova.zone"}}]}]}'
+
+    @staticmethod
+    def _tree_query():
+        return '{"and": [{"==": {"category": "RESOURCE"}},' \
+               '{"==": {"is_deleted": false}},' \
+               '{"==": {"is_placeholder": false}},' \
+               '{"or": [{"==": {"type": "openstack.node"}},' \
+               '{"==": {"type": "nova.host"}},' \
+               '{"==": {"type": "nova.zone"}}]}]}'

vitrage_tempest_tests/tests/api/topology/utils.py

@@ -27,8 +27,8 @@ LOG = logging.getLogger(__name__)
 class TopologyHelper(BaseVitrageTest):
     """Topology test class for Vitrage API tests."""
 
-    def setUp(self):
+    def __init__(self):
-        super(TopologyHelper, self).setUp()
+        super(TopologyHelper, self).__init__()
         self.client = utils.get_client()
         self.depth = ''
         self.query = ''
@@ -55,18 +55,6 @@ class TopologyHelper(BaseVitrageTest):
         return utils.run_vitrage_command_with_user(
             "vitrage topology show", self.conf.service_credentials.user)
 
-    def create_machines(self, machine_number):
-        flavor_id = self.get_flavor_id_from_list()
-        image_id = self.get_image_id_from_list()
-
-        resources = []
-        for r in range(start=0, stop=machine_number, step=1):
-            self.create_vm_with_exist_image("vm_for_test_" + str(r),
-                                            flavor_id, image_id)
-            resources.append("vm_for_test_" + str(r))
-
-        return resources
-
     def create_volume(self):
         flavor_id = self.get_flavor_id_from_list()
         image_id = self.get_image_id_from_list()

vitrage_tempest_tests/tests/utils.py

@@ -55,7 +55,14 @@ def get_from_terminal(command):
 
 
 def run_vitrage_command(command):
-    p = subprocess.Popen(command,
+    auth_url = '--os-auth-url http://10.41.251.201:5000/v2.0'
+    user = '--os-user-name admin'
+    password = '--os-password password'
+    project_name = '--os-project-name admin'
+    full_command = '%s %s %s %s %s' % \
+                   (command, user, password, project_name, auth_url)
+
+    p = subprocess.Popen(full_command,
                          shell=True,
                          executable="/bin/bash",
                          stdout=subprocess.PIPE,
@@ -71,7 +78,7 @@ def run_vitrage_command(command):
 
 def run_vitrage_command_with_user(command, user):
     run_vitrage_command(
-        "cd /home/stack/devstack; . openrc " +
+        "cd /openstack/devstack; . openrc " +
         user + " " + user +
         "; " + command)