Support get_changes in the static datasource

Change-Id: I69496b8216adaaf10a3c8e282eea34ce5ecc94cf
Implements: blueprint static-datasources-changes

releasenotes/notes/static-datasource-changes-914f9a16ad7e46ed.yaml

@@ -0,0 +1,3 @@
+---
+features:
+  - Support get_changes in the static datasource

releasenotes/notes/support-get-changes-in-static-datasource-02715226f103455d.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - The static datasource now supports changes in existing yaml files, and
+    updates the graph accordingly.

vitrage/datasources/static/__init__.py

@@ -62,3 +62,4 @@ class StaticFields(object):
     CATEGORY = 'category'
     ID = 'id'
     NAME = 'name'
+    STATE = 'state'

vitrage/datasources/static/driver.py

@@ -17,6 +17,8 @@ from six.moves import reduce
 
 from oslo_log import log
 
+from vitrage.common.constants import DatasourceProperties as DSProps
+from vitrage.common.constants import GraphAction
 from vitrage.datasources.driver_base import DriverBase
 from vitrage.datasources.static import STATIC_DATASOURCE
 from vitrage.datasources.static import StaticFields
@@ -34,6 +36,7 @@ class StaticDriver(DriverBase):
     def __init__(self, conf):
         super(StaticDriver, self).__init__()
         self.cfg = conf
+        self.entities_cache = []
 
     @staticmethod
     def _is_valid_config(config):
@@ -49,23 +52,48 @@ class StaticDriver(DriverBase):
         pass
 
     def get_all(self, datasource_action):
-        return self.make_pickleable(self._get_all_entities(),
+        return self.make_pickleable(self._get_and_cache_all_entities(),
                                     STATIC_DATASOURCE,
                                     datasource_action)
 
     def get_changes(self, datasource_action):
-        return self.make_pickleable(self._get_changes_entities(),
+        return self.make_pickleable(self._get_and_cache_changed_entities(),
                                     STATIC_DATASOURCE,
                                     datasource_action)
 
+    def _get_and_cache_all_entities(self):
+        self.entities_cache = self._get_all_entities()
+        return self.entities_cache
+
     def _get_all_entities(self):
         files = file_utils.list_files(self.cfg.static.directory, '.yaml', True)
-        return reduce(chain, [self._get_entities_from_file(path)
+        return list(reduce(chain, [self._get_entities_from_file(path)
-                              for path in files], [])
+                                   for path in files], []))
 
-    def _get_changes_entities(self):
+    def _get_and_cache_changed_entities(self):
-        """TODO(yujunz): update from file change or CRUD"""
+        changed_entities = []
-        return []
+        new_entities = self._get_all_entities()
+
+        for new_entity in new_entities:
+            old_entity = self._find_entity(new_entity, self.entities_cache)
+
+            if old_entity:
+                # Add modified entities
+                if not self._equal_entities(old_entity, new_entity):
+                    changed_entities.append(new_entity.copy())
+            else:
+                # Add new entities
+                changed_entities.append(new_entity.copy())
+
+        # Add deleted entities
+        for old_entity in self.entities_cache:
+            if not self._find_entity(old_entity, new_entities):
+                old_entity_copy = old_entity.copy()
+                old_entity_copy[DSProps.EVENT_TYPE] = GraphAction.DELETE_ENTITY
+                changed_entities.append(old_entity_copy)
+
+        self.entities_cache = new_entities
+        return changed_entities
 
     @classmethod
     def _get_entities_from_file(cls, path):
@@ -140,3 +168,24 @@ class StaticDriver(DriverBase):
                       .format(neighbor, rel))
             return None
         return rel
+
+    @staticmethod
+    def _find_entity(search_entity, entities):
+        # naive implementation since we don't expect many static entities
+        for entity in entities:
+            if entity[StaticFields.TYPE] == search_entity[StaticFields.TYPE] \
+                    and entity[StaticFields.ID] == \
+                    search_entity[StaticFields.ID]:
+                return entity
+
+    @staticmethod
+    def _equal_entities(old_entity, new_entity):
+        # TODO(iafek): compare also the relationships
+        return old_entity.get(StaticFields.TYPE) == \
+            new_entity.get(StaticFields.TYPE) and \
+            old_entity.get(StaticFields.ID) == \
+            new_entity.get(StaticFields.ID) and \
+            old_entity.get(StaticFields.NAME) == \
+            new_entity.get(StaticFields.NAME) and \
+            old_entity.get(StaticFields.STATE) == \
+            new_entity.get(StaticFields.STATE)

vitrage/tests/resources/static_datasources/changes_datasources/added_resources/static.yaml

@@ -0,0 +1,48 @@
+metadata:
+  name: switch to host
+  description: static datasource for test
+definitions:
+  entities:
+    - static_id: s1
+      type: switch
+      name: switch-1
+      id: 12345
+      state: available
+    - static_id: s2
+      type: switch
+      name: switch-2
+      id: 23456
+      state: available
+    - static_id: s3
+      type: switch
+      name: switch-3 is new!
+      id: 3333
+      state: available
+    - static_id: r1
+      type: router
+      name: router-1
+      id: 45678
+    - static_id: r2
+      type: router
+      name: router-2 is new!
+      id: 2222
+      state: available
+    - static_id: h1
+      type: nova.host
+      id: 1
+  relationships:
+    - source: s1
+      target: r1
+      relationship_type: attached
+    - source: s2
+      target: r1
+      relationship_type: attached
+    - source: s3
+      target: r2
+      relationship_type: attached
+    - source: r1
+      target: h1
+      relationship_type: attached
+    - source: r2
+      target: h1
+      relationship_type: attached

vitrage/tests/resources/static_datasources/changes_datasources/baseline/static.yaml

@@ -0,0 +1,32 @@
+metadata:
+  name: switch to host
+  description: static datasource for test
+definitions:
+  entities:
+    - static_id: s1
+      type: switch
+      name: switch-1
+      id: 12345
+      state: available
+    - static_id: s2
+      type: switch
+      name: switch-2
+      id: 23456
+      state: available
+    - static_id: r1
+      type: router
+      name: router-1
+      id: 45678
+    - static_id: h1
+      type: nova.host
+      id: 1
+  relationships:
+    - source: s1
+      target: r1
+      relationship_type: attached
+    - source: s2
+      target: r1
+      relationship_type: attached
+    - source: r1
+      target: h1
+      relationship_type: attached

vitrage/tests/resources/static_datasources/changes_datasources/changed_resources/static.yaml

@@ -0,0 +1,32 @@
+metadata:
+  name: switch to host
+  description: static datasource for test
+definitions:
+  entities:
+    - static_id: s1
+      type: switch
+      name: switch-1
+      id: 12345
+      state: error   # state change
+    - static_id: s2
+      type: switch
+      name: switch-2
+      id: 23456
+      state: available
+    - static_id: r1
+      type: router
+      name: router-1 is the best!  # name change
+      id: 45678
+    - static_id: h1
+      type: nova.host
+      id: 1
+  relationships:
+    - source: s1
+      target: r1
+      relationship_type: attached
+    - source: s2
+      target: r1
+      relationship_type: attached
+    - source: r1
+      target: h1
+      relationship_type: attached

vitrage/tests/resources/static_datasources/changes_datasources/deleted_resources/static.yaml

@@ -0,0 +1,18 @@
+metadata:
+  name: switch to host
+  description: static datasource for test
+definitions:
+  entities:
+    - static_id: s1
+      type: switch
+      name: switch-1
+      id: 12345
+      state: available
+    - static_id: r1
+      type: router
+      name: router-1
+      id: 45678
+  relationships:
+    - source: s1
+      target: r1
+      relationship_type: attached

vitrage/tests/resources/static_datasources/changes_datasources/mixed_changes/static.yaml

@@ -0,0 +1,31 @@
+metadata:
+  name: switch to host
+  description: static datasource for test
+definitions:
+  entities:
+    - static_id: s2
+      type: switch
+      name: switch-2
+      id: 23456
+      state: error
+    - static_id: r1
+      type: router
+      name: router-1
+      id: 45678
+    - static_id: r2
+      type: router
+      name: router-2 is new!
+      id: 222
+    - static_id: h1
+      type: nova.host
+      id: 1
+  relationships:
+    - source: s2
+      target: r1
+      relationship_type: attached
+    - source: r1
+      target: h1
+      relationship_type: attached
+    - source: r2
+      target: h1
+      relationship_type: attached

vitrage/tests/unit/datasources/static/test_static_driver.py

@@ -22,54 +22,22 @@ from vitrage.datasources.static import driver
 from vitrage.datasources.static import STATIC_DATASOURCE
 from vitrage.datasources.static import StaticFields
 from vitrage.tests import base
-from vitrage.tests.base import IsEmpty
 from vitrage.tests.mocks import utils
 
 
 class TestStaticDriver(base.BaseTest):
 
-    OPTS = [
+    CHANGES_DIR = '/changes_datasources'
-        cfg.StrOpt(DSOpts.TRANSFORMER,
-                   default='vitrage.datasources.static.transformer.'
-                           'StaticTransformer'),
-        cfg.StrOpt(DSOpts.DRIVER,
-                   default='vitrage.datasources.static.driver.'
-                           'StaticDriver'),
-        cfg.IntOpt(DSOpts.CHANGES_INTERVAL,
-                   default=30,
-                   min=30,
-                   help='interval between checking changes in the '
-                        'configuration files of the static datasources'),
-        cfg.StrOpt('directory',
-                   default=utils.get_resources_dir() + '/static_datasources')
-    ]
-
-    CHANGES_OPTS = [
-        cfg.StrOpt(DSOpts.TRANSFORMER,
-                   default='vitrage.datasources.static.transformer.'
-                           'StaticTransformer'),
-        cfg.StrOpt(DSOpts.DRIVER,
-                   default='vitrage.datasources.static.driver.'
-                           'StaticDriver'),
-        cfg.IntOpt(DSOpts.CHANGES_INTERVAL,
-                   default=30,
-                   min=30,
-                   help='interval between checking changes in the static '
-                        'datasources'),
-        cfg.StrOpt('directory',
-                   default=utils.get_resources_dir() +
-                   '/static_datasources/changes_datasources'),
-    ]
 
     # noinspection PyAttributeOutsideInit,PyPep8Naming
     @classmethod
     def setUpClass(cls):
         super(TestStaticDriver, cls).setUpClass()
-        cls.conf = cfg.ConfigOpts()
+        cls.static_driver = driver.StaticDriver(None)
-        cls.conf.register_opts(cls.OPTS, group=STATIC_DATASOURCE)
-        cls.static_driver = driver.StaticDriver(cls.conf)
 
     def test_get_all(self):
+        self._set_conf()
+
         # Action
         static_entities = self.static_driver.get_all(
             DatasourceAction.INIT_SNAPSHOT)
@@ -81,24 +49,108 @@ class TestStaticDriver(base.BaseTest):
             self._validate_static_entity(entity)
 
     # noinspection PyAttributeOutsideInit
-    def test_get_changes(self):
+    def test_get_changes_with_added_resources(self):
-        # Setup
+        # Get initial resources
-        entities = self.static_driver.get_all(DatasourceAction.UPDATE)
+        self._set_conf(self.CHANGES_DIR + '/baseline')
-        self.assertThat(entities, matchers.HasLength(8))
 
-        self.conf = cfg.ConfigOpts()
+        entities = self.static_driver.get_all(DatasourceAction.UPDATE)
-        self.conf.register_opts(self.CHANGES_OPTS,
+        self.assertThat(entities, matchers.HasLength(4))
-                                group=STATIC_DATASOURCE)
+
-        self.static_driver.cfg = self.conf
+        # Add resources
+        self._set_conf(self.CHANGES_DIR + '/added_resources')
 
         # Action
         changes = self.static_driver.get_changes(
             GraphAction.UPDATE_ENTITY)
 
         # Test Assertions
-        self.assertThat(changes, IsEmpty())
+        expected_changes = [
-        for entity in changes:
+            {'static_id': 's3', 'type': 'switch', 'name': 'switch-3 is new!',
-            self._validate_static_entity(entity)
+             'id': '3333', 'state': 'available'},
+            {'static_id': 'r2', 'type': 'router', 'name': 'router-2 is new!',
+             'id': '2222', 'state': 'available'},
+        ]
+
+        self._validate_static_changes(expected_changes, changes)
+
+    # noinspection PyAttributeOutsideInit
+    def test_get_changes_with_deleted_resources(self):
+        # Get initial resources
+        self._set_conf(self.CHANGES_DIR + '/baseline')
+
+        entities = self.static_driver.get_all(DatasourceAction.UPDATE)
+        self.assertThat(entities, matchers.HasLength(4))
+
+        # Delete resources
+        self._set_conf(self.CHANGES_DIR + '/deleted_resources')
+
+        # Action
+        changes = self.static_driver.get_changes(
+            GraphAction.UPDATE_ENTITY)
+
+        # Test Assertions
+        expected_changes = [
+            {'static_id': 's2', 'type': 'switch', 'name': 'switch-2',
+             'id': '23456', 'state': 'available',
+             'vitrage_event_type': 'delete_entity'},
+            {'static_id': 'h1', 'type': 'nova.host', 'id': '1',
+             'vitrage_event_type': 'delete_entity'},
+        ]
+
+        self._validate_static_changes(expected_changes, changes)
+
+    # noinspection PyAttributeOutsideInit
+    def test_get_changes_with_changed_resources(self):
+        # Get initial resources
+        self._set_conf(self.CHANGES_DIR + '/baseline')
+
+        entities = self.static_driver.get_all(DatasourceAction.UPDATE)
+        self.assertThat(entities, matchers.HasLength(4))
+
+        # Delete resources
+        self._set_conf(self.CHANGES_DIR + '/changed_resources')
+
+        # Action
+        changes = self.static_driver.get_changes(
+            GraphAction.UPDATE_ENTITY)
+
+        # Test Assertions
+        expected_changes = [
+            {'static_id': 's1', 'type': 'switch', 'name': 'switch-1',
+             'id': '12345', 'state': 'error'},
+            {'static_id': 'r1', 'type': 'router',
+             'name': 'router-1 is the best!', 'id': '45678'},
+        ]
+
+        self._validate_static_changes(expected_changes, changes)
+
+    # noinspection PyAttributeOutsideInit
+    def test_get_changes_with_mixed_changes(self):
+        # Get initial resources
+        self._set_conf(self.CHANGES_DIR + '/baseline')
+
+        entities = self.static_driver.get_all(DatasourceAction.UPDATE)
+        self.assertThat(entities, matchers.HasLength(4))
+
+        # Delete resources
+        self._set_conf(self.CHANGES_DIR + '/mixed_changes')
+
+        # Action
+        changes = self.static_driver.get_changes(
+            GraphAction.UPDATE_ENTITY)
+
+        # Test Assertions
+        expected_changes = [
+            {'static_id': 's1', 'type': 'switch', 'name': 'switch-1',
+             'id': '12345', 'state': 'available',
+             'vitrage_event_type': 'delete_entity'},
+            {'static_id': 's2', 'type': 'switch', 'name': 'switch-2',
+             'id': '23456', 'state': 'error'},
+            {'static_id': 'r2', 'type': 'router', 'name': 'router-2 is new!',
+             'id': '222'},
+        ]
+
+        self._validate_static_changes(expected_changes, changes)
 
     def _validate_static_entity(self, entity):
         self.assertIsInstance(entity[StaticFields.METADATA], dict)
@@ -115,3 +167,48 @@ class TestStaticDriver(base.BaseTest):
             and entity[StaticFields.STATIC_ID] == rel[StaticFields.SOURCE]
             or entity[StaticFields.STATIC_ID] == rel[StaticFields.SOURCE]
             and entity[StaticFields.STATIC_ID] == rel[StaticFields.TARGET])
+
+    def _validate_static_changes(self, expected_changes, changes):
+        self.assertThat(changes, matchers.HasLength(len(expected_changes)))
+
+        for entity in changes:
+            self._validate_static_entity(entity)
+
+        for expected_change in expected_changes:
+            found = False
+            for change in changes:
+                if change[StaticFields.TYPE] == \
+                        expected_change[StaticFields.TYPE] and \
+                        change[StaticFields.ID] == \
+                        expected_change[StaticFields.ID]:
+                    found = True
+                    self.assertEqual(expected_change.get('vitrage_event_type'),
+                                     change.get('vitrage_event_type'))
+                    self.assertEqual(expected_change.get(StaticFields.NAME),
+                                     change.get(StaticFields.NAME))
+                    self.assertEqual(expected_change.get(StaticFields.STATE),
+                                     change.get(StaticFields.STATE))
+            self.assertTrue(found)
+
+    def _set_conf(self, sub_dir=None):
+        default_dir = utils.get_resources_dir() + \
+            '/static_datasources' + (sub_dir if sub_dir else '')
+
+        opts = [
+            cfg.StrOpt(DSOpts.TRANSFORMER,
+                       default='vitrage.datasources.static.transformer.'
+                               'StaticTransformer'),
+            cfg.StrOpt(DSOpts.DRIVER,
+                       default='vitrage.datasources.static.driver.'
+                               'StaticDriver'),
+            cfg.IntOpt(DSOpts.CHANGES_INTERVAL,
+                       default=30,
+                       min=30,
+                       help='interval between checking changes in the static '
+                            'datasources'),
+            cfg.StrOpt('directory', default=default_dir),
+        ]
+
+        self.conf = cfg.ConfigOpts()
+        self.conf.register_opts(opts, group=STATIC_DATASOURCE)
+        self.static_driver.cfg = self.conf