diff --git a/releasenotes/notes/static-datasource-changes-914f9a16ad7e46ed.yaml b/releasenotes/notes/static-datasource-changes-914f9a16ad7e46ed.yaml new file mode 100644 index 000000000..0e1696e94 --- /dev/null +++ b/releasenotes/notes/static-datasource-changes-914f9a16ad7e46ed.yaml @@ -0,0 +1,3 @@ +--- +features: + - Support get_changes in the static datasource diff --git a/releasenotes/notes/support-get-changes-in-static-datasource-02715226f103455d.yaml b/releasenotes/notes/support-get-changes-in-static-datasource-02715226f103455d.yaml new file mode 100644 index 000000000..2b0b78b94 --- /dev/null +++ b/releasenotes/notes/support-get-changes-in-static-datasource-02715226f103455d.yaml @@ -0,0 +1,4 @@ +--- +features: + - The static datasource now supports changes in existing yaml files, and + updates the graph accordingly. diff --git a/vitrage/datasources/static/__init__.py b/vitrage/datasources/static/__init__.py index 9d10b6078..e1b197294 100644 --- a/vitrage/datasources/static/__init__.py +++ b/vitrage/datasources/static/__init__.py @@ -62,3 +62,4 @@ class StaticFields(object): CATEGORY = 'category' ID = 'id' NAME = 'name' + STATE = 'state' diff --git a/vitrage/datasources/static/driver.py b/vitrage/datasources/static/driver.py index 202a9634e..53c440160 100644 --- a/vitrage/datasources/static/driver.py +++ b/vitrage/datasources/static/driver.py @@ -17,6 +17,8 @@ from six.moves import reduce from oslo_log import log +from vitrage.common.constants import DatasourceProperties as DSProps +from vitrage.common.constants import GraphAction from vitrage.datasources.driver_base import DriverBase from vitrage.datasources.static import STATIC_DATASOURCE from vitrage.datasources.static import StaticFields @@ -34,6 +36,7 @@ class StaticDriver(DriverBase): def __init__(self, conf): super(StaticDriver, self).__init__() self.cfg = conf + self.entities_cache = [] @staticmethod def _is_valid_config(config): @@ -49,23 +52,48 @@ class StaticDriver(DriverBase): pass def get_all(self, datasource_action): - return self.make_pickleable(self._get_all_entities(), + return self.make_pickleable(self._get_and_cache_all_entities(), STATIC_DATASOURCE, datasource_action) def get_changes(self, datasource_action): - return self.make_pickleable(self._get_changes_entities(), + return self.make_pickleable(self._get_and_cache_changed_entities(), STATIC_DATASOURCE, datasource_action) + def _get_and_cache_all_entities(self): + self.entities_cache = self._get_all_entities() + return self.entities_cache + def _get_all_entities(self): files = file_utils.list_files(self.cfg.static.directory, '.yaml', True) - return reduce(chain, [self._get_entities_from_file(path) - for path in files], []) + return list(reduce(chain, [self._get_entities_from_file(path) + for path in files], [])) - def _get_changes_entities(self): - """TODO(yujunz): update from file change or CRUD""" - return [] + def _get_and_cache_changed_entities(self): + changed_entities = [] + new_entities = self._get_all_entities() + + for new_entity in new_entities: + old_entity = self._find_entity(new_entity, self.entities_cache) + + if old_entity: + # Add modified entities + if not self._equal_entities(old_entity, new_entity): + changed_entities.append(new_entity.copy()) + else: + # Add new entities + changed_entities.append(new_entity.copy()) + + # Add deleted entities + for old_entity in self.entities_cache: + if not self._find_entity(old_entity, new_entities): + old_entity_copy = old_entity.copy() + old_entity_copy[DSProps.EVENT_TYPE] = GraphAction.DELETE_ENTITY + changed_entities.append(old_entity_copy) + + self.entities_cache = new_entities + return changed_entities @classmethod def _get_entities_from_file(cls, path): @@ -140,3 +168,24 @@ class StaticDriver(DriverBase): .format(neighbor, rel)) return None return rel + + @staticmethod + def _find_entity(search_entity, entities): + # naive implementation since we don't expect many static entities + for entity in entities: + if entity[StaticFields.TYPE] == search_entity[StaticFields.TYPE] \ + and entity[StaticFields.ID] == \ + search_entity[StaticFields.ID]: + return entity + + @staticmethod + def _equal_entities(old_entity, new_entity): + # TODO(iafek): compare also the relationships + return old_entity.get(StaticFields.TYPE) == \ + new_entity.get(StaticFields.TYPE) and \ + old_entity.get(StaticFields.ID) == \ + new_entity.get(StaticFields.ID) and \ + old_entity.get(StaticFields.NAME) == \ + new_entity.get(StaticFields.NAME) and \ + old_entity.get(StaticFields.STATE) == \ + new_entity.get(StaticFields.STATE) diff --git a/vitrage/tests/resources/static_datasources/changes_datasources/added_resources/static.yaml b/vitrage/tests/resources/static_datasources/changes_datasources/added_resources/static.yaml new file mode 100644 index 000000000..1a3b16a95 --- /dev/null +++ b/vitrage/tests/resources/static_datasources/changes_datasources/added_resources/static.yaml @@ -0,0 +1,48 @@ +metadata: + name: switch to host + description: static datasource for test +definitions: + entities: + - static_id: s1 + type: switch + name: switch-1 + id: 12345 + state: available + - static_id: s2 + type: switch + name: switch-2 + id: 23456 + state: available + - static_id: s3 + type: switch + name: switch-3 is new! + id: 3333 + state: available + - static_id: r1 + type: router + name: router-1 + id: 45678 + - static_id: r2 + type: router + name: router-2 is new! + id: 2222 + state: available + - static_id: h1 + type: nova.host + id: 1 + relationships: + - source: s1 + target: r1 + relationship_type: attached + - source: s2 + target: r1 + relationship_type: attached + - source: s3 + target: r2 + relationship_type: attached + - source: r1 + target: h1 + relationship_type: attached + - source: r2 + target: h1 + relationship_type: attached diff --git a/vitrage/tests/resources/static_datasources/changes_datasources/baseline/static.yaml b/vitrage/tests/resources/static_datasources/changes_datasources/baseline/static.yaml new file mode 100644 index 000000000..841c31bb8 --- /dev/null +++ b/vitrage/tests/resources/static_datasources/changes_datasources/baseline/static.yaml @@ -0,0 +1,32 @@ +metadata: + name: switch to host + description: static datasource for test +definitions: + entities: + - static_id: s1 + type: switch + name: switch-1 + id: 12345 + state: available + - static_id: s2 + type: switch + name: switch-2 + id: 23456 + state: available + - static_id: r1 + type: router + name: router-1 + id: 45678 + - static_id: h1 + type: nova.host + id: 1 + relationships: + - source: s1 + target: r1 + relationship_type: attached + - source: s2 + target: r1 + relationship_type: attached + - source: r1 + target: h1 + relationship_type: attached diff --git a/vitrage/tests/resources/static_datasources/changes_datasources/changed_resources/static.yaml b/vitrage/tests/resources/static_datasources/changes_datasources/changed_resources/static.yaml new file mode 100644 index 000000000..ade66b4ae --- /dev/null +++ b/vitrage/tests/resources/static_datasources/changes_datasources/changed_resources/static.yaml @@ -0,0 +1,32 @@ +metadata: + name: switch to host + description: static datasource for test +definitions: + entities: + - static_id: s1 + type: switch + name: switch-1 + id: 12345 + state: error # state change + - static_id: s2 + type: switch + name: switch-2 + id: 23456 + state: available + - static_id: r1 + type: router + name: router-1 is the best! # name change + id: 45678 + - static_id: h1 + type: nova.host + id: 1 + relationships: + - source: s1 + target: r1 + relationship_type: attached + - source: s2 + target: r1 + relationship_type: attached + - source: r1 + target: h1 + relationship_type: attached diff --git a/vitrage/tests/resources/static_datasources/changes_datasources/deleted_resources/static.yaml b/vitrage/tests/resources/static_datasources/changes_datasources/deleted_resources/static.yaml new file mode 100644 index 000000000..f914f3026 --- /dev/null +++ b/vitrage/tests/resources/static_datasources/changes_datasources/deleted_resources/static.yaml @@ -0,0 +1,18 @@ +metadata: + name: switch to host + description: static datasource for test +definitions: + entities: + - static_id: s1 + type: switch + name: switch-1 + id: 12345 + state: available + - static_id: r1 + type: router + name: router-1 + id: 45678 + relationships: + - source: s1 + target: r1 + relationship_type: attached diff --git a/vitrage/tests/resources/static_datasources/changes_datasources/mixed_changes/static.yaml b/vitrage/tests/resources/static_datasources/changes_datasources/mixed_changes/static.yaml new file mode 100644 index 000000000..8708caeba --- /dev/null +++ b/vitrage/tests/resources/static_datasources/changes_datasources/mixed_changes/static.yaml @@ -0,0 +1,31 @@ +metadata: + name: switch to host + description: static datasource for test +definitions: + entities: + - static_id: s2 + type: switch + name: switch-2 + id: 23456 + state: error + - static_id: r1 + type: router + name: router-1 + id: 45678 + - static_id: r2 + type: router + name: router-2 is new! + id: 222 + - static_id: h1 + type: nova.host + id: 1 + relationships: + - source: s2 + target: r1 + relationship_type: attached + - source: r1 + target: h1 + relationship_type: attached + - source: r2 + target: h1 + relationship_type: attached diff --git a/vitrage/tests/unit/datasources/static/test_static_driver.py b/vitrage/tests/unit/datasources/static/test_static_driver.py index 6de54ff66..04864905b 100644 --- a/vitrage/tests/unit/datasources/static/test_static_driver.py +++ b/vitrage/tests/unit/datasources/static/test_static_driver.py @@ -22,54 +22,22 @@ from vitrage.datasources.static import driver from vitrage.datasources.static import STATIC_DATASOURCE from vitrage.datasources.static import StaticFields from vitrage.tests import base -from vitrage.tests.base import IsEmpty from vitrage.tests.mocks import utils class TestStaticDriver(base.BaseTest): - OPTS = [ - cfg.StrOpt(DSOpts.TRANSFORMER, - default='vitrage.datasources.static.transformer.' - 'StaticTransformer'), - cfg.StrOpt(DSOpts.DRIVER, - default='vitrage.datasources.static.driver.' - 'StaticDriver'), - cfg.IntOpt(DSOpts.CHANGES_INTERVAL, - default=30, - min=30, - help='interval between checking changes in the ' - 'configuration files of the static datasources'), - cfg.StrOpt('directory', - default=utils.get_resources_dir() + '/static_datasources') - ] - - CHANGES_OPTS = [ - cfg.StrOpt(DSOpts.TRANSFORMER, - default='vitrage.datasources.static.transformer.' - 'StaticTransformer'), - cfg.StrOpt(DSOpts.DRIVER, - default='vitrage.datasources.static.driver.' - 'StaticDriver'), - cfg.IntOpt(DSOpts.CHANGES_INTERVAL, - default=30, - min=30, - help='interval between checking changes in the static ' - 'datasources'), - cfg.StrOpt('directory', - default=utils.get_resources_dir() + - '/static_datasources/changes_datasources'), - ] + CHANGES_DIR = '/changes_datasources' # noinspection PyAttributeOutsideInit,PyPep8Naming @classmethod def setUpClass(cls): super(TestStaticDriver, cls).setUpClass() - cls.conf = cfg.ConfigOpts() - cls.conf.register_opts(cls.OPTS, group=STATIC_DATASOURCE) - cls.static_driver = driver.StaticDriver(cls.conf) + cls.static_driver = driver.StaticDriver(None) def test_get_all(self): + self._set_conf() + # Action static_entities = self.static_driver.get_all( DatasourceAction.INIT_SNAPSHOT) @@ -81,24 +49,108 @@ class TestStaticDriver(base.BaseTest): self._validate_static_entity(entity) # noinspection PyAttributeOutsideInit - def test_get_changes(self): - # Setup - entities = self.static_driver.get_all(DatasourceAction.UPDATE) - self.assertThat(entities, matchers.HasLength(8)) + def test_get_changes_with_added_resources(self): + # Get initial resources + self._set_conf(self.CHANGES_DIR + '/baseline') - self.conf = cfg.ConfigOpts() - self.conf.register_opts(self.CHANGES_OPTS, - group=STATIC_DATASOURCE) - self.static_driver.cfg = self.conf + entities = self.static_driver.get_all(DatasourceAction.UPDATE) + self.assertThat(entities, matchers.HasLength(4)) + + # Add resources + self._set_conf(self.CHANGES_DIR + '/added_resources') # Action changes = self.static_driver.get_changes( GraphAction.UPDATE_ENTITY) # Test Assertions - self.assertThat(changes, IsEmpty()) - for entity in changes: - self._validate_static_entity(entity) + expected_changes = [ + {'static_id': 's3', 'type': 'switch', 'name': 'switch-3 is new!', + 'id': '3333', 'state': 'available'}, + {'static_id': 'r2', 'type': 'router', 'name': 'router-2 is new!', + 'id': '2222', 'state': 'available'}, + ] + + self._validate_static_changes(expected_changes, changes) + + # noinspection PyAttributeOutsideInit + def test_get_changes_with_deleted_resources(self): + # Get initial resources + self._set_conf(self.CHANGES_DIR + '/baseline') + + entities = self.static_driver.get_all(DatasourceAction.UPDATE) + self.assertThat(entities, matchers.HasLength(4)) + + # Delete resources + self._set_conf(self.CHANGES_DIR + '/deleted_resources') + + # Action + changes = self.static_driver.get_changes( + GraphAction.UPDATE_ENTITY) + + # Test Assertions + expected_changes = [ + {'static_id': 's2', 'type': 'switch', 'name': 'switch-2', + 'id': '23456', 'state': 'available', + 'vitrage_event_type': 'delete_entity'}, + {'static_id': 'h1', 'type': 'nova.host', 'id': '1', + 'vitrage_event_type': 'delete_entity'}, + ] + + self._validate_static_changes(expected_changes, changes) + + # noinspection PyAttributeOutsideInit + def test_get_changes_with_changed_resources(self): + # Get initial resources + self._set_conf(self.CHANGES_DIR + '/baseline') + + entities = self.static_driver.get_all(DatasourceAction.UPDATE) + self.assertThat(entities, matchers.HasLength(4)) + + # Delete resources + self._set_conf(self.CHANGES_DIR + '/changed_resources') + + # Action + changes = self.static_driver.get_changes( + GraphAction.UPDATE_ENTITY) + + # Test Assertions + expected_changes = [ + {'static_id': 's1', 'type': 'switch', 'name': 'switch-1', + 'id': '12345', 'state': 'error'}, + {'static_id': 'r1', 'type': 'router', + 'name': 'router-1 is the best!', 'id': '45678'}, + ] + + self._validate_static_changes(expected_changes, changes) + + # noinspection PyAttributeOutsideInit + def test_get_changes_with_mixed_changes(self): + # Get initial resources + self._set_conf(self.CHANGES_DIR + '/baseline') + + entities = self.static_driver.get_all(DatasourceAction.UPDATE) + self.assertThat(entities, matchers.HasLength(4)) + + # Delete resources + self._set_conf(self.CHANGES_DIR + '/mixed_changes') + + # Action + changes = self.static_driver.get_changes( + GraphAction.UPDATE_ENTITY) + + # Test Assertions + expected_changes = [ + {'static_id': 's1', 'type': 'switch', 'name': 'switch-1', + 'id': '12345', 'state': 'available', + 'vitrage_event_type': 'delete_entity'}, + {'static_id': 's2', 'type': 'switch', 'name': 'switch-2', + 'id': '23456', 'state': 'error'}, + {'static_id': 'r2', 'type': 'router', 'name': 'router-2 is new!', + 'id': '222'}, + ] + + self._validate_static_changes(expected_changes, changes) def _validate_static_entity(self, entity): self.assertIsInstance(entity[StaticFields.METADATA], dict) @@ -115,3 +167,48 @@ class TestStaticDriver(base.BaseTest): and entity[StaticFields.STATIC_ID] == rel[StaticFields.SOURCE] or entity[StaticFields.STATIC_ID] == rel[StaticFields.SOURCE] and entity[StaticFields.STATIC_ID] == rel[StaticFields.TARGET]) + + def _validate_static_changes(self, expected_changes, changes): + self.assertThat(changes, matchers.HasLength(len(expected_changes))) + + for entity in changes: + self._validate_static_entity(entity) + + for expected_change in expected_changes: + found = False + for change in changes: + if change[StaticFields.TYPE] == \ + expected_change[StaticFields.TYPE] and \ + change[StaticFields.ID] == \ + expected_change[StaticFields.ID]: + found = True + self.assertEqual(expected_change.get('vitrage_event_type'), + change.get('vitrage_event_type')) + self.assertEqual(expected_change.get(StaticFields.NAME), + change.get(StaticFields.NAME)) + self.assertEqual(expected_change.get(StaticFields.STATE), + change.get(StaticFields.STATE)) + self.assertTrue(found) + + def _set_conf(self, sub_dir=None): + default_dir = utils.get_resources_dir() + \ + '/static_datasources' + (sub_dir if sub_dir else '') + + opts = [ + cfg.StrOpt(DSOpts.TRANSFORMER, + default='vitrage.datasources.static.transformer.' + 'StaticTransformer'), + cfg.StrOpt(DSOpts.DRIVER, + default='vitrage.datasources.static.driver.' + 'StaticDriver'), + cfg.IntOpt(DSOpts.CHANGES_INTERVAL, + default=30, + min=30, + help='interval between checking changes in the static ' + 'datasources'), + cfg.StrOpt('directory', default=default_dir), + ] + + self.conf = cfg.ConfigOpts() + self.conf.register_opts(opts, group=STATIC_DATASOURCE) + self.static_driver.cfg = self.conf