openstack/vitrage
Code Issues Proposed changes

Support get_changes in the static datasource

Browse Source 
Change-Id: I69496b8216adaaf10a3c8e282eea34ce5ecc94cf
Implements: blueprint static-datasources-changes
This commit is contained in:
Ifat Afek 2018-05-13 09:50:48 +00:00
parent 75e121f81b
commit d5b4daf8aa
10 changed files with 369 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
releasenotes/notes/static-datasource-changes-914f9a16ad7e46ed.yaml Normal file
View File

@ -0,0 +1,3 @@
---
features:
- Support get_changes in the static datasource

4
releasenotes/notes/support-get-changes-in-static-datasource-02715226f103455d.yaml Normal file
View File

@ -0,0 +1,4 @@
---
features:
- The static datasource now supports changes in existing yaml files, and
updates the graph accordingly.

1
vitrage/datasources/static/__init__.py
View File

@ -62,3 +62,4 @@ class StaticFields(object):
CATEGORY = 'category'
ID = 'id'
NAME = 'name'
STATE = 'state'

63
vitrage/datasources/static/driver.py
View File

@ -17,6 +17,8 @@ from six.moves import reduce
from oslo_log import log
from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import GraphAction
from vitrage.datasources.driver_base import DriverBase
from vitrage.datasources.static import STATIC_DATASOURCE
from vitrage.datasources.static import StaticFields
@ -34,6 +36,7 @@ class StaticDriver(DriverBase):
def __init__(self, conf):
super(StaticDriver, self).__init__()
self.cfg = conf
self.entities_cache = []
@staticmethod
def _is_valid_config(config):
@ -49,23 +52,48 @@ class StaticDriver(DriverBase):
pass
def get_all(self, datasource_action):
return self.make_pickleable(self._get_all_entities(),
return self.make_pickleable(self._get_and_cache_all_entities(),
STATIC_DATASOURCE,
datasource_action)
def get_changes(self, datasource_action):
return self.make_pickleable(self._get_changes_entities(),
return self.make_pickleable(self._get_and_cache_changed_entities(),
STATIC_DATASOURCE,
datasource_action)
def _get_and_cache_all_entities(self):
self.entities_cache = self._get_all_entities()
return self.entities_cache
def _get_all_entities(self):
files = file_utils.list_files(self.cfg.static.directory, '.yaml', True)
return reduce(chain, [self._get_entities_from_file(path)
for path in files], [])
return list(reduce(chain, [self._get_entities_from_file(path)
for path in files], []))
def _get_changes_entities(self):
"""TODO(yujunz): update from file change or CRUD"""
return []
def _get_and_cache_changed_entities(self):
changed_entities = []
new_entities = self._get_all_entities()
for new_entity in new_entities:
old_entity = self._find_entity(new_entity, self.entities_cache)
if old_entity:
# Add modified entities
if not self._equal_entities(old_entity, new_entity):
changed_entities.append(new_entity.copy())
else:
# Add new entities
changed_entities.append(new_entity.copy())
# Add deleted entities
for old_entity in self.entities_cache:
if not self._find_entity(old_entity, new_entities):
old_entity_copy = old_entity.copy()
old_entity_copy[DSProps.EVENT_TYPE] = GraphAction.DELETE_ENTITY
changed_entities.append(old_entity_copy)
self.entities_cache = new_entities
return changed_entities
@classmethod
def _get_entities_from_file(cls, path):
@ -140,3 +168,24 @@ class StaticDriver(DriverBase):
.format(neighbor, rel))
return None
return rel
@staticmethod
def _find_entity(search_entity, entities):
# naive implementation since we don't expect many static entities
for entity in entities:
if entity[StaticFields.TYPE] == search_entity[StaticFields.TYPE] \
and entity[StaticFields.ID] == \
search_entity[StaticFields.ID]:
return entity
@staticmethod
def _equal_entities(old_entity, new_entity):
# TODO(iafek): compare also the relationships
return old_entity.get(StaticFields.TYPE) == \
new_entity.get(StaticFields.TYPE) and \
old_entity.get(StaticFields.ID) == \
new_entity.get(StaticFields.ID) and \
old_entity.get(StaticFields.NAME) == \
new_entity.get(StaticFields.NAME) and \
old_entity.get(StaticFields.STATE) == \
new_entity.get(StaticFields.STATE)

48
vitrage/tests/resources/static_datasources/changes_datasources/added_resources/static.yaml Normal file
View File

@ -0,0 +1,48 @@
metadata:
name: switch to host
description: static datasource for test
definitions:
entities:
- static_id: s1
type: switch
name: switch-1
id: 12345
state: available
- static_id: s2
type: switch
name: switch-2
id: 23456
state: available
- static_id: s3
type: switch
name: switch-3 is new!
id: 3333
state: available
- static_id: r1
type: router
name: router-1
id: 45678
- static_id: r2
type: router
name: router-2 is new!
id: 2222
state: available
- static_id: h1
type: nova.host
id: 1
relationships:
- source: s1
target: r1
relationship_type: attached
- source: s2
target: r1
relationship_type: attached
- source: s3
target: r2
relationship_type: attached
- source: r1
target: h1
relationship_type: attached
- source: r2
target: h1
relationship_type: attached

32
vitrage/tests/resources/static_datasources/changes_datasources/baseline/static.yaml Normal file
View File

@ -0,0 +1,32 @@
metadata:
name: switch to host
description: static datasource for test
definitions:
entities:
- static_id: s1
type: switch
name: switch-1
id: 12345
state: available
- static_id: s2
type: switch
name: switch-2
id: 23456
state: available
- static_id: r1
type: router
name: router-1
id: 45678
- static_id: h1
type: nova.host
id: 1
relationships:
- source: s1
target: r1
relationship_type: attached
- source: s2
target: r1
relationship_type: attached
- source: r1
target: h1
relationship_type: attached

32
vitrage/tests/resources/static_datasources/changes_datasources/changed_resources/static.yaml Normal file
View File

@ -0,0 +1,32 @@
metadata:
name: switch to host
description: static datasource for test
definitions:
entities:
- static_id: s1
type: switch
name: switch-1
id: 12345
state: error # state change
- static_id: s2
type: switch
name: switch-2
id: 23456
state: available
- static_id: r1
type: router
name: router-1 is the best! # name change
id: 45678
- static_id: h1
type: nova.host
id: 1
relationships:
- source: s1
target: r1
relationship_type: attached
- source: s2
target: r1
relationship_type: attached
- source: r1
target: h1
relationship_type: attached

18
vitrage/tests/resources/static_datasources/changes_datasources/deleted_resources/static.yaml Normal file
View File

@ -0,0 +1,18 @@
metadata:
name: switch to host
description: static datasource for test
definitions:
entities:
- static_id: s1
type: switch
name: switch-1
id: 12345
state: available
- static_id: r1
type: router
name: router-1
id: 45678
relationships:
- source: s1
target: r1
relationship_type: attached

31
vitrage/tests/resources/static_datasources/changes_datasources/mixed_changes/static.yaml Normal file
View File

@ -0,0 +1,31 @@
metadata:
name: switch to host
description: static datasource for test
definitions:
entities:
- static_id: s2
type: switch
name: switch-2
id: 23456
state: error
- static_id: r1
type: router
name: router-1
id: 45678
- static_id: r2
type: router
name: router-2 is new!
id: 222
- static_id: h1
type: nova.host
id: 1
relationships:
- source: s2
target: r1
relationship_type: attached
- source: r1
target: h1
relationship_type: attached
- source: r2
target: h1
relationship_type: attached

191
vitrage/tests/unit/datasources/static/test_static_driver.py
View File

@ -22,54 +22,22 @@ from vitrage.datasources.static import driver
from vitrage.datasources.static import STATIC_DATASOURCE
from vitrage.datasources.static import StaticFields
from vitrage.tests import base
from vitrage.tests.base import IsEmpty
from vitrage.tests.mocks import utils
class TestStaticDriver(base.BaseTest):
OPTS = [
cfg.StrOpt(DSOpts.TRANSFORMER,
default='vitrage.datasources.static.transformer.'
'StaticTransformer'),
cfg.StrOpt(DSOpts.DRIVER,
default='vitrage.datasources.static.driver.'
'StaticDriver'),
cfg.IntOpt(DSOpts.CHANGES_INTERVAL,
default=30,
min=30,
help='interval between checking changes in the '
'configuration files of the static datasources'),
cfg.StrOpt('directory',
default=utils.get_resources_dir() + '/static_datasources')
]
CHANGES_OPTS = [
cfg.StrOpt(DSOpts.TRANSFORMER,
default='vitrage.datasources.static.transformer.'
'StaticTransformer'),
cfg.StrOpt(DSOpts.DRIVER,
default='vitrage.datasources.static.driver.'
'StaticDriver'),
cfg.IntOpt(DSOpts.CHANGES_INTERVAL,
default=30,
min=30,
help='interval between checking changes in the static '
'datasources'),
cfg.StrOpt('directory',
default=utils.get_resources_dir() +
'/static_datasources/changes_datasources'),
]
CHANGES_DIR = '/changes_datasources'
# noinspection PyAttributeOutsideInit,PyPep8Naming
@classmethod
def setUpClass(cls):
super(TestStaticDriver, cls).setUpClass()
cls.conf = cfg.ConfigOpts()
cls.conf.register_opts(cls.OPTS, group=STATIC_DATASOURCE)
cls.static_driver = driver.StaticDriver(cls.conf)
cls.static_driver = driver.StaticDriver(None)
def test_get_all(self):
self._set_conf()
# Action
static_entities = self.static_driver.get_all(
DatasourceAction.INIT_SNAPSHOT)
@ -81,24 +49,108 @@ class TestStaticDriver(base.BaseTest):
self._validate_static_entity(entity)
# noinspection PyAttributeOutsideInit
def test_get_changes(self):
# Setup
entities = self.static_driver.get_all(DatasourceAction.UPDATE)
self.assertThat(entities, matchers.HasLength(8))
def test_get_changes_with_added_resources(self):
# Get initial resources
self._set_conf(self.CHANGES_DIR + '/baseline')
self.conf = cfg.ConfigOpts()
self.conf.register_opts(self.CHANGES_OPTS,
group=STATIC_DATASOURCE)
self.static_driver.cfg = self.conf
entities = self.static_driver.get_all(DatasourceAction.UPDATE)
self.assertThat(entities, matchers.HasLength(4))
# Add resources
self._set_conf(self.CHANGES_DIR + '/added_resources')
# Action
changes = self.static_driver.get_changes(
GraphAction.UPDATE_ENTITY)
# Test Assertions
self.assertThat(changes, IsEmpty())
for entity in changes:
self._validate_static_entity(entity)
expected_changes = [
{'static_id': 's3', 'type': 'switch', 'name': 'switch-3 is new!',
'id': '3333', 'state': 'available'},
{'static_id': 'r2', 'type': 'router', 'name': 'router-2 is new!',
'id': '2222', 'state': 'available'},
]
self._validate_static_changes(expected_changes, changes)
# noinspection PyAttributeOutsideInit
def test_get_changes_with_deleted_resources(self):
# Get initial resources
self._set_conf(self.CHANGES_DIR + '/baseline')
entities = self.static_driver.get_all(DatasourceAction.UPDATE)
self.assertThat(entities, matchers.HasLength(4))
# Delete resources
self._set_conf(self.CHANGES_DIR + '/deleted_resources')
# Action
changes = self.static_driver.get_changes(
GraphAction.UPDATE_ENTITY)
# Test Assertions
expected_changes = [
{'static_id': 's2', 'type': 'switch', 'name': 'switch-2',
'id': '23456', 'state': 'available',
'vitrage_event_type': 'delete_entity'},
{'static_id': 'h1', 'type': 'nova.host', 'id': '1',
'vitrage_event_type': 'delete_entity'},
]
self._validate_static_changes(expected_changes, changes)
# noinspection PyAttributeOutsideInit
def test_get_changes_with_changed_resources(self):
# Get initial resources
self._set_conf(self.CHANGES_DIR + '/baseline')
entities = self.static_driver.get_all(DatasourceAction.UPDATE)
self.assertThat(entities, matchers.HasLength(4))
# Delete resources
self._set_conf(self.CHANGES_DIR + '/changed_resources')
# Action
changes = self.static_driver.get_changes(
GraphAction.UPDATE_ENTITY)
# Test Assertions
expected_changes = [
{'static_id': 's1', 'type': 'switch', 'name': 'switch-1',
'id': '12345', 'state': 'error'},
{'static_id': 'r1', 'type': 'router',
'name': 'router-1 is the best!', 'id': '45678'},
]
self._validate_static_changes(expected_changes, changes)
# noinspection PyAttributeOutsideInit
def test_get_changes_with_mixed_changes(self):
# Get initial resources
self._set_conf(self.CHANGES_DIR + '/baseline')
entities = self.static_driver.get_all(DatasourceAction.UPDATE)
self.assertThat(entities, matchers.HasLength(4))
# Delete resources
self._set_conf(self.CHANGES_DIR + '/mixed_changes')
# Action
changes = self.static_driver.get_changes(
GraphAction.UPDATE_ENTITY)
# Test Assertions
expected_changes = [
{'static_id': 's1', 'type': 'switch', 'name': 'switch-1',
'id': '12345', 'state': 'available',
'vitrage_event_type': 'delete_entity'},
{'static_id': 's2', 'type': 'switch', 'name': 'switch-2',
'id': '23456', 'state': 'error'},
{'static_id': 'r2', 'type': 'router', 'name': 'router-2 is new!',
'id': '222'},
]
self._validate_static_changes(expected_changes, changes)
def _validate_static_entity(self, entity):
self.assertIsInstance(entity[StaticFields.METADATA], dict)
@ -115,3 +167,48 @@ class TestStaticDriver(base.BaseTest):
and entity[StaticFields.STATIC_ID] == rel[StaticFields.SOURCE]
or entity[StaticFields.STATIC_ID] == rel[StaticFields.SOURCE]
and entity[StaticFields.STATIC_ID] == rel[StaticFields.TARGET])
def _validate_static_changes(self, expected_changes, changes):
self.assertThat(changes, matchers.HasLength(len(expected_changes)))
for entity in changes:
self._validate_static_entity(entity)
for expected_change in expected_changes:
found = False
for change in changes:
if change[StaticFields.TYPE] == \
expected_change[StaticFields.TYPE] and \
change[StaticFields.ID] == \
expected_change[StaticFields.ID]:
found = True
self.assertEqual(expected_change.get('vitrage_event_type'),
change.get('vitrage_event_type'))
self.assertEqual(expected_change.get(StaticFields.NAME),
change.get(StaticFields.NAME))
self.assertEqual(expected_change.get(StaticFields.STATE),
change.get(StaticFields.STATE))
self.assertTrue(found)
def _set_conf(self, sub_dir=None):
default_dir = utils.get_resources_dir() + \
'/static_datasources' + (sub_dir if sub_dir else '')
opts = [
cfg.StrOpt(DSOpts.TRANSFORMER,
default='vitrage.datasources.static.transformer.'
'StaticTransformer'),
cfg.StrOpt(DSOpts.DRIVER,
default='vitrage.datasources.static.driver.'
'StaticDriver'),
cfg.IntOpt(DSOpts.CHANGES_INTERVAL,
default=30,
min=30,
help='interval between checking changes in the static '
'datasources'),
cfg.StrOpt('directory', default=default_dir),
]
self.conf = cfg.ConfigOpts()
self.conf.register_opts(opts, group=STATIC_DATASOURCE)
self.static_driver.cfg = self.conf