openstack/vitrage
Code Issues Proposed changes

consistency initializing process

Browse Source 
implements: blueprint entity-graph-consistency-validator

Change-Id: Ib012bd71353b10e35b680014972aaa825509f63c
This commit is contained in:
Alexey Weyl 2016-02-18 11:29:08 +02:00
parent 1e6bacf0fd
commit 7df932bc36
24 changed files with 484 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
etc/vitrage/alarms.sample.json
View File

@ -2,34 +2,58 @@
"alarms":
[
{
"id": "04cf683b-58a8-4b59-941a-9a1594fa0fe7",
"project_id": "da3a1ab32-1c62-22cb-bf04-660bd33cd74d",
"state": "available",
"update_timestamp": "2016-01-18T06:14:20.782134+00:00",
"category": "alarm",
"type": "CPU_HIGH",
"name": "cpu high",
"severity": "major"
"category": "ALARM",
"type": "nagios",
"name": "CPU load",
"state": "Active",
"severity": "WARNING",
"timestamp": "2015-12-01T12:46:41Z",
"info": "WARNING - 15min load 1.66 at 32 CPUs",
"resource_type": "nova.host",
"resource_name": "host-0",
"resource_id": "host-0",
"id": 0,
"vitrage_id": "ALARM:nagios:host0:CPU load"
},
{
"id": "05af123a-56a2-4b59-741a-7b1482dabac2",
"project_id": "da3a1ab32-1c62-22cb-bf04-660bd33cd74d",
"state": "available",
"update_timestamp": "2016-01-18T06:12:28.987651+00:00",
"category": "alarm",
"type": "NO_SPACE_ON_DISK",
"name": "no space on disk",
"severity": "critical"
"category": "ALARM",
"type": "vitrage",
"name": "Machine Suboptimal",
"state": "Active",
"severity": "WARNING",
"timestamp": "2015-12-01T12:46:41Z",
"resource_type": "nova.instance",
"resource_name": "vm0",
"resource_id": "20d12a8a-ea9a-89c6-5947-83bea959362e",
"id": 1,
"vitrage_id": "ALARM:vitrage:vm0:Machine Suboptimal"
},
{
"id": "123f456a-12b2-9c51-345d-8b2604adbada7",
"project_id": "da3a1ab32-1c62-22cb-bf04-660bd33cd74d",
"state": "available",
"update_timestamp": "2016-01-18T06:14:38.123456+00:00",
"category": "alarm",
"type": "OUT_OF_MEMORY",
"name": "out of memory",
"severity": "critical"
"category": "ALARM",
"type": "vitrage",
"name": "Machine Suboptimal",
"state": "Active",
"severity": "WARNING",
"timestamp": "2015-12-01T12:46:41Z",
"resource_type": "nova.instance",
"resource_name": "vm1",
"resource_id": "275097cf-954e-8e24-b185-9514e24b8591",
"id": 2,
"vitrage_id": "ALARM:vitrage:vm1:Machine Suboptimal"
},
{
"category": "ALARM",
"type": "aodh",
"name": "Memory overload",
"state": "Active",
"severity": "WARNING",
"timestamp": "2015-12-01T12:46:41Z",
"info": "WARNING - 15min load 1.66 at 32 CPUs",
"resource_type": "nova.host",
"resource_name": "host-0",
"resource_id": "host-0",
"id": 3,
"vitrage_id": "ALARM:aodh:host0:Memory overload"
}
]
}

66
etc/vitrage/graph.sample.json
View File

@ -11,7 +11,8 @@
"state": "ACTIVE",
"project_id": "0683517e1e354d2ba25cba6937f44e79",
"type": "nova.instance",
"id": "RESOURCE:nova.instance:20d12a8a-ea9a-89c6-5947-83bea959362e"
"id": "20d12a8a-ea9a-89c6-5947-83bea959362e",
"vitrage_id": "RESOURCE:nova.instance:20d12a8a-ea9a-89c6-5947-83bea959362e"
},
{
"category": "RESOURCE",
@ -22,7 +23,8 @@
"state": "ACTIVE",
"project_id": "0683517e1e354d2ba25cba6937f44e79",
"type": "nova.instance",
"id": "RESOURCE:nova.instance:dc35fa2f-4515-1653-ef6b-03b471bb395b"
"id": "dc35fa2f-4515-1653-ef6b-03b471bb395b",
"vitrage_id": "RESOURCE:nova.instance:dc35fa2f-4515-1653-ef6b-03b471bb395b"
},
{
"category": "RESOURCE",
@ -33,7 +35,8 @@
"state": "ACTIVE",
"project_id": "0683517e1e354d2ba25cba6937f44e79",
"type": "nova.instance",
"id": "RESOURCE:nova.instance:9879cf5a-bdcf-3651-3017-961ed887ec86"
"id": "9879cf5a-bdcf-3651-3017-961ed887ec86",
"vitrage_id": "RESOURCE:nova.instance:9879cf5a-bdcf-3651-3017-961ed887ec86"
},
{
"category": "RESOURCE",
@ -44,7 +47,8 @@
"state": "ACTIVE",
"project_id": "0683517e1e354d2ba25cba6937f44e79",
"type": "nova.instance",
"id": "RESOURCE:nova.instance:fe124f4b-9ed7-4591-fcd1-803cf5c33cb1"
"id": "fe124f4b-9ed7-4591-fcd1-803cf5c33cb1",
"vitrage_id": "RESOURCE:nova.instance:fe124f4b-9ed7-4591-fcd1-803cf5c33cb1"
},
{
"category": "RESOURCE",
@ -55,7 +59,8 @@
"state": "ACTIVE",
"project_id": "0683517e1e354d2ba25cba6937f44e79",
"type": "nova.instance",
"id": "RESOURCE:nova.instance:f2e48a97-7350-061e-12d3-84c6dc3e67c0"
"id": "f2e48a97-7350-061e-12d3-84c6dc3e67c0",
"vitrage_id": "RESOURCE:nova.instance:f2e48a97-7350-061e-12d3-84c6dc3e67c0"
},
{
"category": "RESOURCE",
@ -65,7 +70,8 @@
"update_timestamp": "2015-12-01T12:46:41Z",
"state": "available",
"type": "nova.host",
"id": "RESOURCE:nova.host:host-2"
"id": "host-2",
"vitrage_id": "RESOURCE:nova.host:host-2"
},
{
"category": "RESOURCE",
@ -75,7 +81,8 @@
"update_timestamp": "2015-12-01T12:46:41Z",
"state": "available",
"type": "nova.host",
"id": "RESOURCE:nova.host:host-3"
"id": "host-3",
"vitrage_id": "RESOURCE:nova.host:host-3"
},
{
"category": "RESOURCE",
@ -85,7 +92,8 @@
"update_timestamp": "2015-12-01T12:46:41Z",
"state": "available",
"type": "nova.host",
"id": "RESOURCE:nova.host:host-0"
"id": "host-0",
"vitrage_id": "RESOURCE:nova.host:host-0"
},
{
"category": "RESOURCE",
@ -95,7 +103,8 @@
"update_timestamp": "2015-12-01T12:46:41Z",
"state": "available",
"type": "nova.host",
"id": "RESOURCE:nova.host:host-1"
"id": "host-1",
"vitrage_id": "RESOURCE:nova.host:host-1"
},
{
"category": "RESOURCE",
@ -106,7 +115,8 @@
"state": "ACTIVE",
"project_id": "0683517e1e354d2ba25cba6937f44e79",
"type": "nova.instance",
"id": "RESOURCE:nova.instance:275097cf-954e-8e24-b185-9514e24b8591"
"id": "275097cf-954e-8e24-b185-9514e24b8591",
"vitrage_id": "RESOURCE:nova.instance:275097cf-954e-8e24-b185-9514e24b8591"
},
{
"category": "RESOURCE",
@ -117,7 +127,8 @@
"state": "ACTIVE",
"project_id": "0683517e1e354d2ba25cba6937f44e79",
"type": "nova.instance",
"id": "RESOURCE:nova.instance:a0f0805f-c804-cffe-c25a-1b38f555ed68"
"id": "a0f0805f-c804-cffe-c25a-1b38f555ed68",
"vitrage_id": "RESOURCE:nova.instance:a0f0805f-c804-cffe-c25a-1b38f555ed68"
},
{
"category": "RESOURCE",
@ -128,7 +139,8 @@
"state": "ACTIVE",
"project_id": "0683517e1e354d2ba25cba6937f44e79",
"type": "nova.instance",
"id": "RESOURCE:nova.instance:56af57d2-34a4-19b1-5106-b613637a11a7"
"id": "56af57d2-34a4-19b1-5106-b613637a11a7",
"vitrage_id": "RESOURCE:nova.instance:56af57d2-34a4-19b1-5106-b613637a11a7"
},
{
"category": "RESOURCE",
@ -138,7 +150,8 @@
"update_timestamp": "2015-12-01T12:46:41Z",
"state": "available",
"type": "nova.zone",
"id": "RESOURCE:nova.zone:zone-1"
"id": "zone-1",
"vitrage_id": "RESOURCE:nova.zone:zone-1"
},
{
"category": "RESOURCE",
@ -149,7 +162,8 @@
"state": "ACTIVE",
"project_id": "0683517e1e354d2ba25cba6937f44e79",
"type": "nova.instance",
"id": "RESOURCE:nova.instance:16e14c58-d254-2bec-53e4-c766e48810aa"
"id": "16e14c58-d254-2bec-53e4-c766e48810aa",
"vitrage_id": "RESOURCE:nova.instance:16e14c58-d254-2bec-53e4-c766e48810aa"
},
{
"category": "RESOURCE",
@ -160,7 +174,8 @@
"state": "ACTIVE",
"project_id": "0683517e1e354d2ba25cba6937f44e79",
"type": "nova.instance",
"id": "RESOURCE:nova.instance:f35a1e10-74ff-7332-8edf-83cd6ffcb2de"
"id": "f35a1e10-74ff-7332-8edf-83cd6ffcb2de",
"vitrage_id": "RESOURCE:nova.instance:f35a1e10-74ff-7332-8edf-83cd6ffcb2de"
},
{
"category": "RESOURCE",
@ -171,7 +186,8 @@
"state": "ACTIVE",
"project_id": "0683517e1e354d2ba25cba6937f44e79",
"type": "nova.instance",
"id": "RESOURCE:nova.instance:ea8a450e-cab1-2272-f431-494b40c5c378"
"id": "ea8a450e-cab1-2272-f431-494b40c5c378",
"vitrage_id": "RESOURCE:nova.instance:ea8a450e-cab1-2272-f431-494b40c5c378"
},
{
"category": "RESOURCE",
@ -182,7 +198,8 @@
"state": "ACTIVE",
"project_id": "0683517e1e354d2ba25cba6937f44e79",
"type": "nova.instance",
"id": "RESOURCE:nova.instance:6e42bdc3-b776-1b2c-2c7d-b7a8bb98f721"
"id": "6e42bdc3-b776-1b2c-2c7d-b7a8bb98f721",
"vitrage_id": "RESOURCE:nova.instance:6e42bdc3-b776-1b2c-2c7d-b7a8bb98f721"
},
{
"category": "RESOURCE",
@ -193,7 +210,8 @@
"state": "ACTIVE",
"project_id": "0683517e1e354d2ba25cba6937f44e79",
"type": "nova.instance",
"id": "RESOURCE:nova.instance:8c951613-c660-87c0-c18b-0fa3293ce8d8"
"id": "8c951613-c660-87c0-c18b-0fa3293ce8d8",
"vitrage_id": "RESOURCE:nova.instance:8c951613-c660-87c0-c18b-0fa3293ce8d8"
},
{
"category": "RESOURCE",
@ -203,7 +221,8 @@
"update_timestamp": "2015-12-01T12:46:41Z",
"state": "available",
"type": "nova.zone",
"id": "RESOURCE:nova.zone:zone-0"
"id": "zone-0",
"vitrage_id": "RESOURCE:nova.zone:zone-0"
},
{
"category": "RESOURCE",
@ -214,7 +233,8 @@
"state": "ACTIVE",
"project_id": "0683517e1e354d2ba25cba6937f44e79",
"type": "nova.instance",
"id": "RESOURCE:nova.instance:78353ce4-2710-49b5-1341-b8cbb6000ebc"
"id": "78353ce4-2710-49b5-1341-b8cbb6000ebc",
"vitrage_id": "RESOURCE:nova.instance:78353ce4-2710-49b5-1341-b8cbb6000ebc"
},
{
"category": "RESOURCE",
@ -225,7 +245,8 @@
"state": "ACTIVE",
"project_id": "0683517e1e354d2ba25cba6937f44e79",
"type": "nova.instance",
"id": "RESOURCE:nova.instance:35bf479a-75d9-80a9-874e-d3b50fb2dd2e"
"id": "35bf479a-75d9-80a9-874e-d3b50fb2dd2e",
"vitrage_id": "RESOURCE:nova.instance:35bf479a-75d9-80a9-874e-d3b50fb2dd2e"
},
{
"category": "RESOURCE",
@ -233,7 +254,8 @@
"is_deleted": false,
"name": "node",
"type": "node",
"id": "RESOURCE:node"
"id": "node",
"vitrage_id": "RESOURCE:node"
}
],
"links": [

20
etc/vitrage/rca.sample.json
View File

@ -1,7 +1,7 @@
{
"directed": true,
"graph": {
},
"nodes": [
{
@ -13,30 +13,36 @@
"timestamp": "2015-12-01T12:46:41Z",
"info": "WARNING - 15min load 1.66 at 32 CPUs",
"resource_type": "nova.host",
"resource_name": "host0",
"id": 0
"resource_name": "host-0",
"resource_id": "host-0",
"id": 0,
"vitrage_id": "ALARM:nagios:host0:CPU load"
},
{
"category": "ALARM",
"type": "vitrage",
"name": "Machine might be suffering due to high CPU load on the host",
"name": "Machine Suboptimal",
"state": "Active",
"severity": "WARNING",
"timestamp": "2015-12-01T12:46:41Z",
"resource_type": "nova.instance",
"resource_name": "vm0",
"id": 1
"resource_id": "20d12a8a-ea9a-89c6-5947-83bea959362e",
"id": 1,
"vitrage_id": "ALARM:vitrage:vm0:Machine Suboptimal"
},
{
"category": "ALARM",
"type": "vitrage",
"name": "Machine might be suffering due to high CPU load on the host",
"name": "Machine Suboptimal",
"state": "Active",
"severity": "WARNING",
"timestamp": "2015-12-01T12:46:41Z",
"resource_type": "nova.instance",
"resource_name": "vm1",
"id": 2
"resource_id": "275097cf-954e-8e24-b185-9514e24b8591",
"id": 2,
"vitrage_id": "ALARM:vitrage:vm1:Machine Suboptimal"
}
],
"links": [

6
vitrage/cmd/graph.py
View File

@ -20,6 +20,7 @@ from oslo_service import service as os_service
from vitrage.entity_graph.api_handler import service as api_handler_svc
from vitrage.entity_graph.consistency import service as consistency_svc
from vitrage.entity_graph.initialization_status import InitializationStatus
from vitrage.entity_graph.processor import entity_graph
from vitrage.entity_graph import service as entity_graph_svc
from vitrage import service
@ -38,12 +39,13 @@ def main():
e_graph = entity_graph.EntityGraph("Entity Graph")
event_queue = multiprocessing.Queue()
conf = service.prepare_service()
initialization_status = InitializationStatus()
launcher = os_service.ServiceLauncher(conf)
synchronizer = synchronizer_launcher.Launcher(
conf, synchronizer_launcher.create_send_to_queue_callback(event_queue))
launcher.launch_service(entity_graph_svc.VitrageGraphService(
event_queue, e_graph))
event_queue, e_graph, initialization_status))
launcher.launch_service(api_handler_svc.VitrageApiHandlerService(
e_graph))
@ -51,7 +53,7 @@ def main():
synchronizer.launch()
launcher.launch_service(consistency_svc.VitrageGraphConsistencyService(
conf, e_graph))
conf, e_graph, initialization_status))
launcher.wait()

1
vitrage/common/constants.py
View File

@ -64,6 +64,7 @@ class EntityType(object):
NOVA_ZONE = 'nova.zone'
SWITCH = 'switch'
NAGIOS = 'nagios'
VITRAGE = 'vitrage'
class EventAction(object):

6
vitrage/entity_graph/consistency/__init__.py
View File

@ -24,4 +24,10 @@ OPTS = [
default=60,
min=60,
help='minimum time until deleting entity (in seconds)'),
cfg.IntOpt('consistency_initialization_interval',
default=5,
min=1,
help='interval between consistency initialization checks for '
'finding old deduced alarms after initialization '
'(in seconds)'),
]

40
vitrage/entity_graph/consistency/consistency_enforcer.py
View File

@ -27,12 +27,30 @@ LOG = log.getLogger(__name__)
class ConsistencyEnforcer(object):
def __init__(self, cfg, entity_graph):
def __init__(self, cfg, entity_graph, initialization_status):
self.cfg = cfg
self.graph = entity_graph
self.initialization_status = initialization_status
self.notifier = None
def starting_process(self):
pass
def initializing_process(self):
try:
LOG.debug('Started consistency starting check')
if self.initialization_status.is_received_all_end_messages():
LOG.debug('All end messages were received')
timestamp = utcnow()
all_vertices = self.graph.get_vertices()
for vertex in all_vertices:
self.run_evaluator(vertex)
self._notify_deletion_of_deduced_alarms(timestamp)
self.initialization_status.status = \
self.initialization_status.FINISHED
except Exception:
LOG.error("Error in deleting vertices from entity_graph: %s",
traceback.print_exc())
def periodic_process(self):
try:
@ -74,6 +92,22 @@ class ConsistencyEnforcer(object):
return set(self._filter_vertices_to_be_deleted(vertices))
def _find_old_deduced_alarms(self, timestamp):
query = {
'and': [
{'==': {VProps.CATEGORY: EntityCategory.ALARM}},
{'==': {VProps.TYPE: EntityType.VITRAGE}},
{'>=': {VProps.UPDATE_TIMESTAMP: timestamp}}
]
}
return self.graph.get_vertices(query_dict=query)
def _notify_deletion_of_deduced_alarms(self, timestamp):
old_deduced_alarms = self._find_old_deduced_alarms(timestamp)
for vertex in old_deduced_alarms:
# TODO(Alexey): use notifier to inform aodh
self.notifier(vertex)
def _delete_vertices(self, vertices):
for vertex in vertices:
self.graph.remove_vertex(vertex)

12
vitrage/entity_graph/consistency/service.py
View File

@ -23,20 +23,28 @@ LOG = log.getLogger(__name__)
class VitrageGraphConsistencyService(os_service.Service):
def __init__(self, conf, entity_graph):
def __init__(self, conf, entity_graph, initialization_status):
super(VitrageGraphConsistencyService, self).__init__()
self.cfg = conf
self.entity_graph = entity_graph
self.initialization_status = initialization_status
def start(self):
LOG.info("Start VitrageGraphConsistencyService")
super(VitrageGraphConsistencyService, self).start()
consistency_enf = ConsistencyEnforcer(self.cfg, self.entity_graph)
consistency_enf = ConsistencyEnforcer(self.cfg,
self.entity_graph,
self.initialization_status)
self.tg.add_timer(self.cfg.consistency.consistency_interval,
consistency_enf.periodic_process)
# TODO(Alexey): uncomment this when evaluator is ready
# self.tg.add_timer(self.cfg.consistency.
# consistency_initialization_interval,
# consistency_enf.initializing_process)
LOG.info("Finish start VitrageGraphConsistencyService")
def stop(self):

29
vitrage/entity_graph/initialization_status.py Normal file
View File

@ -0,0 +1,29 @@
# Copyright 2016 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
class InitializationStatus(object):
STARTED = 'started'
RECEIVED_ALL_END_MESSAGES = 'received_all_end_messages'
FINISHED = 'finished'
def __init__(self):
self.status = self.STARTED
self.end_messages = {}
def is_initialization_finished(self):
return self.status == self.FINISHED
def is_received_all_end_messages(self):
return self.status == self.RECEIVED_ALL_END_MESSAGES

18
vitrage/entity_graph/processor/processor.py
View File

@ -15,6 +15,7 @@
from oslo_log import log
from vitrage.common.constants import EventAction
from vitrage.common.constants import VertexProperties as VProps
from vitrage.entity_graph.processor import base as processor
from vitrage.entity_graph.processor import entity_graph
from vitrage.entity_graph import transformer_manager
@ -25,13 +26,14 @@ LOG = log.getLogger(__name__)
class Processor(processor.ProcessorBase):
def __init__(self, e_graph=None):
NUMBER_OF_PLUGINS = 5
def __init__(self, initialization_status, e_graph=None):
self.transformer = transformer_manager.TransformerManager()
self._initialize_events_actions()
if e_graph is None:
self.entity_graph = entity_graph.EntityGraph("Entity Graph")
else:
self.entity_graph = e_graph
self.initialization_status = initialization_status
self.entity_graph = entity_graph.EntityGraph("Entity Graph") if \
e_graph is None else e_graph
def process_event(self, event):
"""Decides which action to run on given event
@ -126,7 +128,11 @@ class Processor(processor.ProcessorBase):
deleted_vertex)
def handle_end_message(self, vertex, neighbors):
return
self.initialization_status.end_messages[vertex[VProps.TYPE]] = True
if len(self.initialization_status.end_messages) == \
self.NUMBER_OF_PLUGINS:
self.initialization_status.status = \
self.initialization_status.RECEIVED_ALL_END_MESSAGES
def transform_entity(self, event):
return self.transformer.transform(event)

11
vitrage/entity_graph/service.py
View File

@ -13,22 +13,23 @@
# under the License.
import datetime
import traceback
from oslo_log import log
from oslo_service import service as os_service
from vitrage.entity_graph.processor import processor as proc
LOG = log.getLogger(__name__)
class VitrageGraphService(os_service.Service):
def __init__(self, event_queue, entity_graph):
def __init__(self, event_queue, entity_graph, initialization_status):
super(VitrageGraphService, self).__init__()
self.queue = event_queue
self.processor = proc.Processor(e_graph=entity_graph)
self.processor = proc.Processor(initialization_status,
e_graph=entity_graph)
def start(self):
LOG.info("Start VitrageGraphService")
@ -72,5 +73,5 @@ class VitrageGraphService(os_service.Service):
event = self.queue.get()
LOG.debug("got event: %s" % event)
self.processor.process_event(event)
except Exception as error:
LOG.error("Exception: %s", error)
except Exception:
LOG.error("Exception: %s", traceback.print_exc())

26
vitrage/entity_graph/transformer_manager.py
View File

@ -16,9 +16,7 @@ from oslo_log import log as logging
from oslo_utils import importutils
from vitrage.common.constants import EntityType
from vitrage.common.constants import EventAction
from vitrage.common.constants import SynchronizerProperties as SyncProps
from vitrage.common.constants import SyncMode
from vitrage.common.exception import VitrageTransformerError
from vitrage.synchronizer.plugins.nagios.transformer import NagiosTransformer
from vitrage.synchronizer.plugins.nova.host.transformer import HostTransformer
@ -27,7 +25,6 @@ from vitrage.synchronizer.plugins.nova.instance.transformer import \
from vitrage.synchronizer.plugins.nova.zone.transformer import ZoneTransformer
from vitrage.synchronizer.plugins.static_physical.transformer import \
StaticPhysicalTransformer
from vitrage.synchronizer.plugins import transformer_base
LOG = logging.getLogger(__name__)
@ -78,18 +75,13 @@ class TransformerManager(object):
return transformer
def transform(self, entity_event):
if not self._is_end_message(entity_event):
try:
sync_type = entity_event[SyncProps.SYNC_TYPE]
except KeyError:
raise VitrageTransformerError(
'Entity Event must contains sync_type field.')
try:
sync_type = entity_event[SyncProps.SYNC_TYPE]
except KeyError:
raise VitrageTransformerError(
'Entity Event must contains sync_type field.')
return self.get_transformer(sync_type).transform(entity_event)
else:
return transformer_base.EntityWrapper(None,
None,
EventAction.END_MESSAGE)
return self.get_transformer(sync_type).transform(entity_event)
def extract_key(self, entity_event):
@ -100,9 +92,3 @@ class TransformerManager(object):
'Entity Event must contains sync_type field.')
return self.get_transformer(sync_type).extract_key()
@staticmethod
def _is_end_message(entity_event):
return entity_event[SyncProps.SYNC_MODE] == SyncMode.INIT_SNAPSHOT and\
SyncProps.EVENT_TYPE in entity_event and \
entity_event[SyncProps.EVENT_TYPE] == EventAction.END_MESSAGE

4
vitrage/graph/__init__.py
View File

@ -23,7 +23,7 @@ from networkx_algorithm import * # noqa
from utils import * # noqa
def create_graph(name):
def create_graph(name, root_id=None):
"""Create a Graph instance
For now only return NXGraph
@ -31,7 +31,7 @@ def create_graph(name):
:type name: str
:rtype: Graph
"""
return NXGraph(name)
return NXGraph(name, root_id)
def create_algorithm(graph):

6
vitrage/synchronizer/plugins/static_physical/synchronizer.py
View File

@ -14,6 +14,7 @@
import os
from vitrage.common.constants import EntityType
from vitrage.common.constants import EventAction
from vitrage.common.constants import SynchronizerProperties as SyncProps
@ -23,7 +24,6 @@ from vitrage.synchronizer.plugins.synchronizer_base import SynchronizerBase
class StaticPhysicalSynchronizer(SynchronizerBase):
STATIC_PHYSICAL = 'static_physical'
ENTITIES_SECTION = 'entities'
def __init__(self, conf):
@ -33,12 +33,12 @@ class StaticPhysicalSynchronizer(SynchronizerBase):
def get_all(self, sync_mode):
return self.make_pickleable(self._get_all_entities(),
self.STATIC_PHYSICAL,
EntityType.SWITCH,
sync_mode)
def get_changes(self, sync_mode):
return self.make_pickleable(self._get_changes_entities(),
self.STATIC_PHYSICAL,
EntityType.SWITCH,
sync_mode)
def _get_all_entities(self):

23
vitrage/synchronizer/plugins/static_physical/transformer.py
View File

@ -16,8 +16,11 @@ from oslo_log import log as logging
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import EntityType
from vitrage.common.constants import EventAction
from vitrage.common.constants import SynchronizerProperties as SyncProps
from vitrage.common.constants import SyncMode
from vitrage.common.constants import VertexProperties as VProps
from vitrage.common.exception import VitrageTransformerError
import vitrage.graph.utils as graph_utils
from vitrage.synchronizer.plugins import transformer_base
@ -94,6 +97,26 @@ class StaticPhysicalTransformer(transformer_base.TransformerBase):
LOG.warning('Cannot find zone transformer')
return None
def _extract_action_type(self, entity_event):
sync_mode = entity_event[SyncProps.SYNC_MODE]
if SyncMode.INIT_SNAPSHOT == sync_mode:
return EventAction.CREATE
if SyncMode.SNAPSHOT == sync_mode:
return EventAction.UPDATE
if SyncMode.UPDATE == sync_mode:
if SyncProps.EVENT_TYPE in entity_event:
sync_type = entity_event[SyncProps.EVENT_TYPE]
return EventAction.DELETE if sync_type == EventAction.DELETE \
else EventAction.UPDATE
else:
return EventAction.UPDATE
raise VitrageTransformerError(
'Invalid sync mode: (%s)' % sync_mode)
def extract_key(self, entity_event):
entity_id = entity_event[VProps.ID]
sync_type = entity_event[SyncProps.SYNC_TYPE]

30
vitrage/synchronizer/plugins/transformer_base.py
View File

@ -1,4 +1,5 @@
# Copyright 2015 - Alcatel-Lucent
# Copyright 2016 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -20,7 +21,9 @@ import six
import vitrage.common.constants as cons
from vitrage.common.constants import EntityType
from vitrage.common.constants import EventAction
from vitrage.common.constants import SynchronizerProperties as SyncProps
from vitrage.common.constants import SyncMode
from vitrage.common.exception import VitrageTransformerError
import vitrage.graph.utils as graph_utils
@ -84,11 +87,17 @@ class TransformerBase(object):
:return: entity wrapper
:rtype:EntityWrapper
"""
entity_vertex = self._create_entity_vertex(entity_event)
neighbors = self._create_neighbors(entity_event)
action = self._extract_action_type(entity_event)
return EntityWrapper(entity_vertex, neighbors, action)
if not self._is_end_message(entity_event):
entity_vertex = self._create_entity_vertex(entity_event)
neighbors = self._create_neighbors(entity_event)
action = self._extract_action_type(entity_event)
return EntityWrapper(entity_vertex, neighbors, action)
else:
return EntityWrapper(self._create_end_vertex(entity_event),
None,
EventAction.END_MESSAGE)
@abc.abstractmethod
def _create_entity_vertex(self, entity_event):
@ -163,3 +172,16 @@ class TransformerBase(object):
raise VitrageTransformerError(
'Invalid sync mode: (%s)' % sync_mode)
@staticmethod
def _create_end_vertex(entity_event):
sync_type = entity_event[SyncProps.SYNC_TYPE]
return graph_utils.create_vertex(
'END_MESSAGE:' + sync_type,
entity_type=sync_type)
@staticmethod
def _is_end_message(entity_event):
return entity_event[SyncProps.SYNC_MODE] == SyncMode.INIT_SNAPSHOT and\
SyncProps.EVENT_TYPE in entity_event and \
entity_event[SyncProps.EVENT_TYPE] == EventAction.END_MESSAGE

0
vitrage/tests/unit/entity_graph/consistency/__init__.py → vitrage/tests/functional/entity_graph/__init__.py
View File

15
vitrage/tests/functional/entity_graph/consistency/__init__.py Normal file
View File

@ -0,0 +1,15 @@
# Copyright 2016 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
__author__ = 'stack'

172
vitrage/tests/functional/entity_graph/consistency/test_consistency.py Normal file
View File

@ -0,0 +1,172 @@
# Copyright 2016 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from datetime import timedelta
import time
import unittest
from oslo_config import cfg
from vitrage.common.constants import EdgeLabels
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import EntityType
from vitrage.common.constants import VertexProperties as VProps
from vitrage.common.datetime_utils import utcnow
from vitrage.entity_graph.consistency.consistency_enforcer \
import ConsistencyEnforcer
from vitrage.entity_graph.initialization_status import InitializationStatus
from vitrage.entity_graph.processor.processor import Processor
import vitrage.graph.utils as graph_utils
from vitrage.tests.unit.entity_graph import TestEntityGraphBase
class TestConsistencyBase(TestEntityGraphBase):
OPTS = [
cfg.IntOpt('consistency_interval',
default=1,
min=1),
cfg.IntOpt('min_time_to_delete',
default=1,
min=1),
]
def setUp(self):
super(TestConsistencyBase, self).setUp()
self.initialization_status = InitializationStatus()
self.processor = Processor(self.initialization_status)
self.conf = cfg.ConfigOpts()
self.conf.register_opts(self.OPTS, group='consistency')
self.consistency_enforcer = ConsistencyEnforcer(
self.conf, self.processor.entity_graph, self.initialization_status)
# TODO(Alexey): unskip this test when evaluator is ready
@unittest.skip("testing skipping")
def test_initializing_process(self):
# Setup
num_external_alarms = self.NUM_HOSTS - 2
num_instances_per_host = 4
self._create_processor_with_graph(processor=self.processor)
self._add_alarms()
self._set_end_messages()
self.assertEqual(self._num_total_expected_vertices() +
num_external_alarms + self.NUM_INSTANCES,
len(self.processor.entity_graph.get_vertices()))
# Action
self.consistency_enforcer.initializing_process()
# Test Assertions
num_correct_alarms = num_external_alarms + \
num_external_alarms * num_instances_per_host
self.assertEqual(self._num_total_expected_vertices() +
num_correct_alarms,
len(self.processor.entity_graph.get_vertices()))
instance_vertices = self.processor.entity_graph.get_vertices({
VProps.CATEGORY: EntityCategory.ALARM
})
self.assertEqual(num_correct_alarms, len(instance_vertices))
instance_vertices = self.processor.entity_graph.get_vertices({
VProps.CATEGORY: EntityCategory.ALARM,
VProps.TYPE: EntityType.VITRAGE
})
self.assertEqual(num_external_alarms * num_instances_per_host,
len(instance_vertices))
def test_periodic_process(self):
# Setup
consistency_inteval = self.conf.consistency.consistency_interval
self._periodic_process_setup_stage(consistency_inteval)
# Action
time.sleep(2 * consistency_inteval + 1)
self.consistency_enforcer.periodic_process()
# Test Assertions
instance_vertices = self.processor.entity_graph.get_vertices({
VProps.CATEGORY: EntityCategory.RESOURCE,
VProps.TYPE: EntityType.NOVA_INSTANCE
})
self.assertEqual(self.NUM_INSTANCES - 6, len(instance_vertices))
self.assertEqual(self._num_total_expected_vertices() - 6,
len(self.processor.entity_graph.get_vertices()))
def _periodic_process_setup_stage(self, consistency_inteval):
self._create_processor_with_graph(processor=self.processor)
current_time = utcnow()
# set all vertices to be have timestamp that consistency won't get
self._update_timestamp(self.processor.entity_graph.get_vertices(),
current_time +
timedelta(seconds=1.5 * consistency_inteval))
# check number of instances in graph
instance_vertices = self.processor.entity_graph.get_vertices({
VProps.CATEGORY: EntityCategory.RESOURCE,
VProps.TYPE: EntityType.NOVA_INSTANCE
})
self.assertEqual(self.NUM_INSTANCES, len(instance_vertices))
# set current timestamp of part of the instances
self._update_timestamp(instance_vertices[0:3], current_time)
# set part of the instances as deleted + update to current timestamp
for i in range(3, 6):
instance_vertices[i][VProps.IS_DELETED] = True
self.processor.entity_graph.update_vertex(instance_vertices[i])
# set part of the instances as deleted
for i in range(6, 9):
instance_vertices[i][VProps.IS_DELETED] = True
instance_vertices[i][VProps.UPDATE_TIMESTAMP] = str(
current_time + timedelta(seconds=2 * consistency_inteval + 1))
self.processor.entity_graph.update_vertex(instance_vertices[i])
def _set_end_messages(self):
self.initialization_status.end_messages[EntityType.NOVA_ZONE] = True
self.initialization_status.end_messages[EntityType.NOVA_HOST] = True
self.initialization_status.end_messages[EntityType.NOVA_INSTANCE] = \
True
self.initialization_status.end_messages[EntityType.NAGIOS] = True
self.initialization_status.status = \
self.initialization_status.RECEIVED_ALL_END_MESSAGES
def _add_alarms(self):
# find hosts and instances
host_vertices = self.processor.entity_graph.get_vertices({
VProps.CATEGORY: EntityCategory.RESOURCE,
VProps.TYPE: EntityType.NOVA_HOST
})
# add external alarms + deduced alarms
for host_vertex in host_vertices:
alarm_vertex = self._create_alarm('external_alarm',
EntityType.NAGIOS)
self.processor.entity_graph.add_vertex(alarm_vertex)
edge = graph_utils.create_edge(alarm_vertex.vertex_id,
host_vertex.vertex_id,
EdgeLabels.ON)
self.processor.entity_graph.add_edge(edge)
self.run_evaluator(alarm_vertex)
# remove external alarms
self.processor.entity_graph.remove_vertex(host_vertices[2])
self.processor.entity_graph.remove_vertex(host_vertices[3])
def _update_timestamp(self, lst, timestamp):
for vertex in lst:
vertex[VProps.UPDATE_TIMESTAMP] = str(timestamp)
self.processor.entity_graph.update_vertex(vertex)

23
vitrage/tests/unit/entity_graph/__init__.py
View File

@ -12,12 +12,14 @@
# License for the specific language governing permissions and limitations
# under the License.
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import SynchronizerProperties as SyncProps
from vitrage.common.constants import SyncMode
from vitrage.common.datetime_utils import utcnow
from vitrage.entity_graph.initialization_status import InitializationStatus
from vitrage.entity_graph.processor import processor as proc
import vitrage.graph.utils as graph_utils
from vitrage.tests import base
from vitrage.tests.mocks import mock_syncronizer as mock_sync
@ -26,13 +28,13 @@ class TestEntityGraphBase(base.BaseTest):
NUM_NODES = 1
NUM_ZONES = 2
NUM_HOSTS = 4
NUM_INSTANCES = 15
NUM_INSTANCES = 16
def _create_processor_with_graph(self, processor=None):
events = self._create_mock_events()
if not processor:
processor = proc.Processor()
processor = proc.Processor(InitializationStatus())
for event in events:
processor.process_event(event)
@ -78,6 +80,19 @@ class TestEntityGraphBase(base.BaseTest):
return events_list[0]
@staticmethod
def _create_alarm(vitrage_id, alarm_type):
return graph_utils.create_vertex(
vitrage_id,
entity_id=vitrage_id,
entity_category=EntityCategory.ALARM,
entity_type=alarm_type,
entity_state='Running',
is_deleted=False,
update_timestamp=utcnow(),
is_placeholder=False,
)
def _num_total_expected_vertices(self):
return self.NUM_NODES + self.NUM_ZONES + self.NUM_HOSTS + \
self.NUM_INSTANCES

102
vitrage/tests/unit/entity_graph/consistency/test_consistency.py
View File

@ -1,102 +0,0 @@
# Copyright 2016 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from datetime import timedelta
import time
from oslo_config import cfg
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import EntityType
from vitrage.common.constants import VertexProperties as VProps
from vitrage.common.datetime_utils import utcnow
from vitrage.entity_graph.consistency.consistency_enforcer \
import ConsistencyEnforcer
from vitrage.entity_graph.processor.processor import Processor
from vitrage.tests.unit.entity_graph import TestEntityGraphBase
class TestConsistencyBase(TestEntityGraphBase):
OPTS = [
cfg.IntOpt('consistency_interval',
default=1,
min=1),
cfg.IntOpt('min_time_to_delete',
default=1,
min=1),
]
def setUp(self):
super(TestConsistencyBase, self).setUp()
self.processor = Processor()
self.conf = cfg.ConfigOpts()
self.conf.register_opts(self.OPTS, group='consistency')
self.consistency_enforcer = ConsistencyEnforcer(
self.conf, self.processor.entity_graph)
def test_periodic_process(self):
self._create_processor_with_graph(processor=self.processor)
current_time = utcnow()
consistency_inteval = self.conf.consistency.consistency_interval
# set all vertices to be have timestamp that consistency won't get
self._update_timestamp(self.processor.entity_graph.get_vertices(),
current_time +
timedelta(seconds=1.5 * consistency_inteval))
# check number of instances in graph
instance_vertices = self.processor.entity_graph.get_vertices(
{VProps.CATEGORY: EntityCategory.RESOURCE,
VProps.TYPE: EntityType.NOVA_INSTANCE}
)
self.assertEqual(self.NUM_INSTANCES, len(instance_vertices))
# set current timestamp of part of the instances
self._update_timestamp(instance_vertices[0:3], current_time)
# set part of the instances as deleted + update to current timestamp
for i in range(3, 6):
instance_vertices[i][VProps.IS_DELETED] = True
self.processor.entity_graph.update_vertex(instance_vertices[i])
# set part of the instances as deleted
for i in range(6, 9):
instance_vertices[i][VProps.IS_DELETED] = True
instance_vertices[i][VProps.UPDATE_TIMESTAMP] = str(
current_time + timedelta(seconds=2 * consistency_inteval + 1))
self.processor.entity_graph.update_vertex(instance_vertices[i])
# sleep
time.sleep(2 * consistency_inteval + 1)
# run periodic check
self.consistency_enforcer.periodic_process()
# check number of instances
instance_vertices = self.processor.entity_graph.get_vertices(
{VProps.CATEGORY: EntityCategory.RESOURCE,
VProps.TYPE: EntityType.NOVA_INSTANCE}
)
self.assertEqual(self.NUM_INSTANCES - 6, len(instance_vertices))
self.assertEqual(self._num_total_expected_vertices() - 6,
len(self.processor.entity_graph.get_vertices()))
def test_starting_process(self):
pass
def _update_timestamp(self, list, timestamp):
for vertex in list:
vertex[VProps.UPDATE_TIMESTAMP] = str(timestamp)
self.processor.entity_graph.update_vertex(vertex)

5
vitrage/tests/unit/entity_graph/processor/test_processor.py
View File

@ -18,6 +18,7 @@ from vitrage.common.constants import SynchronizerProperties as SyncProps
from vitrage.common.constants import SyncMode
from vitrage.common.constants import VertexProperties
from vitrage.common.datetime_utils import utcnow
from vitrage.entity_graph.initialization_status import InitializationStatus
from vitrage.entity_graph.processor import processor as proc
from vitrage.tests.unit.entity_graph import TestEntityGraphBase
@ -53,7 +54,7 @@ class TestProcessorBase(TestEntityGraphBase):
@unittest.skip('Not ready yet')
def test_process_event(self):
# check create instance event
processor = proc.Processor()
processor = proc.Processor(InitializationStatus())
event = self._create_event(spec_type=self.INSTANCE_SPEC,
sync_mode=SyncMode.INIT_SNAPSHOT)
processor.process_event(event)
@ -194,7 +195,7 @@ class TestProcessorBase(TestEntityGraphBase):
# add instance entity with host
if processor is None:
processor = proc.Processor()
processor = proc.Processor(InitializationStatus())
(vertex, neighbors, event_type) = processor.transform_entity(event)
processor.create_entity(vertex, neighbors)

2
vitrage/tests/unit/graph/base.py
View File

@ -140,7 +140,7 @@ class GraphTestBase(base.BaseTest):
num_of_tests_per_host):
start = time.time()
g = create_graph(name)
g = create_graph(name, EntityCategory.RESOURCE + ':' + EntityType.NODE)
g.add_vertex(v_node)
g.add_vertex(v_switch)
g.add_edge(e_node_to_switch)

5
vitrage_tempest_tests/tests/base_mock.py
View File

@ -11,7 +11,10 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import testtools
from vitrage.entity_graph.initialization_status import InitializationStatus
from vitrage.entity_graph.processor import processor as proc
from vitrage.tests.mocks import mock_syncronizer as mock_sync
@ -21,7 +24,7 @@ class BaseMock(testtools.TestCase):
def create_processor_with_graph(self):
events = self._create_mock_events()
processor = proc.Processor()
processor = proc.Processor(InitializationStatus())
for event in events:
processor.process_event(event)