openstack/vitrage
Code Issues Proposed changes

template add validate fix backend

Browse Source 
Depends-On: I9e78471cd2b62298742246b658632790bd0b15b0
Change-Id: I6a0b1940c7e516b5a07cc4a2693002afe286d6ce
This commit is contained in:
Idan Hefetz 2018-01-22 12:50:46 +00:00
parent 02995dde4d
commit 9148ec0c2e
7 changed files with 128 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
vitrage/api/controllers/v1/template.py
View File

@ -23,7 +23,6 @@ from vitrage.api.controllers.rest import RootRestController
from vitrage.api.policy import enforce
from vitrage.common.constants import TemplateStatus as TStatus
from vitrage.common.exception import VitrageError
from vitrage.evaluator.template_db import template_repository as template_repo
LOG = log.getLogger(__name__)
@ -84,8 +83,8 @@ class TemplateController(RootRestController):
@pecan.expose('json')
def put(self, **kwargs):
template_path = kwargs['path']
LOG.info("add template: %s", template_path)
templates = kwargs['templates']
LOG.info("add template: %s", templates)
enforce("template add",
pecan.request.headers,
@ -94,7 +93,7 @@ class TemplateController(RootRestController):
template_type = kwargs['template_type']
try:
return self._add(template_path, template_type)
return self._add(templates, template_type)
except Exception as e:
LOG.exception('failed to add template %s', e)
abort(404, str(e))
@ -110,9 +109,10 @@ class TemplateController(RootRestController):
{})
templates = kwargs['templates']
template_type = kwargs.get('template_type')
try:
return self._validate(templates)
return self._validate(templates, template_type)
except Exception as e:
to_unicode = encodeutils.exception_to_unicode(e)
LOG.exception('failed to validate template(s) %s', to_unicode)
@ -143,11 +143,12 @@ class TemplateController(RootRestController):
abort(404, to_unicode)
@staticmethod
def _validate(templates):
def _validate(templates, template_type):
result_json = pecan.request.client.call(pecan.request.context,
'validate_template',
templates=templates)
templates=templates,
template_type=template_type)
try:
return json.loads(result_json)
except Exception as e:
@ -156,12 +157,14 @@ class TemplateController(RootRestController):
abort(404, to_unicode)
@classmethod
def _add(cls, path, template_type):
def _add(cls, templates, template_type):
try:
templates = template_repo.add_template_to_db(
pecan.request.storage, path, template_type)
pecan.request.client.call(pecan.request.context, 'add_template')
return [cls._db_template_to_dict(t) for t in templates]
results = pecan.request.client.call(
pecan.request.context,
'add_template',
templates=templates,
template_type=template_type)
return results
except Exception as e:
LOG.exception('failed to add template file %s ', e)
abort(404, str(e))

107
vitrage/api_handler/apis/template.py
View File

@ -16,11 +16,7 @@ import json
from oslo_log import log
from osprofiler import profiler
from vitrage.evaluator.template_validation.content.template_content_validator \
import content_validation
from vitrage.evaluator.template_validation.status_messages import status_msgs
from vitrage.evaluator.template_validation.template_syntax_validator import \
syntax_validation
from vitrage.evaluator.template_db import template_repository as template_repo
LOG = log.getLogger(__name__)
@ -33,58 +29,37 @@ class TemplateApis(object):
FAILED_MSG = 'validation failed'
OK_MSG = 'validation OK'
def __init__(self, notifier=None):
def __init__(self, notifier=None, db=None):
self.notifier = notifier
self.db = db
def validate_template(self, ctx, templates):
LOG.debug("TemplateApis validate_template templates:"
"%s", str(templates))
results = []
for template in templates:
template_definition = template[1]
path = template[0]
syntax_result = syntax_validation(template_definition)
if not syntax_result.is_valid_config:
self._add_result(path,
self.FAILED_MSG,
syntax_result.description,
syntax_result.comment,
syntax_result.status_code,
results)
continue
content_result = content_validation(
template_definition,
self.def_templates)
if not content_result.is_valid_config:
self._add_result(path,
self.FAILED_MSG,
content_result.description,
content_result.comment,
content_result.status_code,
results)
continue
self._add_result(path,
self.OK_MSG,
'Template validation',
status_msgs[0],
0,
results)
def validate_template(self, ctx, templates, template_type):
LOG.debug("TemplateApis validate_template type: %s content: ",
str(template_type), str(templates))
files_content = [t[1] for t in templates]
paths = [t[0] for t in templates]
results = template_repo.validate_templates(self.db, files_content,
template_type)
results = [_to_result(r, p) for r, p in zip(results, paths)]
return json.dumps({'results': results})
def add_template(self, ctx):
def add_template(self, ctx, templates, template_type):
"""Signal the evaluator
A new template has been added to the database with a status of
LOADING that needs to be handled.
"""
LOG.debug("TemplateApis add_template type: %s content: ",
str(template_type), str(templates))
files_content = [t[1] for t in templates]
results = template_repo.add_templates_to_db(self.db, files_content,
template_type)
LOG.info("Add Template Running")
self.notifier.notify("add template", {'template_action': 'add'})
results = [_db_template_to_dict(r) for r in results]
return results
def delete_template(self, ctx):
"""Signal the evaluator
@ -94,22 +69,32 @@ class TemplateApis(object):
LOG.info("Delete Template Running")
self.notifier.notify("delete template", {'template_action': 'delete'})
@staticmethod
def _add_result(template_path, status, description, message, status_code,
results):
results.append({
'file path': template_path,
'status': status,
'description': description,
'message': str(message),
'status code': status_code
})
@staticmethod
def _get_template_status(result):
def _to_result(result, template_path):
if result.is_valid_config:
return 'pass'
return {
'file path': template_path,
'status': TemplateApis.OK_MSG,
'description': 'Template validation',
'message': str(result.comment),
'status code': result.status_code
}
else:
return 'failed'
return {
'file path': template_path,
'status': TemplateApis.FAILED_MSG,
'description': result.description,
'message': str(result.comment),
'status code': result.status_code
}
def _db_template_to_dict(template):
return {
"uuid": template.uuid,
"name": template.name,
"status": template.status,
"date": str(template.created_at),
"status details": template.status_details,
"type": template.template_type,
}

4
vitrage/api_handler/service.py
View File

@ -27,6 +27,7 @@ from vitrage.api_handler.apis.template import TemplateApis
from vitrage.api_handler.apis.topology import TopologyApis
from vitrage import messaging
from vitrage import rpc as vitrage_rpc
from vitrage import storage
LOG = log.getLogger(__name__)
@ -39,6 +40,7 @@ class VitrageApiHandlerService(os_service.Service):
self.entity_graph = e_graph
self.notifier = VitrageNotifier(self.conf, "vitrage.api",
EVALUATOR_TOPIC)
self.db = storage.get_connection_from_config(conf)
def start(self):
LOG.info("Vitrage Api Handler Service - Starting...")
@ -53,7 +55,7 @@ class VitrageApiHandlerService(os_service.Service):
endpoints = [TopologyApis(self.entity_graph, self.conf),
AlarmApis(self.entity_graph, self.conf),
RcaApis(self.entity_graph, self.conf),
TemplateApis(self.notifier),
TemplateApis(self.notifier, self.db),
EventApis(self.conf),
ResourceApis(self.entity_graph, self.conf)]

88
vitrage/evaluator/template_db/template_repository.py
View File

@ -11,8 +11,6 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from oslo_log import log
from oslo_utils import uuidutils
@ -20,13 +18,9 @@ from vitrage.common.constants import TemplateStatus
from vitrage.common.constants import TemplateTypes as TType
from vitrage.common.exception import VitrageError
from vitrage.evaluator.base import Template
from vitrage.evaluator.template_fields import TemplateFields
from vitrage.evaluator import template_validation
from vitrage.evaluator.template_validation import base
from vitrage.evaluator.template_validation.template_syntax_validator import \
EXCEPTION
from vitrage.storage.sqlalchemy import models
from vitrage.utils import file
LOG = log.getLogger(__name__)
@ -34,46 +28,24 @@ METADATA = 'metadata'
NAME = 'name'
def add_template_to_db(db, path, template_type):
"""Add templates to db
Loads template files, for every template, check it is valid and does
not exist, if so adds it to the database.
:param db:
:param path: path to a file or directory
:param template_type: standard/definition/equivalence
:return: all the templates that were added
:rtype: list of models.Template
"""
added_rows = list()
files = _list_files(path)
for f in files:
template = load_template_file(f)
template_name = template[METADATA][NAME]
templates = db.templates.query(name=template_name)
if [t for t in templates if t.status != TemplateStatus.DELETED]:
LOG.warning("Duplicate templates found %s."
" new template will not be added", template_name)
else:
validation_result = _validate_template(db, template, template_type)
db_row = _to_db_row(validation_result, template, template_type)
def add_templates_to_db(db, templates, template_type):
results = list()
for template in templates:
result = _validate_template(db, template, template_type)
if not _is_duplicate(db, template, result):
db_row = _to_db_row(result, template, template_type)
db.templates.create(db_row)
added_rows.append(db_row)
return added_rows
def _list_files(path):
if os.path.isdir(path):
LOG.info("Adding all templates from %s", path)
return file.list_files(path, '.yaml', with_pathname=True)
elif os.path.isfile(path):
LOG.info("Adding template %s", path)
return [path] # only one file
results.append(db_row)
else:
raise VitrageError("No such file or directory %s" % path)
results.append(_get_duplicate_result(template, template_type))
return results
def validate_templates(db, templates, template_type):
results = list()
for template in templates:
results.append(_validate_template(db, template, template_type))
return results
def _validate_template(db, template, template_type):
@ -89,12 +61,22 @@ def _validate_template(db, template, template_type):
return result
def load_template_file(file_name):
try:
return file.load_yaml_file(file_name, with_exception=True)
except Exception as e:
return {TemplateFields.METADATA: {TemplateFields.NAME: file_name},
EXCEPTION: str(e)}
def _is_duplicate(db, template, result):
if result.is_valid_config:
template_name = template[METADATA][NAME]
templates = db.templates.query(name=template_name)
if [t for t in templates if t.status != TemplateStatus.DELETED]:
return True
def _get_duplicate_result(template, template_type):
return models.Template(
name=template[METADATA][NAME],
uuid="",
status="Failed",
status_details="Duplicate template name",
file_content=template,
template_type=template_type)
def _to_db_row(result, template, template_type):
@ -102,15 +84,13 @@ def _to_db_row(result, template, template_type):
status = TemplateStatus.LOADING if result.is_valid_config else \
TemplateStatus.ERROR
status_details = result.comment
db_row = models.Template(
return models.Template(
name=template[METADATA][NAME],
uuid=uuid,
status=status,
status_details=status_details,
file_content=template,
template_type=template_type,
)
return db_row
template_type=template_type)
def _load_def_templates(db):

29
vitrage/tests/functional/test_configuration.py
View File

@ -11,12 +11,15 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import yaml
from oslo_db.options import database_opts
from vitrage.common.constants import TemplateStatus
from vitrage.common.constants import TemplateTypes as TType
from vitrage.evaluator.template_db.template_repository import \
add_template_to_db
add_templates_to_db
from vitrage import storage
from vitrage.storage.sqlalchemy import models
@ -38,7 +41,8 @@ class TestConfiguration(object):
@classmethod
def add_templates(cls, templates_dir, templates_type=TType.STANDARD):
templates = add_template_to_db(cls._db, templates_dir, templates_type)
yamls = [t for t in TestConfiguration.load_yaml_files(templates_dir)]
templates = add_templates_to_db(cls._db, yamls, templates_type)
for t in templates:
if t.status == TemplateStatus.LOADING:
cls._db.templates.update(t.uuid, 'status',
@ -47,3 +51,24 @@ class TestConfiguration(object):
cls._db.templates.update(t.uuid, 'status',
TemplateStatus.DELETED)
return templates
@staticmethod
def load_yaml_files(path):
if os.path.isdir(path):
file_paths = [path + "/" + fn for fn in os.listdir(path)
if os.path.isfile(path + "/" + fn)]
else:
file_paths = [path]
yamls = []
for file_path in file_paths:
try:
yamls.append(TestConfiguration._load_yaml_file(file_path))
except Exception:
continue
return yamls
@staticmethod
def _load_yaml_file(path):
with open(path, 'r') as stream:
return yaml.load(stream, Loader=yaml.BaseLoader)

11
vitrage_tempest_tests/add_legacy_dir_templates.py
View File

@ -13,12 +13,9 @@
# under the License.
import sys
from vitrage.common.constants import TemplateStatus
from vitrage.common.constants import TemplateTypes as TType
from vitrage.evaluator.template_db.template_repository import \
add_template_to_db
from vitrage import service
from vitrage import storage
from vitrage.tests.functional.test_configuration import TestConfiguration
from vitrage_tempest_tests.tests.common import general_utils
files = ['corrupted_template.yaml', 'e2e_test_basic_actions.yaml',
@ -32,12 +29,10 @@ def main():
resources_path = general_utils.tempest_resources_dir() + '/templates/api/'
conf = service.prepare_service()
db = storage.get_connection_from_config(conf)
TestConfiguration._db = db
for f in files:
full_path = resources_path + f
template = add_template_to_db(db, full_path, TType.STANDARD)
if template[0]['name'] != 'corrupted_template':
db.templates.update(template[0]['uuid'],
'status', TemplateStatus.ACTIVE)
TestConfiguration.add_templates(full_path)
if __name__ == "__main__":
sys.exit(main())

5
vitrage_tempest_tests/tests/api/templates/test_template.py
View File

@ -18,8 +18,7 @@ from oslo_log import log as logging
from vitrage.common.constants import TemplateStatus
from vitrage.common.constants import TemplateTypes as TTypes
from vitrage.evaluator.template_db.template_repository import \
load_template_file
from vitrage.utils import file
from vitrage_tempest_tests.tests.api.templates.base import BaseTemplateTest
from vitrage_tempest_tests.tests.common import general_utils as g_utils
from vitrage_tempest_tests.tests.common.tempest_clients import TempestClients
@ -285,7 +284,7 @@ class TemplatesDBTest(BaseTemplateTest):
type=TTypes.STANDARD,
status=TemplateStatus.ACTIVE)
payload_from_db = self.client.template.show(db_row['uuid'])
payload_from_file = load_template_file(template_path)
payload_from_file = file.load_yaml_file(template_path)
self.assertEqual(payload_from_file, payload_from_db,
"Template content doesn't match")
vitrage_utils.delete_template(db_row['uuid'])