From 9148ec0c2e3bbf1403edcdc24fb9aa9987144dda Mon Sep 17 00:00:00 2001 From: Idan Hefetz Date: Mon, 22 Jan 2018 12:50:46 +0000 Subject: [PATCH] template add validate fix backend Depends-On: I9e78471cd2b62298742246b658632790bd0b15b0 Change-Id: I6a0b1940c7e516b5a07cc4a2693002afe286d6ce --- vitrage/api/controllers/v1/template.py | 27 +++-- vitrage/api_handler/apis/template.py | 105 ++++++++---------- vitrage/api_handler/service.py | 4 +- .../template_db/template_repository.py | 86 ++++++-------- .../tests/functional/test_configuration.py | 29 ++++- .../add_legacy_dir_templates.py | 11 +- .../tests/api/templates/test_template.py | 5 +- 7 files changed, 128 insertions(+), 139 deletions(-) diff --git a/vitrage/api/controllers/v1/template.py b/vitrage/api/controllers/v1/template.py index 1114221d7..8cb0a0b5c 100644 --- a/vitrage/api/controllers/v1/template.py +++ b/vitrage/api/controllers/v1/template.py @@ -23,7 +23,6 @@ from vitrage.api.controllers.rest import RootRestController from vitrage.api.policy import enforce from vitrage.common.constants import TemplateStatus as TStatus from vitrage.common.exception import VitrageError -from vitrage.evaluator.template_db import template_repository as template_repo LOG = log.getLogger(__name__) @@ -84,8 +83,8 @@ class TemplateController(RootRestController): @pecan.expose('json') def put(self, **kwargs): - template_path = kwargs['path'] - LOG.info("add template: %s", template_path) + templates = kwargs['templates'] + LOG.info("add template: %s", templates) enforce("template add", pecan.request.headers, @@ -94,7 +93,7 @@ class TemplateController(RootRestController): template_type = kwargs['template_type'] try: - return self._add(template_path, template_type) + return self._add(templates, template_type) except Exception as e: LOG.exception('failed to add template %s', e) abort(404, str(e)) @@ -110,9 +109,10 @@ class TemplateController(RootRestController): {}) templates = kwargs['templates'] + template_type = kwargs.get('template_type') try: - return self._validate(templates) + return self._validate(templates, template_type) except Exception as e: to_unicode = encodeutils.exception_to_unicode(e) LOG.exception('failed to validate template(s) %s', to_unicode) @@ -143,11 +143,12 @@ class TemplateController(RootRestController): abort(404, to_unicode) @staticmethod - def _validate(templates): + def _validate(templates, template_type): result_json = pecan.request.client.call(pecan.request.context, 'validate_template', - templates=templates) + templates=templates, + template_type=template_type) try: return json.loads(result_json) except Exception as e: @@ -156,12 +157,14 @@ class TemplateController(RootRestController): abort(404, to_unicode) @classmethod - def _add(cls, path, template_type): + def _add(cls, templates, template_type): try: - templates = template_repo.add_template_to_db( - pecan.request.storage, path, template_type) - pecan.request.client.call(pecan.request.context, 'add_template') - return [cls._db_template_to_dict(t) for t in templates] + results = pecan.request.client.call( + pecan.request.context, + 'add_template', + templates=templates, + template_type=template_type) + return results except Exception as e: LOG.exception('failed to add template file %s ', e) abort(404, str(e)) diff --git a/vitrage/api_handler/apis/template.py b/vitrage/api_handler/apis/template.py index 6d405da76..924c87e40 100644 --- a/vitrage/api_handler/apis/template.py +++ b/vitrage/api_handler/apis/template.py @@ -16,11 +16,7 @@ import json from oslo_log import log from osprofiler import profiler -from vitrage.evaluator.template_validation.content.template_content_validator \ - import content_validation -from vitrage.evaluator.template_validation.status_messages import status_msgs -from vitrage.evaluator.template_validation.template_syntax_validator import \ - syntax_validation +from vitrage.evaluator.template_db import template_repository as template_repo LOG = log.getLogger(__name__) @@ -33,58 +29,37 @@ class TemplateApis(object): FAILED_MSG = 'validation failed' OK_MSG = 'validation OK' - def __init__(self, notifier=None): + def __init__(self, notifier=None, db=None): self.notifier = notifier + self.db = db - def validate_template(self, ctx, templates): - LOG.debug("TemplateApis validate_template templates:" - "%s", str(templates)) - - results = [] - for template in templates: - - template_definition = template[1] - path = template[0] - - syntax_result = syntax_validation(template_definition) - if not syntax_result.is_valid_config: - self._add_result(path, - self.FAILED_MSG, - syntax_result.description, - syntax_result.comment, - syntax_result.status_code, - results) - continue - - content_result = content_validation( - template_definition, - self.def_templates) - if not content_result.is_valid_config: - self._add_result(path, - self.FAILED_MSG, - content_result.description, - content_result.comment, - content_result.status_code, - results) - continue - - self._add_result(path, - self.OK_MSG, - 'Template validation', - status_msgs[0], - 0, - results) + def validate_template(self, ctx, templates, template_type): + LOG.debug("TemplateApis validate_template type: %s content: ", + str(template_type), str(templates)) + files_content = [t[1] for t in templates] + paths = [t[0] for t in templates] + results = template_repo.validate_templates(self.db, files_content, + template_type) + results = [_to_result(r, p) for r, p in zip(results, paths)] return json.dumps({'results': results}) - def add_template(self, ctx): + def add_template(self, ctx, templates, template_type): """Signal the evaluator A new template has been added to the database with a status of LOADING that needs to be handled. """ + LOG.debug("TemplateApis add_template type: %s content: ", + str(template_type), str(templates)) + + files_content = [t[1] for t in templates] + results = template_repo.add_templates_to_db(self.db, files_content, + template_type) LOG.info("Add Template Running") self.notifier.notify("add template", {'template_action': 'add'}) + results = [_db_template_to_dict(r) for r in results] + return results def delete_template(self, ctx): """Signal the evaluator @@ -94,22 +69,32 @@ class TemplateApis(object): LOG.info("Delete Template Running") self.notifier.notify("delete template", {'template_action': 'delete'}) - @staticmethod - def _add_result(template_path, status, description, message, status_code, - results): - results.append({ +def _to_result(result, template_path): + if result.is_valid_config: + return { 'file path': template_path, - 'status': status, - 'description': description, - 'message': str(message), - 'status code': status_code - }) + 'status': TemplateApis.OK_MSG, + 'description': 'Template validation', + 'message': str(result.comment), + 'status code': result.status_code + } + else: + return { + 'file path': template_path, + 'status': TemplateApis.FAILED_MSG, + 'description': result.description, + 'message': str(result.comment), + 'status code': result.status_code + } - @staticmethod - def _get_template_status(result): - if result.is_valid_config: - return 'pass' - else: - return 'failed' +def _db_template_to_dict(template): + return { + "uuid": template.uuid, + "name": template.name, + "status": template.status, + "date": str(template.created_at), + "status details": template.status_details, + "type": template.template_type, + } diff --git a/vitrage/api_handler/service.py b/vitrage/api_handler/service.py index 9836412c6..bebf1e18e 100644 --- a/vitrage/api_handler/service.py +++ b/vitrage/api_handler/service.py @@ -27,6 +27,7 @@ from vitrage.api_handler.apis.template import TemplateApis from vitrage.api_handler.apis.topology import TopologyApis from vitrage import messaging from vitrage import rpc as vitrage_rpc +from vitrage import storage LOG = log.getLogger(__name__) @@ -39,6 +40,7 @@ class VitrageApiHandlerService(os_service.Service): self.entity_graph = e_graph self.notifier = VitrageNotifier(self.conf, "vitrage.api", EVALUATOR_TOPIC) + self.db = storage.get_connection_from_config(conf) def start(self): LOG.info("Vitrage Api Handler Service - Starting...") @@ -53,7 +55,7 @@ class VitrageApiHandlerService(os_service.Service): endpoints = [TopologyApis(self.entity_graph, self.conf), AlarmApis(self.entity_graph, self.conf), RcaApis(self.entity_graph, self.conf), - TemplateApis(self.notifier), + TemplateApis(self.notifier, self.db), EventApis(self.conf), ResourceApis(self.entity_graph, self.conf)] diff --git a/vitrage/evaluator/template_db/template_repository.py b/vitrage/evaluator/template_db/template_repository.py index ae718f7b9..9e404e1ee 100644 --- a/vitrage/evaluator/template_db/template_repository.py +++ b/vitrage/evaluator/template_db/template_repository.py @@ -11,8 +11,6 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -import os - from oslo_log import log from oslo_utils import uuidutils @@ -20,13 +18,9 @@ from vitrage.common.constants import TemplateStatus from vitrage.common.constants import TemplateTypes as TType from vitrage.common.exception import VitrageError from vitrage.evaluator.base import Template -from vitrage.evaluator.template_fields import TemplateFields from vitrage.evaluator import template_validation from vitrage.evaluator.template_validation import base -from vitrage.evaluator.template_validation.template_syntax_validator import \ - EXCEPTION from vitrage.storage.sqlalchemy import models -from vitrage.utils import file LOG = log.getLogger(__name__) @@ -34,46 +28,24 @@ METADATA = 'metadata' NAME = 'name' -def add_template_to_db(db, path, template_type): - """Add templates to db - - Loads template files, for every template, check it is valid and does - not exist, if so adds it to the database. - - :param db: - :param path: path to a file or directory - :param template_type: standard/definition/equivalence - :return: all the templates that were added - :rtype: list of models.Template - """ - - added_rows = list() - files = _list_files(path) - for f in files: - template = load_template_file(f) - template_name = template[METADATA][NAME] - - templates = db.templates.query(name=template_name) - if [t for t in templates if t.status != TemplateStatus.DELETED]: - LOG.warning("Duplicate templates found %s." - " new template will not be added", template_name) - else: - validation_result = _validate_template(db, template, template_type) - db_row = _to_db_row(validation_result, template, template_type) +def add_templates_to_db(db, templates, template_type): + results = list() + for template in templates: + result = _validate_template(db, template, template_type) + if not _is_duplicate(db, template, result): + db_row = _to_db_row(result, template, template_type) db.templates.create(db_row) - added_rows.append(db_row) - return added_rows + results.append(db_row) + else: + results.append(_get_duplicate_result(template, template_type)) + return results -def _list_files(path): - if os.path.isdir(path): - LOG.info("Adding all templates from %s", path) - return file.list_files(path, '.yaml', with_pathname=True) - elif os.path.isfile(path): - LOG.info("Adding template %s", path) - return [path] # only one file - else: - raise VitrageError("No such file or directory %s" % path) +def validate_templates(db, templates, template_type): + results = list() + for template in templates: + results.append(_validate_template(db, template, template_type)) + return results def _validate_template(db, template, template_type): @@ -89,12 +61,22 @@ def _validate_template(db, template, template_type): return result -def load_template_file(file_name): - try: - return file.load_yaml_file(file_name, with_exception=True) - except Exception as e: - return {TemplateFields.METADATA: {TemplateFields.NAME: file_name}, - EXCEPTION: str(e)} +def _is_duplicate(db, template, result): + if result.is_valid_config: + template_name = template[METADATA][NAME] + templates = db.templates.query(name=template_name) + if [t for t in templates if t.status != TemplateStatus.DELETED]: + return True + + +def _get_duplicate_result(template, template_type): + return models.Template( + name=template[METADATA][NAME], + uuid="", + status="Failed", + status_details="Duplicate template name", + file_content=template, + template_type=template_type) def _to_db_row(result, template, template_type): @@ -102,15 +84,13 @@ def _to_db_row(result, template, template_type): status = TemplateStatus.LOADING if result.is_valid_config else \ TemplateStatus.ERROR status_details = result.comment - db_row = models.Template( + return models.Template( name=template[METADATA][NAME], uuid=uuid, status=status, status_details=status_details, file_content=template, - template_type=template_type, - ) - return db_row + template_type=template_type) def _load_def_templates(db): diff --git a/vitrage/tests/functional/test_configuration.py b/vitrage/tests/functional/test_configuration.py index e0a27a64f..7c9921059 100644 --- a/vitrage/tests/functional/test_configuration.py +++ b/vitrage/tests/functional/test_configuration.py @@ -11,12 +11,15 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import os +import yaml + from oslo_db.options import database_opts from vitrage.common.constants import TemplateStatus from vitrage.common.constants import TemplateTypes as TType from vitrage.evaluator.template_db.template_repository import \ - add_template_to_db + add_templates_to_db from vitrage import storage from vitrage.storage.sqlalchemy import models @@ -38,7 +41,8 @@ class TestConfiguration(object): @classmethod def add_templates(cls, templates_dir, templates_type=TType.STANDARD): - templates = add_template_to_db(cls._db, templates_dir, templates_type) + yamls = [t for t in TestConfiguration.load_yaml_files(templates_dir)] + templates = add_templates_to_db(cls._db, yamls, templates_type) for t in templates: if t.status == TemplateStatus.LOADING: cls._db.templates.update(t.uuid, 'status', @@ -47,3 +51,24 @@ class TestConfiguration(object): cls._db.templates.update(t.uuid, 'status', TemplateStatus.DELETED) return templates + + @staticmethod + def load_yaml_files(path): + if os.path.isdir(path): + file_paths = [path + "/" + fn for fn in os.listdir(path) + if os.path.isfile(path + "/" + fn)] + else: + file_paths = [path] + + yamls = [] + for file_path in file_paths: + try: + yamls.append(TestConfiguration._load_yaml_file(file_path)) + except Exception: + continue + return yamls + + @staticmethod + def _load_yaml_file(path): + with open(path, 'r') as stream: + return yaml.load(stream, Loader=yaml.BaseLoader) diff --git a/vitrage_tempest_tests/add_legacy_dir_templates.py b/vitrage_tempest_tests/add_legacy_dir_templates.py index bf6f26d3b..353eb7ff4 100644 --- a/vitrage_tempest_tests/add_legacy_dir_templates.py +++ b/vitrage_tempest_tests/add_legacy_dir_templates.py @@ -13,12 +13,9 @@ # under the License. import sys -from vitrage.common.constants import TemplateStatus -from vitrage.common.constants import TemplateTypes as TType -from vitrage.evaluator.template_db.template_repository import \ - add_template_to_db from vitrage import service from vitrage import storage +from vitrage.tests.functional.test_configuration import TestConfiguration from vitrage_tempest_tests.tests.common import general_utils files = ['corrupted_template.yaml', 'e2e_test_basic_actions.yaml', @@ -32,12 +29,10 @@ def main(): resources_path = general_utils.tempest_resources_dir() + '/templates/api/' conf = service.prepare_service() db = storage.get_connection_from_config(conf) + TestConfiguration._db = db for f in files: full_path = resources_path + f - template = add_template_to_db(db, full_path, TType.STANDARD) - if template[0]['name'] != 'corrupted_template': - db.templates.update(template[0]['uuid'], - 'status', TemplateStatus.ACTIVE) + TestConfiguration.add_templates(full_path) if __name__ == "__main__": sys.exit(main()) diff --git a/vitrage_tempest_tests/tests/api/templates/test_template.py b/vitrage_tempest_tests/tests/api/templates/test_template.py index 37b5a3645..f8a50d3d8 100644 --- a/vitrage_tempest_tests/tests/api/templates/test_template.py +++ b/vitrage_tempest_tests/tests/api/templates/test_template.py @@ -18,8 +18,7 @@ from oslo_log import log as logging from vitrage.common.constants import TemplateStatus from vitrage.common.constants import TemplateTypes as TTypes -from vitrage.evaluator.template_db.template_repository import \ - load_template_file +from vitrage.utils import file from vitrage_tempest_tests.tests.api.templates.base import BaseTemplateTest from vitrage_tempest_tests.tests.common import general_utils as g_utils from vitrage_tempest_tests.tests.common.tempest_clients import TempestClients @@ -285,7 +284,7 @@ class TemplatesDBTest(BaseTemplateTest): type=TTypes.STANDARD, status=TemplateStatus.ACTIVE) payload_from_db = self.client.template.show(db_row['uuid']) - payload_from_file = load_template_file(template_path) + payload_from_file = file.load_yaml_file(template_path) self.assertEqual(payload_from_file, payload_from_db, "Template content doesn't match") vitrage_utils.delete_template(db_row['uuid'])