diff --git a/drydock_provisioner/error.py b/drydock_provisioner/error.py index 8f32cf3f..581af9ca 100644 --- a/drydock_provisioner/error.py +++ b/drydock_provisioner/error.py @@ -53,6 +53,14 @@ class InvalidDesignReference(DesignError): """ pass +class UnsupportedDocumentType(DesignError): + """ + **Message:** *Site definition document in an unknown format*. + + **Troubleshoot:** + """ + pass + class StateError(Exception): pass diff --git a/drydock_provisioner/ingester/plugins/deckhand.py b/drydock_provisioner/ingester/plugins/deckhand.py index 646d59c2..51908a1e 100644 --- a/drydock_provisioner/ingester/plugins/deckhand.py +++ b/drydock_provisioner/ingester/plugins/deckhand.py @@ -74,8 +74,8 @@ class DeckhandIngester(IngesterPlugin): raise errors.IngesterError("Error parsing YAML: %s" % (err)) # tracking processing status to provide a complete summary of issues - ps = objects.TaskStatus() - ps.set_status(hd_fields.ActionResult.Success) + ps = objects.Validation() + ps.set_status(hd_fields.ValidationResult.Success) for d in parsed_data: try: (schema_ns, doc_kind, doc_version) = d.get('schema', @@ -87,45 +87,56 @@ class DeckhandIngester(IngesterPlugin): continue if schema_ns == 'drydock': try: + doc_ref = objects.DocumentReference( + doc_type=hd_fields.DocumentType.Deckhand, + doc_schema=d.get('schema'), + doc_name=d.get('metadata', {}).get('name', 'Unknown')) doc_errors = self.validate_drydock_document(d) if len(doc_errors) > 0: - doc_ctx = d.get('metadata', {}).get('name', 'Unknown') for e in doc_errors: - ps.add_status_msg( - msg="%s:%s validation error: %s" % - (doc_kind, doc_version, e), - error=True, - ctx_type='document', - ctx=doc_ctx) + ps.add_detail_msg( + objects.ValidationMessage( + msg="%s:%s schema validation error: %s" % + (doc_kind, doc_version, e), + name="DD001", + docs=[doc_ref], + error=True, + level=hd_fields.MessageLevels.ERROR, + diagnostic= + "Invalid input file - see Drydock Troubleshooting Guide for DD001" + )) ps.set_status(hd_fields.ActionResult.Failure) continue model = self.process_drydock_document(d) - ps.add_status_msg( - msg="Successfully processed Drydock document type %s." - % doc_kind, - error=False, - ctx_type='document', - ctx=model.get_id()) + model.doc_ref = doc_ref models.append(model) except errors.IngesterError as ie: msg = "Error processing document: %s" % str(ie) self.logger.warning(msg) - if d.get('metadata', {}).get('name', None) is not None: - ctx = d.get('metadata').get('name') - else: - ctx = 'Unknown' - ps.add_status_msg( - msg=msg, error=True, ctx_type='document', ctx=ctx) + ps.add_detail_msg( + objects.ValidationMessage( + msg=msg, + name="DD000", + error=True, + level=hd_fields.MessageLevels.ERROR, + docs=[doc_ref], + diagnostic="Exception during document processing " + "- see Drydock Troubleshooting Guide " + "for DD000")) ps.set_status(hd_fields.ActionResult.Failure) except Exception as ex: msg = "Unexpected error processing document: %s" % str(ex) self.logger.error(msg, exc_info=True) - if d.get('metadata', {}).get('name', None) is not None: - ctx = d.get('metadata').get('name') - else: - ctx = 'Unknown' - ps.add_status_msg( - msg=msg, error=True, ctx_type='document', ctx=ctx) + ps.add_detail_msg( + objects.ValidationMessage( + msg=msg, + name="DD000", + error=True, + level=hd_fields.MessageLevels.ERROR, + docs=[doc_ref], + diagnostic="Unexpected exception during document " + "processing - see Drydock Troubleshooting " + "Guide for DD000")) ps.set_status(hd_fields.ActionResult.Failure) return (ps, models) diff --git a/drydock_provisioner/objects/__init__.py b/drydock_provisioner/objects/__init__.py index 3323f56f..a38464d1 100644 --- a/drydock_provisioner/objects/__init__.py +++ b/drydock_provisioner/objects/__init__.py @@ -32,6 +32,7 @@ def register_all(): importlib.import_module('drydock_provisioner.objects.bootaction') importlib.import_module('drydock_provisioner.objects.task') importlib.import_module('drydock_provisioner.objects.builddata') + importlib.import_module('drydock_provisioner.objects.validation') # Utility class for calculating inheritance diff --git a/drydock_provisioner/objects/base.py b/drydock_provisioner/objects/base.py index c3dd7365..69f7edb9 100644 --- a/drydock_provisioner/objects/base.py +++ b/drydock_provisioner/objects/base.py @@ -34,6 +34,11 @@ class DrydockObject(base.VersionedObject): OBJ_PROJECT_NAMESPACE = 'drydock_provisioner.objects' + # Maintain a reference to the source document for the model + fields = { + 'doc_ref': obj_fields.ObjectField('DocumentReference', nullable=True) + } + # Return None for undefined attributes def obj_load_attr(self, attrname): if attrname in self.fields.keys(): diff --git a/drydock_provisioner/objects/fields.py b/drydock_provisioner/objects/fields.py index 885ae172..40531a16 100644 --- a/drydock_provisioner/objects/fields.py +++ b/drydock_provisioner/objects/fields.py @@ -193,3 +193,13 @@ class NetworkLinkTrunkingModeField(fields.BaseEnumField): class ValidationResult(BaseDrydockEnum): Success = 'success' Failure = 'failure' + + +class MessageLevels(BaseDrydockEnum): + INFO = 'Info' + WARN = 'Warning' + ERROR = 'Error' + + +class DocumentType(BaseDrydockEnum): + Deckhand = 'deckhand' diff --git a/drydock_provisioner/objects/validation.py b/drydock_provisioner/objects/validation.py new file mode 100644 index 00000000..c37b1dbc --- /dev/null +++ b/drydock_provisioner/objects/validation.py @@ -0,0 +1,120 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Models for representing asynchronous tasks.""" + +from datetime import datetime + +import oslo_versionedobjects.fields as ovo_fields + +from drydock_provisioner import objects + +import drydock_provisioner.objects.base as base +import drydock_provisioner.error as errors +import drydock_provisioner.objects.fields as hd_fields + +from .task import TaskStatus, TaskStatusMessage + + +class Validation(TaskStatus): + """Specialized status for design validation status.""" + + def __init__(self): + super().__init__() + + def add_detail_msg(self, msg=None): + """Add a detailed validation message. + + :param msg: instance of ValidationMessage + """ + self.message_list.append(msg) + + if msg.error or msg.level == "Error": + self.error_count = self.error_count + 1 + + def to_dict(self): + return { + 'kind': 'Status', + 'apiVersion': 'v1.0', + 'metadata': {}, + 'message': self.message, + 'reason': self.reason, + 'status': self.status, + 'details': { + 'errorCount': self.error_count, + 'messageList': [x.to_dict() for x in self.message_list], + } + } + + +class ValidationMessage(TaskStatusMessage): + """Message describing details of a validation.""" + + def __init__(self, msg, name, error=False, level=None, docs=None, diagnostic=None): + self.name = name + self.message = msg + self.error = error + self.level = level + self.diagnostic = diagnostic + self.ts = datetime.utcnow() + self.docs = docs + + def to_dict(self): + """Convert to a dictionary in prep for JSON/YAML serialization.""" + _dict = { + 'kind': 'ValidationMessage', + 'name': self.name, + 'message': self.message, + 'error': self.error, + 'level': self.level, + 'diagnostic': self.diagnostic, + 'ts': str(self.ts), + 'documents': [x.to_dict() for x in self.docs] + } + return _dict + + +@base.DrydockObjectRegistry.register +class DocumentReference(base.DrydockObject): + """Keep a reference to the original document that data was loaded from.""" + + VERSION = '1.0' + + fields = { + 'doc_type': ovo_fields.StringField(), + 'doc_schema': ovo_fields.StringField(nullable=True), + 'doc_name': ovo_fields.StringField(nullable=True), + } + + def __init__(self, **kwargs): + super().__init__(**kwargs) + if (self.doc_type == hd_fields.DocumentType.Deckhand): + if not all([self.doc_schema, self.doc_name]): + raise ValueError("doc_schema and doc_name required for Deckhand sources.") + else: + raise errors.UnsupportedDocumentType( + "Document type %s not supported." % self.doc_type) + + def to_dict(self): + """Serialize to a dictionary for further serialization.""" + d = dict() + if self.doc_type == hd_fields.DocumentType.Deckhand: + d['schema'] = self.doc_schema + d['name'] = self.doc_name + + return d + + +# Emulate OVO object registration +setattr(objects, Validation.obj_name(), Validation) +setattr(objects, ValidationMessage.obj_name(), ValidationMessage) diff --git a/tests/unit/test_ingester.py b/tests/unit/test_ingester.py index c41d866f..8ac9db9a 100644 --- a/tests/unit/test_ingester.py +++ b/tests/unit/test_ingester.py @@ -19,8 +19,6 @@ import drydock_provisioner.objects as objects class TestClass(object): def test_ingest_deckhand(self, input_files, setup, deckhand_ingester): - objects.register_all() - input_file = input_files.join("deckhand_fullsite.yaml") design_state = DrydockState() @@ -29,14 +27,26 @@ class TestClass(object): design_status, design_data = deckhand_ingester.ingest_data( design_state=design_state, design_ref=design_ref) - print("%s" % str(design_status.to_dict())) - assert design_status.status == objects.fields.ActionResult.Success + assert design_status.status == objects.fields.ValidationResult.Success assert len(design_data.host_profiles) == 2 assert len(design_data.baremetal_nodes) == 2 - def test_ingest_yaml(self, input_files, setup, yaml_ingester): - objects.register_all() + def test_ingest_deckhand_docref_exists(self, input_files, setup, deckhand_ingester): + """Test that each processed document has a doc_ref.""" + input_file = input_files.join('deckhand_fullsite.yaml') + design_state = DrydockState() + design_ref = "file://%s" % str(input_file) + design_status, design_data = deckhand_ingester.ingest_data( + design_state=design_state, design_ref=design_ref) + + assert design_status.status == objects.fields.ValidationResult.Success + for p in design_data.host_profiles: + assert p.doc_ref is not None + assert p.doc_ref.doc_schema == 'drydock/HostProfile/v1' + assert p.doc_ref.doc_name is not None + + def test_ingest_yaml(self, input_files, setup, yaml_ingester): input_file = input_files.join("fullsite.yaml") design_state = DrydockState() @@ -45,7 +55,6 @@ class TestClass(object): design_status, design_data = yaml_ingester.ingest_data( design_state=design_state, design_ref=design_ref) - print("%s" % str(design_status.to_dict())) - assert design_status.status == objects.fields.ActionResult.Success + assert design_status.status == objects.fields.ValidationResult.Success assert len(design_data.host_profiles) == 2 assert len(design_data.baremetal_nodes) == 2