airship/deckhand
Code Issues Proposed changes

Merge "Simplify schema validation"

Browse Source
This commit is contained in:
Zuul 2018-07-10 13:42:57 +00:00 committed by Gerrit Code Review
commit 109b78df59
51 changed files with 410 additions and 610 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
deckhand/common/document.py
View File

@ -71,7 +71,7 @@ class DocumentDict(dict):
@property
def is_control(self):
return self.metadata.get('schema', '').startswith('deckhand/Control')
return self.metadata.get('schema', '').startswith('metadata/Control')
@property
def schema(self):

2
deckhand/control/revision_documents.py
View File

@ -164,8 +164,8 @@ class RenderedDocumentsResource(api_base.BaseResource):
rendered_documents = rendered_documents[:limit]
resp.status = falcon.HTTP_200
resp.body = self.view_builder.list(rendered_documents)
self._post_validate(rendered_documents)
resp.body = self.view_builder.list(rendered_documents)
def _retrieve_documents_for_rendering(self, revision_id, **filters):
"""Retrieve all necessary documents needed for rendering. If a layering

2
deckhand/db/sqlalchemy/models.py
View File

@ -38,6 +38,8 @@ LOG = logging.getLogger(__name__)
# relative to that base.
BASE = None
# TODO(felipemonteiro): Make most (all?) of these tables immutable.
class DeckhandBase(models.ModelBase, models.TimestampMixin):
"""Base class for Deckhand Models."""

93
deckhand/engine/document_validation.py
View File

@ -88,14 +88,6 @@ class BaseValidator(object):
global _DEFAULT_SCHEMAS
self._schema_map = _DEFAULT_SCHEMAS
@abc.abstractmethod
def matches(self, document):
"""Whether this Validator should be used to validate ``document``.
:param dict document: Document to validate.
:returns: True if Validator applies to ``document``, else False.
"""
@abc.abstractmethod
def validate(self, document):
"""Validate whether ``document`` passes schema validation."""
@ -117,9 +109,32 @@ class GenericValidator(BaseValidator):
super(GenericValidator, self).__init__()
self.base_schema = self._schema_map['v1']['deckhand/Base']
def matches(self, document):
# Applies to all schemas, so unconditionally returns True.
return True
def validate_metadata(self, metadata):
"""Validate ``metadata`` against the given schema.
The ``metadata`` section of a Deckhand document describes a schema
defining just the ``metadata`` section. Use that declaration to
choose a schema for validating ``metadata``.
:param dict metadata: Document metadata section to validate
:returns: list of validation errors or empty list for success
"""
errors = list()
schema_name, schema_ver = _get_schema_parts(metadata)
schema = self._schema_map.get(schema_ver, {}).get(schema_name, {})
if not schema:
return ['Invalid metadata schema %s version %s specified.'
% (schema_name, schema_ver)]
LOG.debug("Validating document metadata with schema %s/%s.",
schema_name, schema_ver)
jsonschema.Draft4Validator.check_schema(schema)
schema_validator = jsonschema.Draft4Validator(schema)
errors.extend([e.message
for e in schema_validator.iter_errors(metadata)])
return errors
def validate(self, document, **kwargs):
"""Validate ``document``against basic schema validation.
@ -144,6 +159,10 @@ class GenericValidator(BaseValidator):
schema_validator = jsonschema.Draft4Validator(self.base_schema)
error_messages = [
e.message for e in schema_validator.iter_errors(document)]
if not error_messages:
error_messages.extend(
self.validate_metadata(document.metadata))
except Exception as e:
raise RuntimeError(
'Unknown error occurred while attempting to use Deckhand '
@ -201,14 +220,6 @@ class DataSchemaValidator(GenericValidator):
self._external_data_schemas = [d.data for d in data_schemas]
self._schema_map = self._build_schema_map(data_schemas)
def matches(self, document):
if document.is_abstract:
LOG.info('Skipping schema validation for abstract document [%s]: '
'%s.', document.schema, document.name)
return False
schema_prefix, schema_version = _get_schema_parts(document)
return schema_prefix in self._schema_map.get(schema_version, {})
def _generate_validation_error_output(self, schema, document, error,
root_path):
"""Returns a formatted output with necessary details for debugging why
@ -308,6 +319,18 @@ class DataSchemaValidator(GenericValidator):
:rtype: Generator[Tuple[str, str]]
"""
super(DataSchemaValidator, self).validate(document)
# if this is a pre_validate, the only validation needed is structural
# for non-control documents
if not document.is_control and pre_validate:
return
if document.is_abstract:
LOG.info('Skipping schema validation for abstract document [%s, '
'%s] %s.', *document.meta)
return
schemas_to_use = self._get_schemas(document)
if not schemas_to_use:
LOG.debug('Document schema %s not recognized by %s. No further '
@ -315,28 +338,12 @@ class DataSchemaValidator(GenericValidator):
self.__class__.__name__)
for schema in schemas_to_use:
is_builtin_schema = schema not in self._external_data_schemas
# NOTE(fmontei): The purpose of this `continue` is to not
# PRE-validate documents against externally registered
# `DataSchema` documents, in order to avoid raising spurious
# errors. These spurious errors arise from `DataSchema` documents
# really only applying post-rendering, when documents have all
# the substitutions they need to pass externally registered
# `DataSchema` validations.
if not is_builtin_schema and pre_validate:
continue
if is_builtin_schema:
root_path = '.'
to_validate = document
else:
root_path = '.data'
to_validate = document.get('data', {})
root_path = '.data'
try:
jsonschema.Draft4Validator.check_schema(schema)
schema_validator = jsonschema.Draft4Validator(schema)
errors = schema_validator.iter_errors(to_validate)
errors = schema_validator.iter_errors(document.get('data', {}))
except Exception as e:
LOG.exception(six.text_type(e))
raise RuntimeError(
@ -417,10 +424,7 @@ class DocumentValidation(object):
self._documents.append(document)
# NOTE(fmontei): The order of the validators is important. The
# ``GenericValidator`` must come first.
self._validators = [
GenericValidator(),
DataSchemaValidator(self._external_data_schemas)
]
@ -476,11 +480,10 @@ class DocumentValidation(object):
LOG.info(message)
for validator in self._validators:
if validator.matches(document):
error_outputs = validator.validate(
document, pre_validate=self._pre_validate)
if error_outputs:
result['errors'].extend(error_outputs)
error_outputs = validator.validate(
document, pre_validate=self._pre_validate)
if error_outputs:
result['errors'].extend(error_outputs)
if result['errors']:
result.setdefault('status', 'failure')

123
deckhand/engine/schemas/base_schema.yaml
View File

@ -19,131 +19,36 @@ metadata:
schema: metadata/Control/v1
data:
$schema: http://json-schema.org/schema#
definitions:
parent_selector_requires_actions:
dependencies:
# Requires that if parentSelector is provided, then actions is
# required and must contain at least 1 item.
parentSelector:
required:
- actions
actions_requires_parent_selector:
dependencies:
# Requires that if actions are provided, then so too must
# parentSelector.
actions:
required:
- parentSelector
substitution_dest:
type: object
properties:
path:
type: string
pattern:
type: string
additionalProperties: false
required:
- path
properties:
schema:
type: string
pattern: ^[A-Za-z]+/[A-Za-z]+/v\d+$
metadata:
# True validation of the metadata section will be done using
# the schema specfied in the metadata section
type: object
properties:
name:
type: string
schema:
anyOf:
- type: string
pattern: ^metadata/Document/v\d+$
- type: string
pattern: ^metadata/Control/v\d+$
name:
type: string
labels:
type: object
replacement:
type: boolean
layeringDefinition:
type: object
properties:
layer:
type: string
abstract:
type: boolean
parentSelector:
type: object
minProperties: 1
actions:
type: array
minItems: 1
items:
type: object
properties:
method:
enum:
- replace
- delete
- merge
path:
type: string
additionalProperties: false
required:
- method
- path
additionalProperties: false
allOf:
- $ref: "#/definitions/parent_selector_requires_actions"
- $ref: "#/definitions/actions_requires_parent_selector"
substitutions:
type: array
items:
type: object
properties:
dest:
anyOf:
- $ref: "#/definitions/substitution_dest"
- type: array
minItems: 1
items:
$ref: "#/definitions/substitution_dest"
src:
type: object
properties:
schema:
type: string
pattern: ^[A-Za-z]+/[A-Za-z]+/v\d+$
name:
type: string
path:
type: string
additionalProperties: false
required:
- schema
- name
- path
additionalProperties: false
required:
- dest
- src
storagePolicy:
type: string
enum:
- encrypted
- cleartext
additionalProperties: false
additionalProperties: true
required:
- schema
- name
- 'name'
- 'schema'
# This schema should allow anything in the data section
data:
type:
- "null"
- string
- integer
- array
- object
- 'null'
- 'string'
- 'object'
- 'array'
- 'number'
- 'boolean'
additionalProperties: false
required:
- schema

45
deckhand/engine/schemas/certificate_authority_key_schema.yaml
View File

@ -19,47 +19,4 @@ metadata:
schema: metadata/Control/v1
data:
$schema: http://json-schema.org/schema#
properties:
metadata:
type: object
properties:
layeringDefinition:
type: object
properties:
layer:
type: string
abstract:
type: boolean
parentSelector:
type: object
actions:
type: array
items:
type: object
properties:
method:
enum:
- replace
- delete
- merge
path:
type: string
additionalProperties: false
required:
- method
- path
required:
- layer
storagePolicy:
type: string
enum:
- encrypted
- cleartext
required:
- layeringDefinition
- storagePolicy
data:
type: string
required:
- metadata
- data
type: string

45
deckhand/engine/schemas/certificate_authority_schema.yaml
View File

@ -19,47 +19,4 @@ metadata:
schema: metadata/Control/v1
data:
$schema: http://json-schema.org/schema#
properties:
metadata:
type: object
properties:
layeringDefinition:
type: object
properties:
layer:
type: string
abstract:
type: boolean
parentSelector:
type: object
actions:
type: array
items:
type: object
properties:
method:
enum:
- replace
- delete
- merge
path:
type: string
additionalProperties: false
required:
- method
- path
required:
- layer
storagePolicy:
type: string
enum:
- encrypted
- cleartext
required:
- layeringDefinition
- storagePolicy
data:
type: string
required:
- metadata
- data
type: string

45
deckhand/engine/schemas/certificate_key_schema.yaml
View File

@ -19,47 +19,4 @@ metadata:
schema: metadata/Control/v1
data:
$schema: http://json-schema.org/schema#
properties:
metadata:
type: object
properties:
layeringDefinition:
type: object
properties:
layer:
type: string
abstract:
type: boolean
parentSelector:
type: object
actions:
type: array
items:
type: object
properties:
method:
enum:
- replace
- delete
- merge
path:
type: string
additionalProperties: false
required:
- method
- path
required:
- layer
storagePolicy:
type: string
enum:
- encrypted
- cleartext
required:
- layeringDefinition
- storagePolicy
data:
type: string
required:
- metadata
- data
type: string

45
deckhand/engine/schemas/certificate_schema.yaml
View File

@ -19,47 +19,4 @@ metadata:
schema: metadata/Control/v1
data:
$schema: http://json-schema.org/schema#
properties:
metadata:
type: object
properties:
layeringDefinition:
type: object
properties:
layer:
type: string
abstract:
type: boolean
parentSelector:
type: object
actions:
type: array
items:
type: object
properties:
method:
enum:
- replace
- delete
- merge
path:
type: string
additionalProperties: false
required:
- method
- path
required:
- layer
storagePolicy:
type: string
enum:
- encrypted
- cleartext
required:
- layeringDefinition
- storagePolicy
data:
type: string
required:
- metadata
- data
type: string

33
deckhand/engine/schemas/dataschema_schema.yaml
View File

@ -1,33 +0,0 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
schema: deckhand/DataSchema/v1
metadata:
name: deckhand/DataSchema/v1
schema: metadata/Control/v1
data:
$schema: http://json-schema.org/schema#
type: object
properties:
data:
type: object
properties:
$schema:
type: string
additionalProperties: true
required:
- $schema
required:
- data

17
deckhand/engine/schemas/layering_policy_schema.yaml
View File

@ -21,15 +21,10 @@ data:
$schema: http://json-schema.org/schema#
type: object
properties:
data:
type: object
properties:
layerOrder:
type: array
items:
type: string
additionalProperties: false
required:
- layerOrder
layerOrder:
type: array
items:
type: string
additionalProperties: false
required:
- data
- layerOrder

28
deckhand/engine/schemas/metadata_control.yaml Normal file
View File

@ -0,0 +1,28 @@
---
schema: deckhand/DataSchema/v1
metadata:
name: metadata/Control/v1
schema: metadata/Control/v1
data:
$schema: http://json-schema.org/schema#
type: object
properties:
schema:
anyOf:
- type: string
pattern: ^metadata/Document/v\d+$
- type: string
pattern: ^metadata/Control/v\d+$
name:
type: string
labels:
type: object
additionalProperties:
type: string
additionalProperties: true
required:
- schema
- name
# NOTE(felipemonteiro): layeringDefinition is not needed for any control
# documents as neither LayeringPolicy, ValidationPolicy or DataSchema
# documents are ever layered together.

121
deckhand/engine/schemas/metadata_document.yaml Normal file
View File

@ -0,0 +1,121 @@
---
schema: deckhand/DataSchema/v1
metadata:
name: metadata/Document/v1
schema: metadata/Control/v1
data:
$schema: http://json-schema.org/schema#
definitions:
parent_selector_requires_actions:
dependencies:
# Requires that if parentSelector is provided, then actions is
# required and must contain at least 1 item.
parentSelector:
required:
- actions
actions_requires_parent_selector:
dependencies:
# Requires that if actions are provided, then so too must
# parentSelector.
actions:
required:
- parentSelector
substitution_dest:
type: object
properties:
path:
type: string
pattern:
type: string
additionalProperties: false
required:
- path
type: object
properties:
schema:
anyOf:
- type: string
pattern: ^metadata/Document/v\d+$
- type: string
pattern: ^metadata/Control/v\d+$
name:
type: string
labels:
type: object
replacement:
type: boolean
layeringDefinition:
type: object
properties:
layer:
type: string
abstract:
type: boolean
parentSelector:
type: object
minProperties: 1
actions:
type: array
minItems: 1
items:
type: object
properties:
method:
enum:
- replace
- delete
- merge
path:
type: string
additionalProperties: false
required:
- method
- path
additionalProperties: false
required:
- 'layer'
allOf:
- $ref: "#/definitions/parent_selector_requires_actions"
- $ref: "#/definitions/actions_requires_parent_selector"
substitutions:
type: array
items:
type: object
properties:
dest:
anyOf:
- $ref: "#/definitions/substitution_dest"
- type: array
minItems: 1
items:
$ref: "#/definitions/substitution_dest"
src:
type: object
properties:
schema:
type: string
pattern: ^[A-Za-z]+/[A-Za-z]+/v\d+$
name:
type: string
path:
type: string
additionalProperties: false
required:
- schema
- name
- path
additionalProperties: false
required:
- dest
- src
storagePolicy:
type: string
enum:
- encrypted
- cleartext
additionalProperties: false
required:
- schema
- name
- storagePolicy
- layeringDefinition

45
deckhand/engine/schemas/passphrase_schema.yaml
View File

@ -19,47 +19,4 @@ metadata:
schema: metadata/Control/v1
data:
$schema: http://json-schema.org/schema#
properties:
metadata:
type: object
properties:
layeringDefinition:
type: object
properties:
layer:
type: string
abstract:
type: boolean
parentSelector:
type: object
actions:
type: array
items:
type: object
properties:
method:
enum:
- replace
- delete
- merge
path:
type: string
additionalProperties: false
required:
- method
- path
required:
- layer
storagePolicy:
type: string
enum:
- encrypted
- cleartext
required:
- layeringDefinition
- storagePolicy
data:
type: string
required:
- metadata
- data
type: string

45
deckhand/engine/schemas/private_key_schema.yaml
View File

@ -19,47 +19,4 @@ metadata:
schema: metadata/Control/v1
data:
$schema: http://json-schema.org/schema#
properties:
metadata:
type: object
properties:
layeringDefinition:
type: object
properties:
layer:
type: string
abstract:
type: boolean
parentSelector:
type: object
actions:
type: array
items:
type: object
properties:
method:
enum:
- replace
- delete
- merge
path:
type: string
additionalProperties: false
required:
- method
- path
required:
- layer
storagePolicy:
type: string
enum:
- encrypted
- cleartext
required:
- layeringDefinition
- storagePolicy
data:
type: string
required:
- metadata
- data
type: string

45
deckhand/engine/schemas/public_key_schema.yaml
View File

@ -19,47 +19,4 @@ metadata:
schema: metadata/Control/v1
data:
$schema: http://json-schema.org/schema#
properties:
metadata:
type: object
properties:
layeringDefinition:
type: object
properties:
layer:
type: string
abstract:
type: boolean
parentSelector:
type: object
actions:
type: array
items:
type: object
properties:
method:
enum:
- replace
- delete
- merge
path:
type: string
additionalProperties: false
required:
- method
- path
required:
- layer
storagePolicy:
type: string
enum:
- encrypted
- cleartext
required:
- layeringDefinition
- storagePolicy
data:
type: string
required:
- metadata
- data
type: string

67
deckhand/engine/schemas/validation_policy_schema.yaml
View File

@ -21,56 +21,19 @@ data:
$schema: http://json-schema.org/schema#
type: object
properties:
metadata:
type: object
properties:
layeringDefinition:
type: object
properties:
layer:
type: string
abstract:
type: boolean
parentSelector:
type: object
actions:
type: array
items:
type: object
properties:
method:
enum:
- replace
- delete
- merge
path:
type: string
additionalProperties: false
required:
- method
- path
required:
- layer
required:
- layeringDefinition
data:
properties:
validations:
type: array
items:
type: object
properties:
name:
type: string
pattern: ^.*-(validation|verification)$
expiresAfter:
type: string
additionalProperties: false
required:
- name
required:
- validations
additionalProperties: false
validations:
type: array
items:
type: object
properties:
name:
type: string
pattern: ^.*-(validation|verification)$
expiresAfter:
type: string
additionalProperties: false
required:
- name
required:
- metadata
- data
- validations
additionalProperties: false

1
deckhand/factories.py
View File

@ -103,6 +103,7 @@ class DocumentFactory(DeckhandFactory):
"data": {},
"metadata": {
"labels": {"": ""},
"storagePolicy": "cleartext",
"layeringDefinition": {
"abstract": False,
"layer": "layer"

7
deckhand/tests/functional/gabbits/layering/layering-with-replacement-single-bucket.yaml
View File

@ -43,6 +43,7 @@ tests:
metadata:
schema: metadata/Document/v1
name: a
storagePolicy: cleartext
labels:
selector: foo
layeringDefinition:
@ -56,6 +57,7 @@ tests:
metadata:
schema: metadata/Document/v1
name: a
storagePolicy: cleartext
labels:
selector: baz
replacement: true
@ -75,6 +77,7 @@ tests:
metadata:
schema: metadata/Document/v1
name: c
storagePolicy: cleartext
layeringDefinition:
abstract: False
layer: global
@ -140,6 +143,7 @@ tests:
metadata:
schema: metadata/Document/v1
name: a
storagePolicy: cleartext
labels:
selector: foo
layeringDefinition:
@ -153,6 +157,7 @@ tests:
metadata:
schema: metadata/Document/v1
name: a
storagePolicy: cleartext
labels:
selector: baz
replacement: true
@ -172,6 +177,7 @@ tests:
metadata:
schema: metadata/Document/v1
name: b
storagePolicy: cleartext
labels:
selector: qux
layeringDefinition:
@ -190,6 +196,7 @@ tests:
metadata:
schema: metadata/Document/v1
name: c
storagePolicy: cleartext
layeringDefinition:
abstract: False
layer: global

3
deckhand/tests/functional/gabbits/replacement/multi-layer-replacement.yaml
View File

@ -46,6 +46,7 @@ tests:
metadata:
schema: metadata/Document/v1
name: nova-global
storagePolicy: cleartext
labels:
name: nova-global
component: nova
@ -62,6 +63,7 @@ tests:
metadata:
schema: metadata/Document/v1
name: nova
storagePolicy: cleartext
labels:
name: nova-5ec
component: nova
@ -80,6 +82,7 @@ tests:
schema: metadata/Document/v1
replacement: true
name: nova
storagePolicy: cleartext
layeringDefinition:
abstract: false
layer: site

3
deckhand/tests/functional/gabbits/resources/chained-substitution.yaml
View File

@ -11,6 +11,7 @@ schema: example/Source/v1
metadata:
schema: metadata/Document/v1
name: source
storagePolicy: cleartext
layeringDefinition:
abstract: false
layer: one
@ -20,6 +21,7 @@ schema: example/Middle/v1
metadata:
schema: metadata/Document/v1
name: middle
storagePolicy: cleartext
layeringDefinition:
abstract: false
layer: one
@ -36,6 +38,7 @@ schema: example/Dest/v1
metadata:
schema: metadata/Document/v1
name: dest
storagePolicy: cleartext
layeringDefinition:
abstract: false
layer: one

2
deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-2-layers.yaml
View File

@ -12,6 +12,7 @@ schema: example/Kind/v1
metadata:
schema: metadata/Document/v1
name: global-1234
storagePolicy: cleartext
labels:
key1: value1
key2: value2
@ -27,6 +28,7 @@ schema: example/Kind/v1
metadata:
schema: metadata/Document/v1
name: site-1234
storagePolicy: cleartext
layeringDefinition:
layer: site
parentSelector:

3
deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-3-layers.yaml
View File

@ -13,6 +13,7 @@ schema: example/Kind/v1
metadata:
schema: metadata/Document/v1
name: global-1234
storagePolicy: cleartext
labels:
key1: value1
layeringDefinition:
@ -27,6 +28,7 @@ schema: example/Kind/v1
metadata:
schema: metadata/Document/v1
name: region-1234
storagePolicy: cleartext
labels:
key1: value1
layeringDefinition:
@ -45,6 +47,7 @@ schema: example/Kind/v1
metadata:
schema: metadata/Document/v1
name: site-1234
storagePolicy: cleartext
labels:
foo: bar
baz: qux

1
deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-split-bucket-a.yaml
View File

@ -3,6 +3,7 @@ schema: example/Kind/v1
metadata:
schema: metadata/Document/v1
name: global-1234
storagePolicy: cleartext
labels:
key1: value1
layeringDefinition:

3
deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-split-bucket-b.yaml
View File

@ -3,6 +3,7 @@ schema: example/Kind/v1
metadata:
schema: metadata/Document/v1
name: region-1234
storagePolicy: cleartext
labels:
key1: value1
layeringDefinition:
@ -21,6 +22,7 @@ schema: example/Kind/v1
metadata:
schema: metadata/Document/v1
name: site-with-merge-action
storagePolicy: cleartext
labels:
foo: bar
baz: qux
@ -38,6 +40,7 @@ schema: example/Kind/v1
metadata:
schema: metadata/Document/v1
name: site-with-delete-action
storagePolicy: cleartext
layeringDefinition:
layer: site
parentSelector:

2
deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-with-delete.yaml
View File

@ -13,6 +13,7 @@ schema: example/Kind/v1
metadata:
schema: metadata/Document/v1
name: global-1234
storagePolicy: cleartext
labels:
key1: value1
layeringDefinition:
@ -27,6 +28,7 @@ schema: example/Kind/v1
metadata:
schema: metadata/Document/v1
name: site-1234
storagePolicy: cleartext
layeringDefinition:
layer: site
parentSelector:

3
deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-with-update.yaml
View File

@ -13,6 +13,7 @@ schema: example/Kind/v1
metadata:
schema: metadata/Document/v1
name: global-1234
storagePolicy: cleartext
labels:
key1: value1
layeringDefinition:
@ -27,6 +28,7 @@ schema: example/Kind/v1
metadata:
schema: metadata/Document/v1
name: region-1234
storagePolicy: cleartext
labels:
key1: value1
layeringDefinition:
@ -45,6 +47,7 @@ schema: example/Kind/v1
metadata:
schema: metadata/Document/v1
name: site-1234
storagePolicy: cleartext
layeringDefinition:
layer: site
parentSelector:

1
deckhand/tests/functional/gabbits/resources/design-doc-substitution-generic-sample.yaml
View File

@ -41,6 +41,7 @@ schema: armada/Chart/v1
metadata:
name: example-chart-01
schema: metadata/Document/v1
storagePolicy: cleartext
layeringDefinition:
layer: region
substitutions:

1
deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-b.yaml
View File

@ -31,6 +31,7 @@ schema: armada/Chart/v1
metadata:
name: example-chart-01
schema: metadata/Document/v1
storagePolicy: cleartext
layeringDefinition:
layer: region
substitutions:

1
deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample.yaml
View File

@ -41,6 +41,7 @@ schema: armada/Chart/v1
metadata:
name: example-chart-01
schema: metadata/Document/v1
storagePolicy: cleartext
layeringDefinition:
layer: region
substitutions:

3
deckhand/tests/functional/gabbits/resources/layering-and-substitution-dag-sample.yaml
View File

@ -35,6 +35,7 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: armada-chart-03
storagePolicy: cleartext
layeringDefinition:
abstract: false
layer: site
@ -72,6 +73,7 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: armada-chart-02
storagePolicy: cleartext
layeringDefinition:
abstract: false
layer: site
@ -109,6 +111,7 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: armada-chart-01
storagePolicy: cleartext
labels:
key1: value1
layeringDefinition:

5
deckhand/tests/functional/gabbits/resources/layering-and-substitution-sample.yaml
View File

@ -3,6 +3,7 @@ schema: deckhand/Certificate/v1
metadata:
name: example-cert
schema: metadata/Document/v1
storagePolicy: cleartext
layeringDefinition:
layer: site
storagePolicy: cleartext
@ -13,6 +14,7 @@ schema: deckhand/CertificateKey/v1
metadata:
name: example-key
schema: metadata/Document/v1
storagePolicy: cleartext
layeringDefinition:
layer: site
storagePolicy: cleartext
@ -23,6 +25,7 @@ schema: deckhand/Passphrase/v1
metadata:
name: example-password
schema: metadata/Document/v1
storagePolicy: cleartext
layeringDefinition:
abstract: false
layer: site
@ -33,6 +36,7 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: armada-chartgroup-01
storagePolicy: cleartext
labels:
key1: value1
layeringDefinition:
@ -45,6 +49,7 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: armada-chart-01
storagePolicy: cleartext
layeringDefinition:
layer: site
parentSelector:

2
deckhand/tests/functional/gabbits/resources/layering-needs-substitution-source.yaml
View File

@ -13,6 +13,7 @@ schema: example/Kind/v1
metadata:
schema: metadata/Document/v1
name: abstract-1234
storagePolicy: cleartext
labels:
key1: value1
layeringDefinition:
@ -27,6 +28,7 @@ schema: example/Kind/v1
metadata:
schema: metadata/Document/v1
name: concrete-1234
storagePolicy: cleartext
layeringDefinition:
layer: site
parentSelector:

3
deckhand/tests/functional/gabbits/resources/passphrase.yaml
View File

@ -4,5 +4,8 @@ metadata:
schema: metadata/Document/v1
name: my-passphrase
storagePolicy: cleartext
layeringDefinition:
abstract: false
layer: site
data: not-a-real-password
...

2
deckhand/tests/functional/gabbits/resources/replacement.yaml
View File

@ -32,6 +32,7 @@ schema: armada/Chart/v1
metadata:
name: example-chart-01
schema: metadata/Document/v1
storagePolicy: cleartext
labels:
name: parent-chart
layeringDefinition:
@ -53,6 +54,7 @@ schema: armada/Chart/v1
metadata:
name: example-chart-01
schema: metadata/Document/v1
storagePolicy: cleartext
replacement: true
layeringDefinition:
layer: site

1
deckhand/tests/functional/gabbits/resources/sample-doc.yaml
View File

@ -3,6 +3,7 @@ schema: promenade/ResourceType/v1
metadata:
schema: metadata/Document/v1
name: a-unique-config-name-12345
storagePolicy: cleartext
labels:
component: apiserver
hostname: server0

1
deckhand/tests/functional/gabbits/resources/substitution-results-in-none-bug.yaml
View File

@ -22,6 +22,7 @@ schema: deckhand/Dest/v1
metadata:
name: dest
schema: metadata/Document/v1
storagePolicy: cleartext
layeringDefinition:
layer: one
substitutions:

7
deckhand/tests/functional/gabbits/revision-diff/revision-diff-success.yaml
View File

@ -39,6 +39,7 @@ tests:
metadata:
schema: metadata/Document/v1
name: doc-a
storagePolicy: cleartext
layeringDefinition:
abstract: false
layer: site
@ -56,6 +57,7 @@ tests:
metadata:
schema: metadata/Document/v1
name: doc-b
storagePolicy: cleartext
layeringDefinition:
abstract: false
layer: site
@ -73,6 +75,7 @@ tests:
metadata:
schema: metadata/Document/v1
name: doc-c
storagePolicy: cleartext
layeringDefinition:
abstract: false
layer: site
@ -90,6 +93,7 @@ tests:
metadata:
schema: metadata/Document/v1
name: doc-d
storagePolicy: cleartext
layeringDefinition:
abstract: false
layer: site
@ -157,6 +161,7 @@ tests:
metadata:
schema: metadata/Document/v1
name: doc-c
storagePolicy: cleartext
layeringDefinition:
abstract: false
layer: site
@ -192,6 +197,7 @@ tests:
metadata:
schema: metadata/Document/v1
name: doc-m
storagePolicy: cleartext
layeringDefinition:
abstract: false
layer: site
@ -231,6 +237,7 @@ tests:
metadata:
schema: metadata/Document/v1
name: doc-e
storagePolicy: cleartext
layeringDefinition:
abstract: false
layer: site

4
deckhand/tests/functional/gabbits/schema-validation/schema-validation-success.yaml
View File

@ -73,6 +73,7 @@ tests:
metadata:
schema: metadata/Document/v1
name: good
storagePolicy: cleartext
layeringDefinition:
abstract: false
layer: site
@ -114,6 +115,7 @@ tests:
metadata:
schema: metadata/Document/v1
name: bad
storagePolicy: cleartext
layeringDefinition:
abstract: false
layer: site
@ -173,6 +175,7 @@ tests:
metadata:
schema: metadata/Document/v1
name: bad
storagePolicy: cleartext
layeringDefinition:
abstract: false
layer: site
@ -191,6 +194,7 @@ tests:
metadata:
name: test-certificate
schema: metadata/Document/v1
storagePolicy: cleartext
layeringDefinition:
layer: site
storagePolicy: cleartext

1
deckhand/tests/functional/gabbits/substitution/substitution-source-feeds-multi-dest.yaml
View File

@ -50,6 +50,7 @@ tests:
metadata:
name: example-chart-01
schema: metadata/Document/v1
storagePolicy: cleartext
layeringDefinition:
layer: site
substitutions:

1
deckhand/tests/unit/control/test_revision_documents_controller.py
View File

@ -38,6 +38,7 @@ schema: aic/Versions/v1
metadata:
name: with-anchor
schema: metadata/Document/v1
storagePolicy: cleartext
labels:
selector: foo1
layeringDefinition:

1
deckhand/tests/unit/control/test_validations_controller.py
View File

@ -678,6 +678,7 @@ class TestValidationsControllerPostValidate(BaseValidationsControllerTest):
'error_section': {
'data': {'a': 'fail'},
'metadata': {'labels': {'global': 'global1'},
'storagePolicy': 'cleartext',
'layeringDefinition': {'abstract': False,
'layer': 'global'},
'name': doc_to_test['metadata']['name'],

13
deckhand/tests/unit/engine/test_document_layering_and_replacement.py
View File

@ -30,6 +30,7 @@ schema: deckhand/LayeringPolicy/v1
metadata:
schema: metadata/Control/v1
name: layering-policy
storagePolicy: cleartext
data:
layerOrder:
- global
@ -39,6 +40,7 @@ schema: aic/Versions/v1
metadata:
schema: metadata/Document/v1
name: a
storagePolicy: cleartext
labels:
selector: foo
layeringDefinition:
@ -52,6 +54,7 @@ schema: aic/Versions/v1
metadata:
schema: metadata/Document/v1
name: a
storagePolicy: cleartext
labels:
selector: baz
replacement: true
@ -71,6 +74,7 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: c
storagePolicy: cleartext
layeringDefinition:
abstract: False
layer: global
@ -124,6 +128,7 @@ schema: deckhand/LayeringPolicy/v1
metadata:
schema: metadata/Control/v1
name: layering-policy
storagePolicy: cleartext
data:
layerOrder:
- global
@ -133,6 +138,7 @@ schema: aic/Versions/v1
metadata:
schema: metadata/Document/v1
name: a
storagePolicy: cleartext
labels:
selector: foo
layeringDefinition:
@ -149,6 +155,7 @@ metadata:
labels:
selector: baz
replacement: true
storagePolicy: cleartext
layeringDefinition:
abstract: False
layer: site
@ -165,6 +172,7 @@ schema: aic/Versions/v1
metadata:
schema: metadata/Document/v1
name: b
storagePolicy: cleartext
labels:
selector: qux
layeringDefinition:
@ -183,6 +191,7 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: c
storagePolicy: cleartext
layeringDefinition:
abstract: False
layer: global
@ -265,6 +274,7 @@ schema: deckhand/LayeringPolicy/v1
metadata:
schema: metadata/Control/v1
name: layering-policy
storagePolicy: cleartext
data:
layerOrder:
- global
@ -275,6 +285,7 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: nova-global
storagePolicy: cleartext
labels:
name: nova-global
component: nova
@ -291,6 +302,7 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: nova
storagePolicy: cleartext
labels:
name: nova-5ec
component: nova
@ -308,6 +320,7 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
replacement: true
storagePolicy: cleartext
name: nova
layeringDefinition:
abstract: false

6
deckhand/tests/unit/engine/test_document_validation.py
View File

@ -66,8 +66,10 @@ class TestDocumentValidation(engine_test_base.TestDocumentValidationBase):
self, mock_jsonschema):
m_args = mock.Mock()
mock_jsonschema.Draft4Validator(m_args).iter_errors.side_effect = [
# Return empty list of errors for base schema validator and pretend
# that 1 error is returned for next validator.
# Return empty list of errors for base schema and metadata
# validator and pretend that 1 error is returned for next
# validator.
[],
[],
[mock.Mock(path=[], schema_path=[], message='scary-secret-here')]
]

78
deckhand/tests/unit/engine/test_document_validation_negative.py
View File

@ -23,26 +23,39 @@ from deckhand import types
class TestDocumentValidationNegative(test_base.TestDocumentValidationBase):
"""Negative testing suite for document validation."""
BASIC_PROPERTIES = (
# Control documents don't require layeringDefinition as none of them
# are rendered -- they are static documents. It is also not meaningful
# to encrypt control documents.
BASIC_CONTROL_PROPERTIES = (
'metadata',
'metadata.schema',
'metadata.name',
'metadata.layeringDefinition',
'metadata.layeringDefinition.layer',
'schema'
'schema',
)
CRITICAL_PROPERTIES = (
BASIC_DOCUMENT_PROPERTIES = BASIC_CONTROL_PROPERTIES + (
'metadata.layeringDefinition',
'metadata.layeringDefinition.layer',
'metadata.storagePolicy',
)
CRITICAL_CONTROL_PROPERTIES = (
'schema',
'metadata',
'metadata.schema',
'metadata.name',
)
CRITICAL_DOCUMENT_PROPERTIES = CRITICAL_CONTROL_PROPERTIES + (
'metadata.layeringDefinition',
'metadata.layeringDefinition.layer',
'metadata.substitutions.0.dest',
'metadata.substitutions.0.dest.path',
'metadata.substitutions.0.src',
'metadata.substitutions.0.src.schema',
'metadata.substitutions.0.src.name',
'metadata.substitutions.0.src.path'
'metadata.substitutions.0.src.path',
'metadata.storagePolicy',
)
def _do_validations(self, document_validator, expected, expected_err):
@ -68,11 +81,19 @@ class TestDocumentValidationNegative(test_base.TestDocumentValidationBase):
validations[-1]['errors'][-1]['message'])
def _test_missing_required_sections(self, document, properties_to_remove):
if document['metadata']['schema'].startswith(types.CONTROL):
critial_properties = self.CRITICAL_CONTROL_PROPERTIES
elif document['metadata']['schema'].startswith(types.DOCUMENT):
critial_properties = self.CRITICAL_DOCUMENT_PROPERTIES
else:
self.fail('Document `metadata.schema` must start with '
'"metadata/Document" or "metadata/Control".')
for idx, property_to_remove in enumerate(properties_to_remove):
missing_prop = property_to_remove.split('.')[-1]
invalid_data = self._corrupt_data(document, property_to_remove)
exception_raised = property_to_remove in self.CRITICAL_PROPERTIES
exception_raised = property_to_remove in critial_properties
expected_err_msg = "'%s' is a required property" % missing_prop
payload = [invalid_data]
@ -87,42 +108,36 @@ class TestDocumentValidationNegative(test_base.TestDocumentValidationBase):
def test_certificate_authority_key_missing_required_sections(self):
document = self._read_data('sample_certificate_authority_key')
properties_to_remove = tuple(self.BASIC_PROPERTIES) + (
properties_to_remove = tuple(self.BASIC_DOCUMENT_PROPERTIES) + (
'metadata.storagePolicy',)
self._test_missing_required_sections(document, properties_to_remove)
def test_certificate_authority_missing_required_sections(self):
document = self._read_data('sample_certificate_authority')
properties_to_remove = tuple(self.BASIC_PROPERTIES) + (
properties_to_remove = tuple(self.BASIC_DOCUMENT_PROPERTIES) + (
'metadata.storagePolicy',)
self._test_missing_required_sections(document, properties_to_remove)
def test_certificate_key_missing_required_sections(self):
document = self._read_data('sample_certificate_key')
properties_to_remove = tuple(self.BASIC_PROPERTIES) + (
properties_to_remove = tuple(self.BASIC_DOCUMENT_PROPERTIES) + (
'metadata.storagePolicy',)
self._test_missing_required_sections(document, properties_to_remove)
def test_certificate_missing_required_sections(self):
document = self._read_data('sample_certificate')
properties_to_remove = tuple(self.BASIC_PROPERTIES) + (
properties_to_remove = tuple(self.BASIC_DOCUMENT_PROPERTIES) + (
'metadata.storagePolicy',)
self._test_missing_required_sections(document, properties_to_remove)
def test_data_schema_missing_required_sections(self):
properties_to_remove = (
'metadata',
'metadata.schema',
'metadata.name',
'schema',
'data.$schema'
)
document = self._read_data('sample_data_schema')
properties_to_remove = tuple(self.BASIC_CONTROL_PROPERTIES)
self._test_missing_required_sections(document, properties_to_remove)
def test_generic_document_missing_required_sections(self):
document = self._read_data('sample_document')
properties_to_remove = self.CRITICAL_PROPERTIES
properties_to_remove = self.CRITICAL_DOCUMENT_PROPERTIES
self._test_missing_required_sections(document, properties_to_remove)
def test_generic_document_missing_multiple_required_sections(self):
@ -152,6 +167,12 @@ class TestDocumentValidationNegative(test_base.TestDocumentValidationBase):
error_re = r"%s is a required property" % missing_property
self.assertRegex(str(e.error_list).replace("\'", ""), error_re)
def test_layering_policy_missing_required_sections(self):
properties_to_remove = tuple(self.BASIC_CONTROL_PROPERTIES) + (
'data.layerOrder',)
document = self._read_data('sample_layering_policy')
self._test_missing_required_sections(document, properties_to_remove)
def test_document_invalid_layering_definition_action(self):
document = self._read_data('sample_document')
missing_data = self._corrupt_data(
@ -166,38 +187,27 @@ class TestDocumentValidationNegative(test_base.TestDocumentValidationBase):
doc_validator.validate_all)
self.assertRegex(str(e.error_list[0]).replace("\'", ""), error_re)
def test_layering_policy_missing_required_sections(self):
properties_to_remove = (
'metadata',
'metadata.schema',
'metadata.name',
'schema',
'data.layerOrder'
)
document = self._read_data('sample_layering_policy')
self._test_missing_required_sections(document, properties_to_remove)
def test_passphrase_missing_required_sections(self):
document = self._read_data('sample_passphrase')
properties_to_remove = tuple(self.BASIC_PROPERTIES) + (
properties_to_remove = tuple(self.BASIC_DOCUMENT_PROPERTIES) + (
'metadata.storagePolicy',)
self._test_missing_required_sections(document, properties_to_remove)
def test_privatekey_missing_required_sections(self):
document = self._read_data('sample_private_key')
properties_to_remove = tuple(self.BASIC_PROPERTIES) + (
properties_to_remove = tuple(self.BASIC_DOCUMENT_PROPERTIES) + (
'metadata.storagePolicy',)
self._test_missing_required_sections(document, properties_to_remove)
def test_publickey_missing_required_sections(self):
document = self._read_data('sample_public_key')
properties_to_remove = tuple(self.BASIC_PROPERTIES) + (
properties_to_remove = tuple(self.BASIC_DOCUMENT_PROPERTIES) + (
'metadata.storagePolicy',)
self._test_missing_required_sections(document, properties_to_remove)
def test_validation_policy_missing_required_sections(self):
document = self._read_data('sample_validation_policy')
properties_to_remove = tuple(self.BASIC_PROPERTIES) + (
properties_to_remove = tuple(self.BASIC_CONTROL_PROPERTIES) + (
'data.validations', 'data.validations.0.name')
self._test_missing_required_sections(document, properties_to_remove)

1
deckhand/tests/unit/resources/sample_document.yaml
View File

@ -4,6 +4,7 @@ schema: promenade/ResourceType/v1
metadata:
schema: metadata/Document/v1
name: a-unique-config-name-12345
storagePolicy: cleartext
labels:
component: apiserver
hostname: server0

1
deckhand/tests/unit/resources/sample_document_simple.yaml
View File

@ -3,6 +3,7 @@ schema: armada/Manifest/v1
metadata:
schema: metadata/Document/v1
name: cluster-bootstrap
storagePolicy: cleartext
layeringDefinition:
abstract: false
layer: site

9
deckhand/types.py
View File

@ -59,3 +59,12 @@ ENCRYPTION_TYPES = (
'cleartext',
'encrypted',
)
METADATA_SCHEMA_TYPES = (
CONTROL,
DOCUMENT
) = (
'metadata/Control',
'metadata/Document'
)

BIN
doc/source/images/architecture-pegleg.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 37 KiB

After

Width:  |  Height:  |  Size: 37 KiB

BIN
doc/source/images/architecture.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 23 KiB

After

Width:  |  Height:  |  Size: 23 KiB

45
doc/source/validation.rst
View File

@ -218,9 +218,41 @@ Base Schema
This schema is used to sanity-check all documents that are passed to
Deckhand. Failure to pass this schema results in a critical error.
Metadata Schemas
----------------
Metadata schemas validate the ``metadata`` section of every document
ingested by Deckhand.
* ``Metadata Control`` schema.
JSON schema against which the metadata section of each ``metadata/Control``
document type is validated. Applies to all static documents meant to
configure Deckhand behavior, like LayeringPolicy, ValidationPolicy,
and DataSchema documents.
.. literalinclude:: ../../deckhand/engine/schemas/metadata_control.yaml
:language: yaml
:lines: 15-
:caption: Schema for ``metadata/Control`` metadata document sections.
* ``Metadata Document`` schema.
JSON schema against which the metadata section of each ``metadata/Document``
document type is validated. Applies to all site definition documents or
"regular" documents that require rendering.
.. literalinclude:: ../../deckhand/engine/schemas/metadata_document.yaml
:language: yaml
:lines: 15-
:caption: Schema for ``metadata/Document`` metadata document sections.
DataSchema Schemas
------------------
DataSchema schemas validate the ``data`` section of every document ingested
by Deckhand.
All schemas below are ``DataSchema`` documents. They define additional
properties not included in the base schema or override default properties in
the base schema.
@ -284,19 +316,6 @@ corresponding to the created revision.
This schema is used to sanity-check all ``Certificate`` documents that are
passed to Deckhand.
* ``DataSchema`` schema.
JSON schema against which all documents with ``deckhand/DataSchema/v1``
schema are validated.
.. literalinclude:: ../../deckhand/engine/schemas/dataschema_schema.yaml
:language: yaml
:lines: 15-
:caption: Schema for ``DataSchema`` documents.
This schema is used to sanity-check all ``DataSchema`` documents that are
passed to Deckhand.
* ``LayeringPolicy`` schema.
JSON schema against which all documents with ``deckhand/LayeringPolicy/v1``