airship/deckhand
Code Issues Proposed changes

Support filtering revision (documents) by any legal filter

Browse Source 
This PS adds support for filtering revisions and
revision documents documents by any legal filter
(those enumerated in the design document).

Deckhand now supports the following filter arguments:
  * schema
  * metadata.name
  * metadata.label
  * metadata.layeringDefinition.abstract
  * metadata.layeringDefinition.layer
  * status.bucket

Deckhand now supports the following filter arguments for filtering
revisions:
  * tag

Deckhand now supports multiple filters, e.g.:
  * ?metdata.layeringDefinition.layer=site&metadata.name=foo

Deckhand now supports repeated filters, e.g.:
  * ?metadata.label=foo=bar&metadata.label=baz=qux

The following has yet to be implemented will be done in a future
follow-up PS:
  - support sorting by specific keywords as well
  - support query limit and offset filters

Change-Id: I8558481e075715fe7fab98140094d37782a986d9
This commit is contained in:
Felipe Monteiro 2017-09-29 20:15:10 +01:00
parent ddc6d40584
commit 3e62ace8ed
14 changed files with 458 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
deckhand/control/README.rst
View File

@ -164,4 +164,4 @@ Document creation can be tested locally using (from root deckhand directory):
--data-binary "@deckhand/tests/unit/resources/sample_document.yaml"
# revision_id copy/pasted from previous response.
$ curl -i -X GET localhost:9000/api/v1.0/revisions/0e99c8b9-bab4-4fc7-8405-7dbd22c33a30/documents
$ curl -i -X GET localhost:9000/api/v1.0/revisions/1

40
deckhand/control/common.py
View File

@ -34,14 +34,52 @@ def sanitize_params(allowed_params):
:param allowed_params: The request's query string parameters.
"""
# A mapping between the filter keys users provide and the actual DB
# representation of the filter.
_mapping = {
# Mappings for revision documents.
'status.bucket': 'bucket_name',
'metadata.label': 'metadata.labels',
# Mappings for revisions.
'tag': 'tags.[*].tag'
}
def decorator(func):
@functools.wraps(func)
def wrapper(self, req, *func_args, **func_kwargs):
req_params = req.params or {}
sanitized_params = {}
for key in req_params.keys():
def _convert_to_dict(sanitized_params, filter_key, filter_val):
# Key-value pairs like metadata.label=foo=bar need to be
# converted to {'metadata.label': {'foo': 'bar'}} because
# 'metadata.labels' in a document is a dictionary. Later,
# we can check whether the filter dict is a subset of the
# actual dict for metadata labels.
for val in list(filter_val):
if '=' in val:
sanitized_params.setdefault(filter_key, {})
pair = val.split('=')
try:
sanitized_params[filter_key][pair[0]] = pair[1]
except IndexError:
pass
for key, val in req_params.items():
if not isinstance(val, list):
val = [val]
is_key_val_pair = '=' in val[0]
if key in allowed_params:
if key in _mapping:
if is_key_val_pair:
_convert_to_dict(
sanitized_params, _mapping[key], val)
else:
sanitized_params[_mapping[key]] = req_params[key]
else:
if is_key_val_pair:
_convert_to_dict(sanitized_params, key, val)
else:
sanitized_params[key] = req_params[key]
func_args = func_args + (sanitized_params,)

3
deckhand/control/revision_documents.py
View File

@ -31,7 +31,8 @@ class RevisionDocumentsResource(api_base.BaseResource):
@common.sanitize_params([
'schema', 'metadata.name', 'metadata.layeringDefinition.abstract',
'metadata.layeringDefinition.layer', 'metadata.label'])
'metadata.layeringDefinition.layer', 'metadata.label',
'status.bucket'])
def on_get(self, req, resp, sanitized_params, revision_id):
"""Returns all documents for a `revision_id`.

6
deckhand/control/revisions.py
View File

@ -15,6 +15,7 @@
import falcon
from deckhand.control import base as api_base
from deckhand.control import common
from deckhand.control.views import revision as revision_view
from deckhand.db.sqlalchemy import api as db_api
from deckhand import errors
@ -53,8 +54,9 @@ class RevisionsResource(api_base.BaseResource):
resp.append_header('Content-Type', 'application/x-yaml')
resp.body = self.to_yaml_body(revision_resp)
def _list_revisions(self, req, resp):
revisions = db_api.revision_get_all()
@common.sanitize_params(['tag'])
def _list_revisions(self, req, resp, sanitized_params):
revisions = db_api.revision_get_all(**sanitized_params)
revisions_resp = self.view_builder.list(revisions)
resp.status = falcon.HTTP_200

171
deckhand/db/sqlalchemy/api.py
View File

@ -179,8 +179,7 @@ def _documents_create(bucket_name, values_list, session=None):
return document
for values in values_list:
values['_metadata'] = values.pop('metadata')
values['name'] = values['_metadata']['name']
values = _fill_in_metadata_defaults(values)
values['is_secret'] = 'secret' in values['data']
# Hash the document's metadata and data to later efficiently check
@ -228,6 +227,20 @@ def _documents_create(bucket_name, values_list, session=None):
return changed_documents
def _fill_in_metadata_defaults(values):
values['_metadata'] = values.pop('metadata')
values['name'] = values['_metadata']['name']
if not values['_metadata'].get('storagePolicy', None):
values['_metadata']['storagePolicy'] = 'cleartext'
if ('layeringDefinition' in values['_metadata']
and 'abstract' not in values['_metadata']['layeringDefinition']):
values['_metadata']['layeringDefinition']['abstract'] = False
return values
def _make_hash(data):
return hashlib.sha256(
json.dumps(data, sort_keys=True).encode('utf-8')).hexdigest()
@ -291,6 +304,7 @@ def bucket_get_or_create(bucket_name, session=None):
####################
def revision_create(session=None):
"""Create a revision.
@ -354,7 +368,63 @@ def _update_revision_history(documents):
return documents
def revision_get_all(session=None):
def _apply_filters(dct, **filters):
"""Apply filters to ``dct``.
Apply filters in ``filters`` to the dictionary ``dct``.
:param dct: The dictionary to check against all the ``filters``.
:param filters: Dictionary of key-value pairs used for filtering out
unwanted results.
:return: True if the dictionary satisfies all the filters, else False.
"""
def _transform_filter_bool(actual_val, filter_val):
# Transform boolean values into string literals.
if (isinstance(actual_val, bool)
and isinstance(filter_val, six.string_types)):
try:
filter_val = ast.literal_eval(filter_val.title())
except ValueError:
# If not True/False, set to None to avoid matching
# `actual_val` which is always boolean.
filter_val = None
return filter_val
match = True
for filter_key, filter_val in filters.items():
actual_val = utils.jsonpath_parse(dct, filter_key)
# If the filter is a list of possibilities, e.g. ['site', 'region']
# for metadata.layeringDefinition.layer, check whether the actual
# value is present.
if isinstance(filter_val, (list, tuple)):
if actual_val not in [_transform_filter_bool(actual_val, x)
for x in filter_val]:
match = False
break
else:
# Else if both the filter value and the actual value in the doc
# are dictionaries, check whether the filter dict is a subset
# of the actual dict.
if (isinstance(actual_val, dict)
and isinstance(filter_val, dict)):
is_subset = set(
filter_val.items()).issubset(set(actual_val.items()))
if not is_subset:
match = False
break
else:
# Else both filters are string literals.
if actual_val != _transform_filter_bool(
actual_val, filter_val):
match = False
break
return match
def revision_get_all(session=None, **filters):
"""Return list of all revisions.
:param session: Database session object.
@ -364,11 +434,15 @@ def revision_get_all(session=None):
revisions = session.query(models.Revision)\
.all()
revisions_dict = [r.to_dict() for r in revisions]
for revision in revisions_dict:
revision['documents'] = _update_revision_history(revision['documents'])
result = []
for revision in revisions:
revision_dict = revision.to_dict()
if _apply_filters(revision_dict, **filters):
revision_dict['documents'] = _update_revision_history(
revision_dict['documents'])
result.append(revision_dict)
return revisions_dict
return result
def revision_delete_all(session=None):
@ -382,6 +456,39 @@ def revision_delete_all(session=None):
.delete(synchronize_session=False)
def _filter_revision_documents(documents, unique_only, **filters):
"""Return the list of documents that match filters.
:param unique_only: Return only unique documents if ``True``.
:param filters: Dictionary attributes (including nested) used to filter
out revision documents.
:returns: List of documents that match specified filters.
"""
# TODO(fmontei): Implement this as an sqlalchemy query.
filtered_documents = {}
unique_filters = ('name', 'schema')
for document in documents:
# NOTE(fmontei): Only want to include non-validation policy documents
# for this endpoint.
if document['schema'] == types.VALIDATION_POLICY_SCHEMA:
continue
if _apply_filters(document, **filters):
# Filter out redundant documents from previous revisions, i.e.
# documents schema and metadata.name are repeated.
if unique_only:
unique_key = tuple(
[document[filter] for filter in unique_filters])
else:
unique_key = document['id']
if unique_key not in filtered_documents:
filtered_documents[unique_key] = document
# TODO(fmontei): Sort by user-specified parameter.
return sorted(filtered_documents.values(), key=lambda d: d['created_at'])
@require_revision_exists
def revision_get_documents(revision_id=None, include_history=True,
unique_only=True, session=None, **filters):
@ -438,56 +545,6 @@ def revision_get_documents(revision_id=None, include_history=True,
return filtered_documents
def _filter_revision_documents(documents, unique_only, **filters):
"""Return the list of documents that match filters.
:param unique_only: Return only unique documents if ``True``.
:param filters: Dictionary attributes (including nested) used to filter
out revision documents.
:returns: List of documents that match specified filters.
"""
# TODO(fmontei): Implement this as an sqlalchemy query.
filtered_documents = {}
unique_filters = [c for c in models.Document.UNIQUE_CONSTRAINTS
if c != 'revision_id']
for document in documents:
# NOTE(fmontei): Only want to include non-validation policy documents
# for this endpoint.
if document['schema'] == types.VALIDATION_POLICY_SCHEMA:
continue
match = True
for filter_key, filter_val in filters.items():
actual_val = utils.multi_getattr(filter_key, document)
if (isinstance(actual_val, bool)
and isinstance(filter_val, six.string_types)):
try:
filter_val = ast.literal_eval(filter_val.title())
except ValueError:
# If not True/False, set to None to avoid matching
# `actual_val` which is always boolean.
filter_val = None
if actual_val != filter_val:
match = False
if match:
# Filter out redundant documents from previous revisions, i.e.
# documents schema and metadata.name are repeated.
if unique_only:
unique_key = tuple(
[document[filter] for filter in unique_filters])
else:
unique_key = document['id']
if unique_key not in filtered_documents:
filtered_documents[unique_key] = document
# TODO(fmontei): Sort by user-specified parameter.
return sorted(filtered_documents.values(), key=lambda d: d['created_at'])
# NOTE(fmontei): No need to include `@require_revision_exists` decorator as
# the this function immediately calls `revision_get_documents` for both
# revision IDs, which has the decorator applied to it.

3
deckhand/tests/functional/gabbits/resources/design-doc-layering-sample.yaml
View File

@ -45,6 +45,9 @@ schema: example/Kind/v1
metadata:
schema: metadata/Document/v1
name: site-1234
labels:
foo: bar
baz: qux
layeringDefinition:
layer: site
parentSelector:

88
deckhand/tests/functional/gabbits/revision-documents-filters.yaml Normal file
View File

@ -0,0 +1,88 @@
# 1. Test success paths for filtering revision documents for the following filters:
# * schema
# * metadata.name
# * metadata.label
# * metadata.layeringDefinition.abstract
# * metadata.layeringDefinition.layer
# * status.bucket
defaults:
request_headers:
content-type: application/x-yaml
response_headers:
content-type: application/x-yaml
tests:
- name: purge
desc: Begin testing from known state.
DELETE: /api/v1.0/revisions
status: 204
- name: initialize
desc: Create initial documents
PUT: /api/v1.0/bucket/mop/documents
status: 200
data: <@resources/design-doc-layering-sample.yaml
- name: filter_by_schema
desc: Verify revision documents filtered by schema
GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/documents?schema=deckhand/LayeringPolicy/v1
status: 200
response_multidoc_jsonpaths:
$.[0].metadata.name: layering-policy
$.[0].schema: deckhand/LayeringPolicy/v1
- name: filter_by_metadata_name
desc: Verify revision documents filtered by metadata.name
GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/documents?metadata.name=layering-policy
status: 200
response_multidoc_jsonpaths:
$.[0].metadata.name: layering-policy
- name: filter_by_metadata_label
desc: Verify revision documents filtered by metadata.name
GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/documents?metadata.label=key1=value1
status: 200
response_multidoc_jsonpaths:
$.[*].metadata.name:
- global-1234
- region-1234
$.[*].metadata.labels:
- key1: value1
- key1: value1
- name: filter_by_metadata_layeringdefinition_abstract
desc: Verify revision documents filtered by metadata.layeringDefinition.abstract
GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/documents?metadata.layeringDefinition.abstract=true
status: 200
response_multidoc_jsonpaths:
$.[*].metadata.name:
- global-1234
- region-1234
$.[*].metadata.layeringDefinition.abstract:
- true
- true
- name: filter_by_metadata_layeringdefinition_layer
desc: Verify revision documents filtered by metadata.layeringDefinition.layer
GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/documents?metadata.layeringDefinition.layer=site
status: 200
response_multidoc_jsonpaths:
$.[0].metadata.name: site-1234
$.[0].metadata.layeringDefinition.layer: site
- name: filter_by_bucket_name
desc: Verify revision documents filtered by status.bucket
GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/documents?status.bucket=mop
status: 200
response_multidoc_jsonpaths:
$.[*].metadata.name:
- layering-policy
- global-1234
- region-1234
- site-1234
$.[*].status.bucket:
- mop
- mop
- mop
- mop

65
deckhand/tests/functional/gabbits/revision-documents-multiple-filters.yaml Normal file
View File

@ -0,0 +1,65 @@
# 1. Test success paths for filtering revision documents using multiple filters
# for the following filters:
# * metadata.label
# * metadata.layeringDefinition.abstract
# * metadata.layeringDefinition.layer
# 2. Test success paths for multiple different-keyed filters.
# 3. Test success paths for multiple same-keyed filters.
defaults:
request_headers:
content-type: application/x-yaml
response_headers:
content-type: application/x-yaml
tests:
- name: purge
desc: Begin testing from known state.
DELETE: /api/v1.0/revisions
status: 204
- name: initialize
desc: Create initial documents
PUT: /api/v1.0/bucket/mop/documents
status: 200
data: <@resources/design-doc-layering-sample.yaml
- name: filter_by_multiple_different_filters_expect_site
desc: Verify revision documents filtered by multiple repeated keys that are different
GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/documents?metadata.layeringDefinition.layer=site&metadata.layeringDefinition.abstract=false
status: 200
response_multidoc_jsonpaths:
$.[0].metadata.name: site-1234
$.[0].metadata.layeringDefinition.layer: site
$.[0].metadata.layeringDefinition.abstract: false
- name: filter_by_multiple_different_filters_expect_region
desc: Verify revision documents filtered by multiple repeated keys that are different
GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/documents?metadata.layeringDefinition.layer=region&metadata.layeringDefinition.abstract=true
status: 200
response_multidoc_jsonpaths:
$.[0].metadata.name: region-1234
$.[0].metadata.layeringDefinition.layer: region
$.[0].metadata.layeringDefinition.abstract: true
- name: filter_by_repeated_metadata_layeringDefinition_layer
desc: Verify revision documents filtered by multiple repeated keys that are the same
GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/documents?metadata.layeringDefinition.layer=site&metadata.layeringDefinition.layer=region
status: 200
response_multidoc_jsonpaths:
$.[*].metadata.name:
- region-1234
- site-1234
$.[*].metadata.layeringDefinition.layer:
- region
- site
- name: filter_by_repeated_metadata_label
desc: Verify revision documents filtered by multiple repeated keys that are the same
GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/documents?metadata.label=foo=bar&metadata.label=baz=qux
status: 200
response_multidoc_jsonpaths:
$.[0].metadata.name: site-1234
$.[0].metadata.labels:
foo: bar
baz: qux

65
deckhand/tests/functional/gabbits/revision-filters.yaml Normal file
View File

@ -0,0 +1,65 @@
# 1. Test success paths for filtering revisions for the following filters:
# * tag
# 2. Test failure paths for filtering revisions for the following filters:
# * tag
defaults:
request_headers:
content-type: application/x-yaml
response_headers:
content-type: application/x-yaml
tests:
- name: purge
desc: Begin testing from known state.
DELETE: /api/v1.0/revisions
status: 204
- name: initialize
desc: Create initial documents
PUT: /api/v1.0/bucket/mop/documents
status: 200
data: <@resources/design-doc-layering-sample.yaml
- name: create_tag
desc: Create a tag for testing filtering a revision by tag
POST: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/tags/foo
status: 201
- name: create_another_tag
desc: Create another tag for testing filtering a revision by many tags
POST: /api/v1.0/revisions/$HISTORY['initialize'].$RESPONSE['$.[0].status.revision']/tags/bar
status: 201
- name: verify_revision_list_for_one_valid_filter
desc: Verify that revision is returned for filter tag="foo"
GET: /api/v1.0/revisions?tag=foo
status: 200
response_multidoc_jsonpaths:
$.[0].count: 1
$.[0].results[0].id: $HISTORY['initialize'].$RESPONSE['$.[0].status.revision']
$.[0].results[0].buckets: [mop]
$.[0].results[0].tags:
# Tags are sorted alphabetically.
- bar
- foo
- name: verify_revision_list_for_many_valid_filters
desc: Verify that revision is returned for filter tag="foo" or tag="bar"
GET: /api/v1.0/revisions?tag=foo&tag=bar
status: 200
response_multidoc_jsonpaths:
$.[0].count: 1
$.[0].results[0].id: $HISTORY['initialize'].$RESPONSE['$.[0].status.revision']
$.[0].results[0].buckets: [mop]
$.[0].results[0].tags:
- bar
- foo
- name: verify_revision_list_for_invalid_filter
desc: Verify that no revisions are returned for tag="baz"
GET: /api/v1.0/revisions?tag=baz
status: 200
response_multidoc_jsonpaths:
$.[0].count: 0
$.[0].results: []

33
deckhand/tests/unit/db/test_revision_documents.py Normal file
View File

@ -0,0 +1,33 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from deckhand.tests import test_utils
from deckhand.tests.unit.db import base
class TestRevisionDocumentsFiltering(base.TestDbBase):
def test_document_filtering_by_bucket_name(self):
document = base.DocumentFixture.get_minimal_fixture()
bucket_name = test_utils.rand_name('bucket')
self.create_documents(bucket_name, document)
revision_id = self.create_documents(bucket_name, [])[0]['revision_id']
filters = {'bucket_name': bucket_name}
retrieved_documents = self.list_revision_documents(
revision_id, **filters)
self.assertEqual(1, len(retrieved_documents))
self.assertEqual(bucket_name, retrieved_documents[0]['bucket_name'])

41
deckhand/utils.py
View File

@ -15,6 +15,8 @@
import re
import string
import jsonpath_ng
def to_camel_case(s):
"""Convert string to camel case."""
@ -28,35 +30,30 @@ def to_snake_case(name):
return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
def multi_getattr(multi_key, dict_data):
"""Iteratively check for nested attributes in the YAML data.
def jsonpath_parse(document, jsonpath):
"""Parse value given JSON path in the document.
Check for nested attributes included in "dest" attributes in the data
section of the YAML file. For example, a "dest" attribute of
".foo.bar.baz" should mean that the YAML data adheres to:
Retrieve the value corresponding to document[jsonpath] where ``jsonpath``
is a multi-part key. A multi-key is a series of keys and nested keys
concatenated together with ".". For exampple, ``jsonpath`` of
".foo.bar.baz" should mean that ``document`` has the format:
.. code-block:: yaml
---
foo:
bar:
baz: <data_to_be_substituted_here>
baz: <data_to_be_extracted_here>
:param multi_key: A multi-part key that references nested data in the
substitutable part of the YAML data, e.g. ".foo.bar.baz".
:param substitutable_data: The section of data in the YAML data that
is intended to be substituted with secrets.
:returns: nested entry in ``dict_data`` if present; else None.
:param document: Dictionary used for extracting nested entry.
:param jsonpath: A multi-part key that references nested data in a
dictionary.
:returns: Nested entry in ``document`` if present, else None.
"""
attrs = multi_key.split('.')
# Ignore the first attribute if it is "." as that is a self-reference.
if attrs[0] == '':
attrs = attrs[1:]
if jsonpath.startswith('.'):
jsonpath = '$' + jsonpath
data = dict_data
for attr in attrs:
if attr not in data:
return None
data = data.get(attr)
return data
p = jsonpath_ng.parse(jsonpath)
matches = p.find(document)
if matches:
return matches[0].value

5
doc/design.md
View File

@ -702,6 +702,11 @@ Lists existing revisions and reports basic details including a summary of
validation status for each `deckhand/ValidationPolicy` that is part of that
revision.
Supported query string parameters:
* `tag` - string, optional, repeatable - Used to select revisions that have
been tagged with particular tags.
Sample response:
```yaml

17
releasenotes/notes/revision-document-filtering-0e57274a4fc1bc07.yaml Normal file
View File

@ -0,0 +1,17 @@
---
features:
- |
Deckhand now supports the following filter arguments for filtering revision
documents:
* schema
* metadata.name
* metadata.label
* metadata.layeringDefinition.abstract
* metadata.layeringDefinition.layer
* status.bucket
Deckhand now supports the following filter arguments for filtering
revisions:
* tag

4
requirements.txt
View File

@ -11,16 +11,16 @@ PasteDeploy>=1.5.0 # MIT
Paste # MIT
Routes>=2.3.1 # MIT
jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT
six>=1.9.0 # MIT
oslo.concurrency>=3.8.0 # Apache-2.0
stevedore>=1.20.0 # Apache-2.0
jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT
python-keystoneclient>=3.8.0 # Apache-2.0
python-memcached==1.58
keystonemiddleware>=4.12.0 # Apache-2.0
psycopg2==2.7.3.1
uwsgi==2.0.15
jsonpath-ng==1.4.3
jsonschema==2.6.0
oslo.cache>=1.5.0 # Apache-2.0
oslo.concurrency>=3.8.0 # Apache-2.0