From 81b3e420137275d0e3f24894b42740a3a4aed89d Mon Sep 17 00:00:00 2001 From: Felipe Monteiro Date: Tue, 19 Sep 2017 16:05:06 +0100 Subject: [PATCH] Revision rollback API. This commit implements the revision rollback API, allowing users to rollback to a previous revision, whereby a new revision is created. An exception (400 Bad Request) is raised if the revision being rolled back to is identical to the latest revision or if no changes exist between the latest revision and the one being rolled back to. Included in this commit: - API endpoint for revision rollback. - Back-end logic for rolling back to a previous revision. The associated documents are also re-recreated. The revision_id assigned to each document depends on whether it has changed between the latest revision and the one being rolled back to: if changed, the new revision_id is assigned, else the original one, to maintain the correct revision history. - Associated unit tests. - Unskip all associated functional tests. Change-Id: I5c120a92e106544f7f8a4266fc386fb60622d6b3 --- deckhand/control/api.py | 4 +- deckhand/control/rollback.py | 39 ++++++ deckhand/db/sqlalchemy/api.py | 122 +++++++++++++++--- deckhand/errors.py | 6 + .../rollback-success-single-bucket.yaml | 35 ++--- deckhand/tests/unit/control/test_api.py | 9 +- deckhand/tests/unit/db/base.py | 3 + .../tests/unit/db/test_revision_rollback.py | 81 ++++++++++++ .../db/test_revision_rollback_negative.py | 47 +++++++ 9 files changed, 302 insertions(+), 44 deletions(-) create mode 100644 deckhand/control/rollback.py create mode 100644 deckhand/tests/unit/db/test_revision_rollback.py create mode 100644 deckhand/tests/unit/db/test_revision_rollback_negative.py diff --git a/deckhand/control/api.py b/deckhand/control/api.py index 3e0be64c..2c0f306b 100644 --- a/deckhand/control/api.py +++ b/deckhand/control/api.py @@ -24,6 +24,7 @@ from deckhand.control import revision_diffing from deckhand.control import revision_documents from deckhand.control import revision_tags from deckhand.control import revisions +from deckhand.control import rollback from deckhand.control import versions from deckhand.db.sqlalchemy import api as db_api @@ -68,7 +69,8 @@ def start_api(state_manager=None): revision_documents.RevisionDocumentsResource()), ('revisions/{revision_id}/tags', revision_tags.RevisionTagsResource()), ('revisions/{revision_id}/tags/{tag}', - revision_tags.RevisionTagsResource()) + revision_tags.RevisionTagsResource()), + ('rollback/{revision_id}', rollback.RollbackResource()) ] for path, res in v1_0_routes: diff --git a/deckhand/control/rollback.py b/deckhand/control/rollback.py new file mode 100644 index 00000000..a35af04f --- /dev/null +++ b/deckhand/control/rollback.py @@ -0,0 +1,39 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import falcon + +from deckhand.control import base as api_base +from deckhand.control.views import revision as revision_view +from deckhand.db.sqlalchemy import api as db_api +from deckhand import errors + + +class RollbackResource(api_base.BaseResource): + """API resource for realizing revision rollback.""" + + view_builder = revision_view.ViewBuilder() + + def on_post(self, req, resp, revision_id): + try: + revision = db_api.revision_rollback(revision_id) + except errors.RevisionNotFound as e: + raise falcon.HTTPNotFound(description=e.format_message()) + except errors.InvalidRollback as e: + raise falcon.HTTPBadRequest(description=e.format_message()) + + revision_resp = self.view_builder.show(revision) + resp.status = falcon.HTTP_201 + resp.append_header('Content-Type', 'application/x-yaml') + resp.body = self.to_yaml_body(revision_resp) diff --git a/deckhand/db/sqlalchemy/api.py b/deckhand/db/sqlalchemy/api.py index eee99b27..858877d7 100644 --- a/deckhand/db/sqlalchemy/api.py +++ b/deckhand/db/sqlalchemy/api.py @@ -144,8 +144,7 @@ def documents_create(bucket_name, documents, session=None): doc.save(session=session) doc.safe_delete(session=session) deleted_documents.append(doc) - - resp.extend([d.to_dict() for d in deleted_documents]) + resp.append(doc.to_dict()) if documents_to_create: LOG.debug('Creating documents: %s.', @@ -155,12 +154,11 @@ def documents_create(bucket_name, documents, session=None): doc['bucket_id'] = bucket['id'] doc['revision_id'] = revision['id'] doc.save(session=session) - - # NOTE(fmontei): The orig_revision_id is not copied into the - # revision_id for each created document, because the revision_id here - # should reference the just-created revision. In case the user needs - # the original revision_id, that is returned as well. - resp.extend([d.to_dict() for d in documents_to_create]) + resp.append(doc.to_dict()) + # NOTE(fmontei): The orig_revision_id is not copied into the + # revision_id for each created document, because the revision_id here + # should reference the just-created revision. In case the user needs + # the original revision_id, that is returned as well. return resp @@ -180,6 +178,13 @@ def _documents_create(bucket_name, values_list, session=None): for values in values_list: values['_metadata'] = values.pop('metadata') values['name'] = values['_metadata']['name'] + + # Hash the combination of the document's metadata and data to later + # efficiently check whether those data have changed. + dict_to_hash = values['_metadata'].copy() + dict_to_hash.update(values['data']) + values['hash'] = utils.make_hash(dict_to_hash) + values['is_secret'] = 'secret' in values['data'] # Hash the combination of the document's metadata and data to later # efficiently check whether those data have changed. @@ -302,6 +307,7 @@ def revision_create(session=None): def revision_get(revision_id, session=None): """Return the specified `revision_id`. + :param revision_id: The ID corresponding to the ``Revision`` object. :param session: Database session object. :returns: Dictionary representation of retrieved revision. :raises: RevisionNotFound if the revision was not found. @@ -335,6 +341,17 @@ def require_revision_exists(f): return wrapper +def _update_revision_history(documents): + # Since documents that are unchanged across revisions need to be saved for + # each revision, we need to ensure that the original revision is shown + # for the document's `revision_id` to maintain the correct revision + # history. + for doc in documents: + if doc['orig_revision_id']: + doc['revision_id'] = doc['orig_revision_id'] + return documents + + def revision_get_all(session=None): """Return list of all revisions. @@ -419,17 +436,6 @@ def revision_get_documents(revision_id=None, include_history=True, return filtered_documents -def _update_revision_history(documents): - # Since documents that are unchanged across revisions need to be saved for - # each revision, we need to ensure that the original revision is shown - # for the document's `revision_id` to maintain the correct revision - # history. - for doc in documents: - if doc['orig_revision_id']: - doc['revision_id'] = doc['orig_revision_id'] - return documents - - def _filter_revision_documents(documents, unique_only, **filters): """Return the list of documents that match filters. @@ -725,3 +731,81 @@ def revision_tag_delete_all(revision_id, session=None): session.query(models.RevisionTag)\ .filter_by(revision_id=revision_id)\ .delete(synchronize_session=False) + + +#################### + + +@require_revision_exists +def revision_rollback(revision_id, session=None): + """Rollback the latest revision to revision specified by ``revision_id``. + + Rolls back the latest revision to the revision specified by ``revision_id`` + thereby creating a new, carbon-copy revision. + + :param revision_id: Revision ID to which to rollback. + :returns: The newly created revision. + """ + session = session or get_session() + + # We know that the last revision exists, since require_revision_exists + # ensures revision_id exists, which at the very least is the last revision. + latest_revision = session.query(models.Revision)\ + .order_by(models.Revision.created_at.desc())\ + .first() + latest_revision_hashes = [d['hash'] for d in latest_revision['documents']] + + # If the rollback revision is the same as the latest revision, then there's + # no point in rolling back. + if latest_revision['id'] == revision_id: + raise errors.InvalidRollback(revision_id=revision_id) + + orig_revision = revision_get(revision_id, session=session) + + # A mechanism for determining whether a particular document has changed + # between revisions. Keyed with the document_id, the value is True if + # it has changed, else False. + doc_diff = {} + for orig_doc in orig_revision['documents']: + if orig_doc['hash'] not in latest_revision_hashes: + doc_diff[orig_doc['id']] = True + else: + doc_diff[orig_doc['id']] = False + + # If no changges have been made between the target revision to rollback to + # and the latest revision, raise an exception. + if set(doc_diff.values()) == set([False]): + raise errors.InvalidRollback(revision_id=revision_id) + + # Create the new revision, + new_revision = models.Revision() + with session.begin(): + new_revision.save(session=session) + + # Create the documents for the revision. + for orig_document in orig_revision['documents']: + orig_document['revision_id'] = new_revision['id'] + orig_document['_metadata'] = orig_document.pop('metadata') + + new_document = models.Document() + new_document.update({x: orig_document[x] for x in ( + 'name', '_metadata', 'data', 'hash', 'schema', 'bucket_id')}) + + new_document['revision_id'] = new_revision['id'] + + # If the document has changed, then use the revision_id of the new + # revision, otherwise use the original revision_id to preserve the + # revision history. + if doc_diff[orig_document['id']]: + new_document['orig_revision_id'] = new_revision['id'] + else: + new_document['orig_revision_id'] = orig_revision['id'] + + with session.begin(): + new_document.save(session=session) + + new_revision = new_revision.to_dict() + new_revision['documents'] = _update_revision_history( + new_revision['documents']) + + return new_revision diff --git a/deckhand/errors.py b/deckhand/errors.py index 307b5f9a..56f4ae6c 100644 --- a/deckhand/errors.py +++ b/deckhand/errors.py @@ -127,6 +127,12 @@ class RevisionTagBadFormat(DeckhandException): code = 400 +class InvalidRollback(DeckhandException): + msg_fmt = ("The requested rollback for target revision %(revision)s is " + "invalid as the latest revision matches the target revision.") + code = 400 + + class BarbicanException(DeckhandException): def __init__(self, message, code): diff --git a/deckhand/tests/functional/gabbits/rollback-success-single-bucket.yaml b/deckhand/tests/functional/gabbits/rollback-success-single-bucket.yaml index 1ca0a85a..4836c1ec 100644 --- a/deckhand/tests/functional/gabbits/rollback-success-single-bucket.yaml +++ b/deckhand/tests/functional/gabbits/rollback-success-single-bucket.yaml @@ -23,34 +23,29 @@ tests: desc: Begin testing from known state. DELETE: /api/v1.0/revisions status: 204 - skip: Not implemented. - name: initialize desc: Create initial documents PUT: /api/v1.0/bucket/mop/documents status: 200 data: <@resources/design-doc-layering-sample.yaml - skip: Not implemented. - name: update_single_document desc: Update a single document, ignore other documents in the bucket PUT: /api/v1.0/bucket/mop/documents status: 200 data: <@resources/design-doc-layering-sample-with-update.yaml - skip: Not implemented. - name: delete_document desc: Delete a single document PUT: /api/v1.0/bucket/mop/documents status: 200 data: <@resources/design-doc-layering-sample-with-delete.yaml - skip: Not implemented. - name: rollback desc: Rollback to revision 1 - POST: /api/v1.0/rollback/$HISTORY.$RESPONSE['$.documents[0].status.revision'] + POST: /api/v1.0/rollback/$HISTORY['initialize'].$RESPONSE['$.[0].status.revision'] status: 201 - skip: Not implemented. # Verify document history - name: verify_revision_1 @@ -59,22 +54,21 @@ tests: status: 200 response_multidoc_jsonpaths: $.[*].metadata.name: - - global-1234 - layering-policy + - global-1234 - region-1234 - site-1234 $.[*].status.revision: - - "$RESPONSE['$.[0].status.revision']" - - "$RESPONSE['$.[0].status.revision']" - - "$RESPONSE['$.[0].status.revision']" - - "$RESPONSE['$.[0].status.revision']" + - "$HISTORY['initialize'].$RESPONSE['$.[0].status.revision']" + - "$HISTORY['initialize'].$RESPONSE['$.[0].status.revision']" + - "$HISTORY['initialize'].$RESPONSE['$.[0].status.revision']" + - "$HISTORY['initialize'].$RESPONSE['$.[0].status.revision']" $.[*].status.bucket: - mop - mop - mop - mop $.[3].data.b: 4 - skip: Not implemented. - name: verify_revision_2 desc: Verify updated document count and revisions @@ -82,22 +76,21 @@ tests: status: 200 response_multidoc_jsonpaths: $.[*].metadata.name: - - global-1234 - layering-policy + - global-1234 - region-1234 - site-1234 $.[*].status.revision: - "$HISTORY['initialize'].$RESPONSE['$.[0].status.revision']" - "$HISTORY['initialize'].$RESPONSE['$.[0].status.revision']" - "$HISTORY['initialize'].$RESPONSE['$.[0].status.revision']" - - "$RESPONSE['$.[0].status.revision']" + - "$HISTORY['update_single_document'].$RESPONSE['$.[0].status.revision']" $.[*].status.bucket: - mop - mop - mop - mop $.[3].data.b: 5 - skip: Not implemented. - name: verify_revision_3 desc: Verify document deletion @@ -105,8 +98,8 @@ tests: status: 200 response_multidoc_jsonpaths: $.[*].metadata.name: - - global-1234 - layering-policy + - global-1234 - site-1234 $.[*].status.revision: - "$HISTORY['initialize'].$RESPONSE['$.[0].status.revision']" @@ -117,27 +110,25 @@ tests: - mop - mop $.[2].data.b: 5 - skip: Not implemented. - name: verify_revision_4 desc: Verify rollback revision - GET: /api/v1.0/revisions/$HISTORY['rollback'].$RESPONSE['$.[0].status.revision']/documents + GET: /api/v1.0/revisions/$HISTORY['rollback'].$RESPONSE['$.[0].id']/documents status: 200 response_multidoc_jsonpaths: $.[*].metadata.name: - - global-1234 - layering-policy + - global-1234 - region-1234 - site-1234 $.[*].status.revision: - "$HISTORY['initialize'].$RESPONSE['$.[0].status.revision']" - "$HISTORY['initialize'].$RESPONSE['$.[0].status.revision']" - - "$HISTORY['rollback'].$RESPONSE['$.[0].status.revision']" - - "$HISTORY['rollback'].$RESPONSE['$.[0].status.revision']" + - "$HISTORY['rollback'].$RESPONSE['$.[0].id']" + - "$HISTORY['rollback'].$RESPONSE['$.[0].id']" $.[*].status.bucket: - mop - mop - mop - mop $.[3].data.b: 4 - skip: Not implemented. diff --git a/deckhand/tests/unit/control/test_api.py b/deckhand/tests/unit/control/test_api.py index ed509925..8b19e056 100644 --- a/deckhand/tests/unit/control/test_api.py +++ b/deckhand/tests/unit/control/test_api.py @@ -21,6 +21,8 @@ from deckhand.control import revision_diffing from deckhand.control import revision_documents from deckhand.control import revision_tags from deckhand.control import revisions +from deckhand.control import rollback +from deckhand.control import versions from deckhand.tests.unit import base as test_base @@ -29,7 +31,7 @@ class TestApi(test_base.DeckhandTestCase): def setUp(self): super(TestApi, self).setUp() for resource in (buckets, revision_diffing, revision_documents, - revision_tags, revisions): + revision_tags, revisions, rollback, versions): resource_name = resource.__name__.split('.')[-1] resource_obj = mock.patch.object( resource, '%sResource' % resource_name.title().replace( @@ -63,7 +65,10 @@ class TestApi(test_base.DeckhandTestCase): mock.call('/api/v1.0/revisions/{revision_id}/tags', self.revision_tags_resource()), mock.call('/api/v1.0/revisions/{revision_id}/tags/{tag}', - self.revision_tags_resource()) + self.revision_tags_resource()), + mock.call('/api/v1.0/rollback/{revision_id}', + self.rollback_resource()), + mock.call('/versions', self.versions_resource()) ]) mock_db_api.drop_db.assert_called_once_with() diff --git a/deckhand/tests/unit/db/base.py b/deckhand/tests/unit/db/base.py index 2448a967..047c164f 100644 --- a/deckhand/tests/unit/db/base.py +++ b/deckhand/tests/unit/db/base.py @@ -101,6 +101,9 @@ class TestDbBase(base.DeckhandWithDBTestCase): def list_revisions(self): return db_api.revision_get_all() + def rollback_revision(self, revision_id): + return db_api.revision_rollback(revision_id) + def _validate_object(self, obj): for attr in BASE_EXPECTED_FIELDS: if attr.endswith('_at'): diff --git a/deckhand/tests/unit/db/test_revision_rollback.py b/deckhand/tests/unit/db/test_revision_rollback.py new file mode 100644 index 00000000..b075d890 --- /dev/null +++ b/deckhand/tests/unit/db/test_revision_rollback.py @@ -0,0 +1,81 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from deckhand.tests import test_utils +from deckhand.tests.unit.db import base + + +class TestRevisionRollback(base.TestDbBase): + + def test_create_update_rollback(self): + # Revision 1: Create 4 documents. + payload = base.DocumentFixture.get_minimal_multi_fixture(count=4) + bucket_name = test_utils.rand_name('bucket') + created_documents = self.create_documents(bucket_name, payload) + orig_revision_id = created_documents[0]['revision_id'] + + # Revision 2: Update the last document. + payload[-1]['data'] = {'foo': 'bar'} + self.create_documents(bucket_name, payload) + + # Revision 3: rollback to revision 1. + rollback_revision = self.rollback_revision(orig_revision_id) + + self.assertEqual(3, rollback_revision['id']) + self.assertEqual( + [1, 1, 1, 3], + [d['revision_id'] for d in rollback_revision['documents']]) + self.assertEqual( + [1, 1, 1, 3], + [d['orig_revision_id'] for d in rollback_revision['documents']]) + + rollback_documents = self.list_revision_documents( + rollback_revision['id']) + self.assertEqual([1, 1, 1, 3], + [d['revision_id'] for d in rollback_documents]) + self.assertEqual([1, 1, 1, 3], + [d['orig_revision_id'] for d in rollback_documents]) + + def test_create_update_delete_rollback(self): + # Revision 1: Create 4 documents. + payload = base.DocumentFixture.get_minimal_multi_fixture(count=4) + bucket_name = test_utils.rand_name('bucket') + created_documents = self.create_documents(bucket_name, payload) + orig_revision_id = created_documents[0]['revision_id'] + + # Revision 2: Update the last document. + payload[-1]['data'] = {'foo': 'bar'} + self.create_documents(bucket_name, payload) + + # Revision 3: Delete the third document. + payload.pop(2) + self.create_documents(bucket_name, payload) + + # Rollback 4: rollback to revision 1. + rollback_revision = self.rollback_revision(orig_revision_id) + + self.assertEqual(4, rollback_revision['id']) + self.assertEqual( + [1, 1, 4, 4], + [d['revision_id'] for d in rollback_revision['documents']]) + self.assertEqual( + [1, 1, 4, 4], + [d['orig_revision_id'] for d in rollback_revision['documents']]) + + rollback_documents = self.list_revision_documents( + rollback_revision['id']) + self.assertEqual([1, 1, 4, 4], + [d['revision_id'] for d in rollback_documents]) + self.assertEqual([1, 1, 4, 4], + [d['orig_revision_id'] for d in rollback_documents]) diff --git a/deckhand/tests/unit/db/test_revision_rollback_negative.py b/deckhand/tests/unit/db/test_revision_rollback_negative.py new file mode 100644 index 00000000..77f91ff9 --- /dev/null +++ b/deckhand/tests/unit/db/test_revision_rollback_negative.py @@ -0,0 +1,47 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from deckhand import errors +from deckhand.tests import test_utils +from deckhand.tests.unit.db import base + + +class TestRevisionRollbackNegative(base.TestDbBase): + + def test_rollback_same_revision_raises_error(self): + # Revision 1: Create 4 documents. + payload = base.DocumentFixture.get_minimal_multi_fixture(count=4) + bucket_name = test_utils.rand_name('bucket') + created_documents = self.create_documents(bucket_name, payload) + orig_revision_id = created_documents[0]['revision_id'] + + # Attempt to rollback to the latest revision, which should result + # in an error. + self.assertRaises( + errors.InvalidRollback, self.rollback_revision, orig_revision_id) + + def test_rollback_unchanged_revision_history_raises_error(self): + # Revision 1: Create 4 documents. + payload = base.DocumentFixture.get_minimal_multi_fixture(count=4) + bucket_name = test_utils.rand_name('bucket') + created_documents = self.create_documents(bucket_name, payload) + orig_revision_id = created_documents[0]['revision_id'] + + # Create a 2nd revision that is a carbon-copy of 1st. + self.create_documents(bucket_name, payload) + + # Attempt to rollback to the 1st revision, which should result in an + # error, as it is identical to the latest revision. + self.assertRaises( + errors.InvalidRollback, self.rollback_revision, orig_revision_id)