From af0bfd813de64810a9eb48fdf549aeb12a8a7943 Mon Sep 17 00:00:00 2001 From: Felipe Monteiro Date: Mon, 18 Sep 2017 20:11:32 +0100 Subject: [PATCH] Deckhand postgresql compatibility. Currently, Deckhand is not fully compatible with postgresql as it uses sqlite for all of its testing, including functional testing. Since postgresql will be used in prod, Deckhand obviously must support it, in addition to sqlite, needed for unit testing. This commit alters the functional testing script to use postgresql as well as makes necessary back-end changes to support postgresql. Included in this commit: - alter tools/functional-tests.sh so that it uses postgresql as the db connection - modifies primary key for Bucket DB model to be an Integer rather than a String - updates foreign key to point to new primary key - updates necessary integration logic so that the bucket name is still known by the Document DB model and returned in appropriate response bodies Change-Id: I7bc806fb18f7b47c13978dcd806d422a573a06b3 --- deckhand/control/views/document.py | 4 ++-- deckhand/control/views/revision.py | 5 +++-- deckhand/db/sqlalchemy/api.py | 11 ++++++----- deckhand/db/sqlalchemy/models.py | 27 +++++++++++++++++++-------- deckhand/tests/unit/db/base.py | 2 +- requirements.txt | 3 ++- tools/functional-tests.sh | 11 ++--------- 7 files changed, 35 insertions(+), 28 deletions(-) diff --git a/deckhand/control/views/document.py b/deckhand/control/views/document.py index 498ad23d..4aaa83ba 100644 --- a/deckhand/control/views/document.py +++ b/deckhand/control/views/document.py @@ -34,7 +34,7 @@ class ViewBuilder(common.ViewBuilder): # need to return bucket_id and revision_id. if len(documents) == 1 and documents[0]['deleted']: resp_obj = {'status': {}} - resp_obj['status']['bucket'] = documents[0]['bucket_id'] + resp_obj['status']['bucket'] = documents[0]['bucket_name'] resp_obj['status']['revision'] = documents[0]['revision_id'] return [resp_obj] @@ -47,7 +47,7 @@ class ViewBuilder(common.ViewBuilder): resp_obj = {x: document[x] for x in attrs} resp_obj.setdefault('status', {}) - resp_obj['status']['bucket'] = document['bucket_id'] + resp_obj['status']['bucket'] = document['bucket_name'] resp_obj['status']['revision'] = document['revision_id'] resp_list.append(resp_obj) diff --git a/deckhand/control/views/revision.py b/deckhand/control/views/revision.py index 2b735bc4..da15f554 100644 --- a/deckhand/control/views/revision.py +++ b/deckhand/control/views/revision.py @@ -37,7 +37,7 @@ class ViewBuilder(common.ViewBuilder): body['tags'].update([t['tag'] for t in revision['tags']]) body['buckets'].update( - [d['bucket_id'] for d in rev_documents]) + [d['bucket_name'] for d in rev_documents]) body['tags'] = sorted(body['tags']) body['buckets'] = sorted(body['buckets']) @@ -77,7 +77,8 @@ class ViewBuilder(common.ViewBuilder): for tag in revision['tags']: tags.setdefault(tag['tag'], {'name': tag['tag']}) - buckets = sorted(set([d['bucket_id'] for d in revision['documents']])) + buckets = sorted( + set([d['bucket_name'] for d in revision['documents']])) return { 'id': revision.get('id'), diff --git a/deckhand/db/sqlalchemy/api.py b/deckhand/db/sqlalchemy/api.py index 59f9b7de..f9419ab4 100644 --- a/deckhand/db/sqlalchemy/api.py +++ b/deckhand/db/sqlalchemy/api.py @@ -114,7 +114,7 @@ def documents_create(bucket_name, documents, session=None): # `documents`: the difference between the former and the latter. document_history = [(d['schema'], d['name']) for d in revision_get_documents( - bucket_id=bucket_name)] + bucket_name=bucket_name)] documents_to_delete = [ h for h in document_history if h not in [(d['schema'], d['metadata']['name']) for d in documents]] @@ -136,7 +136,7 @@ def documents_create(bucket_name, documents, session=None): doc['name'] = d[1] doc['data'] = {} doc['_metadata'] = {} - doc['bucket_id'] = bucket['name'] + doc['bucket_id'] = bucket['id'] doc['revision_id'] = revision['id'] # Save and mark the document as `deleted` in the database. @@ -151,9 +151,10 @@ def documents_create(bucket_name, documents, session=None): [(d['schema'], d['name']) for d in documents_to_create]) for doc in documents_to_create: with session.begin(): - doc['bucket_id'] = bucket['name'] + doc['bucket_id'] = bucket['id'] doc['revision_id'] = revision['id'] doc.save(session=session) + # NOTE(fmontei): The orig_revision_id is not copied into the # revision_id for each created document, because the revision_id here # should reference the just-created revision. In case the user needs @@ -200,12 +201,12 @@ def _documents_create(bucket_name, values_list, session=None): # If the document already exists in another bucket, raise an error. # Ignore redundant validation policies as they are allowed to exist # in multiple buckets. - if (existing_document['bucket_id'] != bucket_name and + if (existing_document['bucket_name'] != bucket_name and existing_document['schema'] != types.VALIDATION_POLICY_SCHEMA): raise errors.DocumentExists( schema=existing_document['schema'], name=existing_document['name'], - bucket=existing_document['bucket_id']) + bucket=existing_document['bucket_name']) if not _document_changed(existing_document): # Since the document has not changed, reference the original diff --git a/deckhand/db/sqlalchemy/models.py b/deckhand/db/sqlalchemy/models.py index 705d4e85..b1e0c18e 100644 --- a/deckhand/db/sqlalchemy/models.py +++ b/deckhand/db/sqlalchemy/models.py @@ -19,6 +19,7 @@ from sqlalchemy import Boolean from sqlalchemy import Column from sqlalchemy import DateTime from sqlalchemy.ext import declarative +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy import ForeignKey from sqlalchemy import Integer from sqlalchemy.orm import relationship @@ -64,10 +65,8 @@ class DeckhandBase(models.ModelBase, models.TimestampMixin): def items(self): return self.__dict__.items() - def to_dict(self, raw_dict=False): + def to_dict(self): """Convert the object into dictionary format. - - :param raw_dict: Renames the key "_metadata" to "metadata". """ d = self.__dict__.copy() # Remove private state instance, as it is not serializable and causes @@ -93,8 +92,9 @@ def gen_unique_constraint(table_name, *fields): class Bucket(BASE, DeckhandBase): __tablename__ = 'buckets' - name = Column(String(36), primary_key=True) - documents = relationship("Document") + id = Column(Integer, primary_key=True) + name = Column(String(36), unique=True) + documents = relationship("Document", backref="bucket") class Revision(BASE, DeckhandBase): @@ -141,8 +141,7 @@ class Document(BASE, DeckhandBase): _metadata = Column(oslo_types.JsonEncodedDict(), nullable=False) data = Column(oslo_types.JsonEncodedDict(), nullable=True) is_secret = Column(Boolean, nullable=False, default=False) - - bucket_id = Column(Integer, ForeignKey('buckets.name', ondelete='CASCADE'), + bucket_id = Column(Integer, ForeignKey('buckets.id', ondelete='CASCADE'), nullable=False) revision_id = Column( Integer, ForeignKey('revisions.id', ondelete='CASCADE'), @@ -160,11 +159,23 @@ class Document(BASE, DeckhandBase): Integer, ForeignKey('revisions.id', ondelete='CASCADE'), nullable=True) + @hybrid_property + def bucket_name(self): + if hasattr(self, 'bucket') and self.bucket: + return self.bucket.name + return None + def to_dict(self, raw_dict=False): + """Convert the object into dictionary format. + + :param raw_dict: Renames the key "_metadata" to "metadata". + """ d = super(Document, self).to_dict() + d['bucket_name'] = self.bucket_name + # NOTE(fmontei): ``metadata`` is reserved by the DB, so ``_metadata`` # must be used to store document metadata information in the DB. - if not raw_dict and '_metadata' in self.keys(): + if not raw_dict: d['metadata'] = d.pop('_metadata') return d diff --git a/deckhand/tests/unit/db/base.py b/deckhand/tests/unit/db/base.py index 6674b78c..cba4f32e 100644 --- a/deckhand/tests/unit/db/base.py +++ b/deckhand/tests/unit/db/base.py @@ -69,7 +69,7 @@ class TestDbBase(base.DeckhandWithDBTestCase): if do_validation: for idx, doc in enumerate(docs): self.validate_document(expected=documents[idx], actual=doc) - self.assertEqual(bucket_name, doc['bucket_id']) + self.assertEqual(bucket_name, doc['bucket_name']) return docs diff --git a/requirements.txt b/requirements.txt index 02c6a8b2..5c2cdb1b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,6 +18,8 @@ stevedore>=1.20.0 # Apache-2.0 jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT python-keystoneclient>=3.8.0 # Apache-2.0 keystonemiddleware>=4.12.0 # Apache-2.0 +psycopg2==2.7.3.1 +uwsgi==2.0.15 oslo.cache>=1.5.0 # Apache-2.0 oslo.concurrency>=3.8.0 # Apache-2.0 @@ -33,4 +35,3 @@ oslo.serialization!=2.19.1,>=1.10.0 # Apache-2.0 oslo.utils>=3.20.0 # Apache-2.0 python-barbicanclient>=4.0.0 # Apache-2.0 -uwsgi==2.0.15 diff --git a/tools/functional-tests.sh b/tools/functional-tests.sh index 7ff5047e..0134bb20 100755 --- a/tools/functional-tests.sh +++ b/tools/functional-tests.sh @@ -34,17 +34,12 @@ POSTGRES_IP=$( --format='{{ .NetworkSettings.Networks.bridge.IPAddress }}' \ $POSTGRES_ID ) -POSTGRES_PORT=$( - sudo docker inspect \ - --format='{{(index (index .NetworkSettings.Ports "5432/tcp") 0).HostPort}}' \ - $POSTGRES_ID -) log_section Creating config file CONF_DIR=$(mktemp -d) export DECKHAND_TEST_URL=http://localhost:9000 -export DATABASE_URL=postgres://deckhand:password@$POSTGRES_IP:$POSTGRES_PORT/deckhand +export DATABASE_URL=postgresql+psycopg2://deckhand:password@$POSTGRES_IP:5432/deckhand # Used by Deckhand's initialization script to search for config files. export OS_DECKHAND_CONFIG_DIR=$CONF_DIR @@ -61,9 +56,7 @@ use_stderr = true [barbican] [database] -# XXX For now, connection to postgres is not setup. -#connection = $DATABASE_URL -connection = sqlite:// +connection = $DATABASE_URL [keystone_authtoken] EOCONF