Datastore containerization

Significant changes:

* Using docker image to install datastore.
* Datastore image is common to different datastores.
* Using backup docker image to do backup and restore.
* Support MariaDB replication
* Set most of the functional jobs as non-voting as nested
  virtualization is not supported in CI.

Change-Id: Ia9c97a63a961eebc336b70d28dc77638144c1834

.zuul.yaml

@@ -14,15 +14,14 @@
         - openstack-tox-pylint
         - trove-tox-bandit-baseline:
             voting: false
-        - trove-tempest:
+        - trove-tempest
+        - trove-functional-mysql:
             voting: false
-        - trove-functional-mysql
+        - trove-scenario-mysql-single:
-        - trove-scenario-mysql-single
-        - trove-scenario-mysql-multi
-        - trove-scenario-mariadb-single
-        - trove-scenario-postgresql-single:
             voting: false
-        - trove-scenario-postgresql-multi:
+        - trove-scenario-mysql-multi:
+            voting: false
+        - trove-scenario-mariadb-single:
             voting: false
         - trove-scenario-mariadb-multi:
             voting: false
@@ -34,9 +33,12 @@
       queue: trove
       jobs:
         - openstack-tox-pylint
-        - trove-functional-mysql
+        - trove-functional-mysql:
-        - trove-scenario-mysql-single
+            voting: false
-        - trove-scenario-mysql-multi
+        - trove-scenario-mysql-single:
+            voting: false
+        - trove-scenario-mysql-multi:
+            voting: false
     experimental:
       jobs:
         - trove-grenade
@@ -145,7 +147,7 @@
       trove_resize_time_out: 1800
       trove_test_datastore: 'mysql'
       trove_test_group: 'mysql'
-      trove_test_datastore_version: '5.7'
+      trove_test_datastore_version: '5.7.29'
 
 - job:
     name: trove-functional-mysql-nondev
@@ -153,11 +155,11 @@
     vars:
       devstack_localrc:
         TROVE_RESIZE_TIME_OUT: 1800
-        TROVE_NON_DEV_IMAGE_URL_MYSQL: https://tarballs.opendev.org/openstack/trove/images/trove-master-mysql-ubuntu-xenial.qcow2
+        TROVE_NON_DEV_IMAGE_URL: https://tarballs.opendev.org/openstack/trove/images/trove-master-mysql-ubuntu-xenial.qcow2
       trove_resize_time_out: 1800
       trove_test_datastore: 'mysql'
       trove_test_group: 'mysql'
-      trove_test_datastore_version: '5.7'
+      trove_test_datastore_version: '5.7.29'
 
 - job:
     name: trove-grenade
@@ -212,7 +214,7 @@
     vars:
       trove_test_datastore: mariadb
       trove_test_group: mariadb-supported-single
-      trove_test_datastore_version: 10.4
+      trove_test_datastore_version: 10.4.12
       devstack_localrc:
         TROVE_ENABLE_IMAGE_BUILD: false
 
@@ -222,7 +224,7 @@
     vars:
       trove_test_datastore: mariadb
       trove_test_group: mariadb-supported-multi
-      trove_test_datastore_version: 10.4
+      trove_test_datastore_version: 10.4.12
       devstack_localrc:
         TROVE_ENABLE_IMAGE_BUILD: false
 
@@ -232,7 +234,7 @@
     vars:
       trove_test_datastore: mysql
       trove_test_group: mysql-supported-single
-      trove_test_datastore_version: 5.7
+      trove_test_datastore_version: 5.7.29
 
 - job:
     name: trove-scenario-mysql-multi
@@ -240,7 +242,7 @@
     vars:
       trove_test_datastore: mysql
       trove_test_group: mysql-supported-multi
-      trove_test_datastore_version: 5.7
+      trove_test_datastore_version: 5.7.29
 
 - job:
     name: trove-scenario-percona-multi

api-ref/source/parameters.yaml

@@ -688,10 +688,9 @@ replica_count:
   type: integer
 replica_of:
   description: |
-    ID or name of an existing instance to replicate
+    ID or name of an existing instance to replicate from.
-    from.
   in: body
-  required: false
+  required: true
   type: string
 restore_point:
   description: |
@@ -735,9 +734,10 @@ shard_id:
    type: string
 slave_of:
   description: |
-    To detach a replica, set ``slave_of`` to null.
+    To detach a replica, set ``slave_of`` to null. Deprecated in favor of
+    ``replica_of``
   in: body
-  required: true
+  required: false
   type: string
 tenant_id:
   description: |

api-ref/source/samples/instance-patch-detach-replica-request.json

@@ -1,6 +1,5 @@
 {
     "instance": {
-        "replica_of": null,
+        "replica_of": null
-        "slave_of": null
     }
 }

backup/Dockerfile

@@ -0,0 +1,41 @@
+FROM ubuntu:18.04
+LABEL maintainer="anlin.kong@gmail.com"
+
+ARG APTOPTS="-y -qq --no-install-recommends --allow-unauthenticated"
+ARG PERCONA_XTRABACKUP_VERSION=24
+ENV DEBIAN_FRONTEND noninteractive
+ENV APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1
+
+RUN apt-get update \
+    && apt-get install $APTOPTS gnupg2 lsb-release apt-utils apt-transport-https ca-certificates software-properties-common curl \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install percona-xtrabackup for mysql
+RUN curl -sSL https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb -o percona-release.deb \
+    && dpkg -i percona-release.deb \
+    && percona-release enable-only tools release \
+    && apt-get update \
+    && apt-get install $APTOPTS percona-xtrabackup-${PERCONA_XTRABACKUP_VERSION} \
+    && apt-get clean
+
+# Install mariabackup for mariadb
+Run apt-key adv --fetch-keys 'https://mariadb.org/mariadb_release_signing_key.asc' \
+    && add-apt-repository "deb [arch=amd64] http://mirror2.hs-esslingen.de/mariadb/repo/10.4/ubuntu $(lsb_release -cs) main" \
+    && apt-get update \
+    && apt-get install $APTOPTS mariadb-backup \
+    && apt-get clean
+
+RUN apt-get update \
+    && apt-get install $APTOPTS build-essential python3-setuptools python3-all python3-all-dev python3-pip libffi-dev libssl-dev libxml2-dev libxslt1-dev libyaml-dev \
+    && apt-get clean
+
+COPY . /opt/trove/backup
+WORKDIR /opt/trove/backup
+
+RUN pip3 --no-cache-dir install -U -r requirements.txt
+
+RUN curl -sSL https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64 -o /usr/local/bin/dumb-init \
+    && chmod +x /usr/local/bin/dumb-init
+
+ENTRYPOINT ["dumb-init", "--single-child", "--"]

backup/drivers/base.py

@@ -0,0 +1,207 @@
+# Copyright 2020 Catalyst Cloud
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import os
+import signal
+import subprocess
+
+from oslo_config import cfg
+from oslo_log import log as logging
+
+CONF = cfg.CONF
+LOG = logging.getLogger(__name__)
+
+
+class BaseRunner(object):
+    """Base class for Backup Strategy implementations."""
+
+    # Subclass should provide the commands.
+    cmd = None
+    restore_cmd = None
+    prepare_cmd = None
+
+    encrypt_key = CONF.backup_encryption_key
+    default_data_dir = '/var/lib/mysql/data'
+
+    def __init__(self, *args, **kwargs):
+        self.process = None
+        self.pid = None
+        self.base_filename = kwargs.get('filename')
+        self.storage = kwargs.pop('storage', None)
+        self.location = kwargs.pop('location', '')
+        self.checksum = kwargs.pop('checksum', '')
+
+        if 'restore_location' not in kwargs:
+            kwargs['restore_location'] = self.default_data_dir
+        self.restore_location = kwargs['restore_location']
+
+        self.command = self.cmd % kwargs
+        self.restore_command = (self.decrypt_cmd +
+                                self.unzip_cmd +
+                                (self.restore_cmd % kwargs))
+        self.prepare_command = self.prepare_cmd % kwargs
+
+    @property
+    def filename(self):
+        """Subclasses may overwrite this to declare a format (.tar)."""
+        return self.base_filename
+
+    @property
+    def manifest(self):
+        """Target file name."""
+        return "%s%s%s" % (self.filename,
+                           self.zip_manifest,
+                           self.encrypt_manifest)
+
+    @property
+    def zip_cmd(self):
+        return ' | gzip'
+
+    @property
+    def unzip_cmd(self):
+        return 'gzip -d -c | '
+
+    @property
+    def zip_manifest(self):
+        return '.gz'
+
+    @property
+    def encrypt_cmd(self):
+        return (' | openssl enc -aes-256-cbc -md sha512 -pbkdf2 -iter 10000 '
+                '-salt -pass pass:%s' %
+                self.encrypt_key) if self.encrypt_key else ''
+
+    @property
+    def decrypt_cmd(self):
+        if self.encrypt_key:
+            return ('openssl enc -d -aes-256-cbc -md sha512 -pbkdf2 -iter '
+                    '10000 -salt -pass pass:%s | '
+                    % self.encrypt_key)
+        else:
+            return ''
+
+    @property
+    def encrypt_manifest(self):
+        return '.enc' if self.encrypt_key else ''
+
+    def _run(self):
+        LOG.info("Running backup cmd: %s", self.command)
+        self.process = subprocess.Popen(self.command, shell=True,
+                                        stdout=subprocess.PIPE,
+                                        stderr=subprocess.PIPE,
+                                        preexec_fn=os.setsid)
+        self.pid = self.process.pid
+
+    def __enter__(self):
+        """Start up the process."""
+        self.pre_backup()
+        self._run()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        """Clean up everything."""
+        if getattr(self, 'process', None):
+            try:
+                # Send a sigterm to the session leader, so that all
+                # child processes are killed and cleaned up on terminate
+                os.killpg(self.process.pid, signal.SIGTERM)
+                self.process.terminate()
+            except OSError:
+                pass
+
+            if exc_type is not None:
+                return False
+
+            try:
+                err = self.process.stderr.read()
+                if err:
+                    raise Exception(err)
+            except OSError:
+                pass
+
+            if not self.check_process():
+                raise Exception()
+
+        self.post_backup()
+
+        return True
+
+    def read(self, chunk_size):
+        return self.process.stdout.read(chunk_size)
+
+    def get_metadata(self):
+        """Hook for subclasses to get metadata from the backup."""
+        return {}
+
+    def check_process(self):
+        """Hook for subclasses to check process for errors."""
+        return True
+
+    def check_restore_process(self):
+        """Hook for subclasses to check the restore process for errors."""
+        return True
+
+    def pre_backup(self):
+        """Hook for subclasses to run commands before backup."""
+        pass
+
+    def post_backup(self):
+        """Hook for subclasses to run commands after backup."""
+        pass
+
+    def pre_restore(self):
+        """Hook that is called before the restore command."""
+        pass
+
+    def post_restore(self):
+        """Hook that is called after the restore command."""
+        pass
+
+    def unpack(self, location, checksum, command):
+        stream = self.storage.load(location, checksum)
+
+        LOG.info('Running restore from stream, command: %s', command)
+        self.process = subprocess.Popen(command, shell=True,
+                                        stdin=subprocess.PIPE,
+                                        stderr=subprocess.PIPE)
+        content_length = 0
+        for chunk in stream:
+            self.process.stdin.write(chunk)
+            content_length += len(chunk)
+        self.process.stdin.close()
+
+        try:
+            err = self.process.stderr.read()
+            if err:
+                raise Exception(err)
+        except OSError:
+            pass
+
+        if not self.check_restore_process():
+            raise Exception()
+
+        return content_length
+
+    def run_restore(self):
+        return self.unpack(self.location, self.checksum, self.restore_command)
+
+    def restore(self):
+        """Restore backup to data directory.
+
+        :returns Restored data size.
+        """
+        self.pre_restore()
+        content_length = self.run_restore()
+        self.post_restore()
+        return content_length

backup/drivers/innobackupex.py

@@ -0,0 +1,137 @@
+# Copyright 2020 Catalyst Cloud
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import re
+
+from oslo_concurrency import processutils
+from oslo_config import cfg
+from oslo_log import log as logging
+
+from backup.drivers import mysql_base
+
+LOG = logging.getLogger(__name__)
+CONF = cfg.CONF
+
+
+class InnoBackupEx(mysql_base.MySQLBaseRunner):
+    """Implementation of Backup and Restore for InnoBackupEx."""
+    backup_log = '/tmp/innobackupex.log'
+    prepare_log = '/tmp/prepare.log'
+    restore_cmd = ('xbstream -x -C %(restore_location)s --parallel=2'
+                   ' 2>/tmp/xbstream_extract.log')
+    prepare_cmd = ('innobackupex'
+                   ' --defaults-file=%(restore_location)s/backup-my.cnf'
+                   ' --ibbackup=xtrabackup'
+                   ' --apply-log'
+                   ' %(restore_location)s'
+                   ' 2>' + prepare_log)
+
+    @property
+    def cmd(self):
+        cmd = ('innobackupex'
+               ' --stream=xbstream'
+               ' --parallel=2 ' +
+               self.user_and_pass + ' %s' % self.default_data_dir +
+               ' 2>' + self.backup_log
+               )
+        return cmd + self.zip_cmd + self.encrypt_cmd
+
+    def check_restore_process(self):
+        """Check whether xbstream restore is successful."""
+        LOG.info('Checking return code of xbstream restore process.')
+        return_code = self.process.wait()
+        if return_code != 0:
+            LOG.error('xbstream exited with %s', return_code)
+            return False
+
+        with open('/tmp/xbstream_extract.log', 'r') as xbstream_log:
+            for line in xbstream_log:
+                # Ignore empty lines
+                if not line.strip():
+                    continue
+
+                LOG.error('xbstream restore failed with: %s',
+                          line.rstrip('\n'))
+                return False
+
+        return True
+
+    def post_restore(self):
+        """Hook that is called after the restore command."""
+        LOG.info("Running innobackupex prepare: %s.", self.prepare_command)
+        processutils.execute(self.prepare_command, shell=True)
+
+        LOG.info("Checking innobackupex prepare log")
+        with open(self.prepare_log, 'r') as prepare_log:
+            output = prepare_log.read()
+            if not output:
+                msg = "innobackupex prepare log file empty"
+                raise Exception(msg)
+
+            last_line = output.splitlines()[-1].strip()
+            if not re.search('completed OK!', last_line):
+                msg = "innobackupex prepare did not complete successfully"
+                raise Exception(msg)
+
+
+class InnoBackupExIncremental(InnoBackupEx):
+    """InnoBackupEx incremental backup."""
+
+    incremental_prep = ('innobackupex'
+                        ' --defaults-file=%(restore_location)s/backup-my.cnf'
+                        ' --ibbackup=xtrabackup'
+                        ' --apply-log'
+                        ' --redo-only'
+                        ' %(restore_location)s'
+                        ' %(incremental_args)s'
+                        ' 2>/tmp/innoprepare.log')
+
+    def __init__(self, *args, **kwargs):
+        if not kwargs.get('lsn'):
+            raise AttributeError('lsn attribute missing')
+        self.parent_location = kwargs.pop('parent_location', '')
+        self.parent_checksum = kwargs.pop('parent_checksum', '')
+        self.restore_content_length = 0
+
+        super(InnoBackupExIncremental, self).__init__(*args, **kwargs)
+
+    @property
+    def cmd(self):
+        cmd = ('innobackupex'
+               ' --stream=xbstream'
+               ' --incremental'
+               ' --incremental-lsn=%(lsn)s ' +
+               self.user_and_pass + ' %s' % self.default_data_dir +
+               ' 2>' + self.backup_log)
+        return cmd + self.zip_cmd + self.encrypt_cmd
+
+    def get_metadata(self):
+        _meta = super(InnoBackupExIncremental, self).get_metadata()
+
+        _meta.update({
+            'parent_location': self.parent_location,
+            'parent_checksum': self.parent_checksum,
+        })
+        return _meta
+
+    def run_restore(self):
+        """Run incremental restore.
+
+        First grab all parents and prepare them with '--redo-only'. After
+        all backups are restored the super class InnoBackupEx post_restore
+        method is called to do the final prepare with '--apply-log'
+        """
+        LOG.debug('Running incremental restore')
+        self.incremental_restore(self.location, self.checksum)
+        return self.restore_content_length

backup/drivers/mariabackup.py

@@ -0,0 +1,87 @@
+# Copyright 2020 Catalyst Cloud
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+from oslo_config import cfg
+from oslo_log import log as logging
+
+from backup.drivers import mysql_base
+
+LOG = logging.getLogger(__name__)
+CONF = cfg.CONF
+
+
+class MariaBackup(mysql_base.MySQLBaseRunner):
+    """Implementation of Backup and Restore using mariabackup."""
+    backup_log = '/tmp/mariabackup.log'
+    restore_log = '/tmp/mbstream_extract.log'
+    restore_cmd = ('mbstream -x -C %(restore_location)s 2>' + restore_log)
+    prepare_cmd = ''
+
+    @property
+    def cmd(self):
+        cmd = ('mariabackup --backup --stream=xbstream ' +
+               self.user_and_pass + ' 2>' + self.backup_log)
+        return cmd + self.zip_cmd + self.encrypt_cmd
+
+    def check_restore_process(self):
+        LOG.debug('Checking return code of mbstream restore process.')
+        return_code = self.process.wait()
+        if return_code != 0:
+            LOG.error('mbstream exited with %s', return_code)
+            return False
+
+        return True
+
+
+class MariaBackupIncremental(MariaBackup):
+    """Incremental backup and restore using mariabackup."""
+    incremental_prep = ('mariabackup --prepare '
+                        '--target-dir=%(restore_location)s '
+                        '%(incremental_args)s '
+                        '2>/tmp/innoprepare.log')
+
+    def __init__(self, *args, **kwargs):
+        if not kwargs.get('lsn'):
+            raise AttributeError('lsn attribute missing')
+        self.parent_location = kwargs.pop('parent_location', '')
+        self.parent_checksum = kwargs.pop('parent_checksum', '')
+        self.restore_content_length = 0
+
+        super(MariaBackupIncremental, self).__init__(*args, **kwargs)
+
+    @property
+    def cmd(self):
+        cmd = (
+            'mariabackup --backup --stream=xbstream'
+            ' --incremental-lsn=%(lsn)s ' +
+            self.user_and_pass +
+            ' 2>' +
+            self.backup_log
+        )
+        return cmd + self.zip_cmd + self.encrypt_cmd
+
+    def get_metadata(self):
+        meta = super(MariaBackupIncremental, self).get_metadata()
+
+        meta.update({
+            'parent_location': self.parent_location,
+            'parent_checksum': self.parent_checksum,
+        })
+        return meta
+
+    def run_restore(self):
+        """Run incremental restore."""
+        LOG.debug('Running incremental restore')
+        self.incremental_restore(self.location, self.checksum)
+        return self.restore_content_length

backup/drivers/mysql_base.py

@@ -0,0 +1,139 @@
+# Copyright 2020 Catalyst Cloud
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+import os
+import re
+import shutil
+
+from oslo_concurrency import processutils
+from oslo_config import cfg
+from oslo_log import log as logging
+
+from backup.drivers import base
+
+CONF = cfg.CONF
+LOG = logging.getLogger(__name__)
+
+
+class MySQLBaseRunner(base.BaseRunner):
+    def __init__(self, *args, **kwargs):
+        super(MySQLBaseRunner, self).__init__(*args, **kwargs)
+
+    @property
+    def user_and_pass(self):
+        return ('--user=%(user)s --password=%(password)s --host=%(host)s' %
+                {'user': CONF.db_user,
+                 'password': CONF.db_password,
+                 'host': CONF.db_host})
+
+    @property
+    def filename(self):
+        return '%s.xbstream' % self.base_filename
+
+    def check_process(self):
+        """Check the backup output for 'completed OK!'."""
+        LOG.debug('Checking backup process output.')
+        with open(self.backup_log, 'r') as backup_log:
+            output = backup_log.read()
+            if not output:
+                LOG.error("Backup log file %s empty.", self.backup_log)
+                return False
+
+            last_line = output.splitlines()[-1].strip()
+            if not re.search('completed OK!', last_line):
+                LOG.error("Backup did not complete successfully.")
+                return False
+
+        return True
+
+    def get_metadata(self):
+        LOG.debug('Getting metadata for backup %s', self.base_filename)
+        meta = {}
+        lsn = re.compile(r"The latest check point \(for incremental\): "
+                         r"'(\d+)'")
+        with open(self.backup_log, 'r') as backup_log:
+            output = backup_log.read()
+            match = lsn.search(output)
+            if match:
+                meta = {'lsn': match.group(1)}
+
+        LOG.info("Updated metadata for backup %s: %s", self.base_filename,
+                 meta)
+
+        return meta
+
+    def incremental_restore_cmd(self, incremental_dir):
+        """Return a command for a restore with a incremental location."""
+        args = {'restore_location': incremental_dir}
+        return (self.decrypt_cmd + self.unzip_cmd + self.restore_cmd % args)
+
+    def incremental_prepare_cmd(self, incremental_dir):
+        if incremental_dir is not None:
+            incremental_arg = '--incremental-dir=%s' % incremental_dir
+        else:
+            incremental_arg = ''
+
+        args = {
+            'restore_location': self.restore_location,
+            'incremental_args': incremental_arg,
+        }
+
+        return self.incremental_prep % args
+
+    def incremental_prepare(self, incremental_dir):
+        prepare_cmd = self.incremental_prepare_cmd(incremental_dir)
+
+        LOG.info("Running restore prepare command: %s.", prepare_cmd)
+        processutils.execute(prepare_cmd, shell=True)
+
+    def incremental_restore(self, location, checksum):
+        """Recursively apply backups from all parents.
+
+        If we are the parent then we restore to the restore_location and
+        we apply the logs to the restore_location only.
+
+        Otherwise if we are an incremental we restore to a subfolder to
+        prevent stomping on the full restore data. Then we run apply log
+        with the '--incremental-dir' flag
+
+        :param location: The source backup location.
+        :param checksum: Checksum of the source backup for validation.
+        """
+        metadata = self.storage.load_metadata(location, checksum)
+        incremental_dir = None
+
+        if 'parent_location' in metadata:
+            LOG.info("Restoring parent: %(parent_location)s"
+                     " checksum: %(parent_checksum)s.", metadata)
+
+            parent_location = metadata['parent_location']
+            parent_checksum = metadata['parent_checksum']
+            # Restore parents recursively so backup are applied sequentially
+            self.incremental_restore(parent_location, parent_checksum)
+            # for *this* backup set the incremental_dir
+            # just use the checksum for the incremental path as it is
+            # sufficiently unique /var/lib/mysql/<checksum>
+            incremental_dir = os.path.join('/var/lib/mysql', checksum)
+            os.makedirs(incremental_dir)
+            command = self.incremental_restore_cmd(incremental_dir)
+        else:
+            # The parent (full backup) use the same command from InnobackupEx
+            # super class and do not set an incremental_dir.
+            command = self.restore_command
+
+        self.restore_content_length += self.unpack(location, checksum, command)
+        self.incremental_prepare(incremental_dir)
+
+        # Delete after restoring this part of backup
+        if incremental_dir:
+            shutil.rmtree(incremental_dir)

backup/main.py

@@ -0,0 +1,149 @@
+# Copyright 2020 Catalyst Cloud
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import os
+
+from oslo_config import cfg
+from oslo_log import log as logging
+from oslo_utils import importutils
+import sys
+
+topdir = os.path.normpath(
+    os.path.join(os.path.abspath(sys.argv[0]), os.pardir, os.pardir))
+sys.path.insert(0, topdir)
+
+LOG = logging.getLogger(__name__)
+CONF = cfg.CONF
+
+cli_opts = [
+    cfg.StrOpt('backup-id'),
+    cfg.StrOpt(
+        'storage-driver',
+        default='swift',
+        choices=['swift']
+    ),
+    cfg.StrOpt(
+        'driver',
+        default='innobackupex',
+        choices=['innobackupex', 'xtrabackup', 'mariabackup']
+    ),
+    cfg.BoolOpt('backup'),
+    cfg.StrOpt('backup-encryption-key'),
+    cfg.StrOpt('db-user'),
+    cfg.StrOpt('db-password'),
+    cfg.StrOpt('db-host'),
+    cfg.StrOpt('os-token'),
+    cfg.StrOpt('os-auth-url'),
+    cfg.StrOpt('os-tenant-id'),
+    cfg.StrOpt('swift-container', default='database_backups'),
+    cfg.DictOpt('swift-extra-metadata'),
+    cfg.StrOpt('restore-from'),
+    cfg.StrOpt('restore-checksum'),
+    cfg.BoolOpt('incremental'),
+    cfg.StrOpt('parent-location'),
+    cfg.StrOpt(
+        'parent-checksum',
+        help='It is up to the storage driver to decide to validate the '
+             'checksum or not. '
+    ),
+]
+
+driver_mapping = {
+    'innobackupex': 'backup.drivers.innobackupex.InnoBackupEx',
+    'innobackupex_inc': 'backup.drivers.innobackupex.InnoBackupExIncremental',
+    'mariabackup': 'backup.drivers.mariabackup.MariaBackup',
+    'mariabackup_inc': 'backup.drivers.mariabackup.MariaBackupIncremental',
+}
+storage_mapping = {
+    'swift': 'backup.storage.swift.SwiftStorage',
+}
+
+
+def stream_backup_to_storage(runner_cls, storage):
+    parent_metadata = {}
+
+    if CONF.incremental:
+        if not CONF.parent_location:
+            LOG.error('--parent-location should be provided for incremental '
+                      'backup')
+            exit(1)
+
+        parent_metadata = storage.load_metadata(CONF.parent_location,
+                                                CONF.parent_checksum)
+        parent_metadata.update(
+            {
+                'parent_location': CONF.parent_location,
+                'parent_checksum': CONF.parent_checksum
+            }
+        )
+
+    try:
+        with runner_cls(filename=CONF.backup_id, **parent_metadata) as bkup:
+            checksum, location = storage.save(
+                bkup,
+                metadata=CONF.swift_extra_metadata
+            )
+            LOG.info('Backup successfully, checksum: %s, location: %s',
+                     checksum, location)
+    except Exception as err:
+        LOG.exception('Failed to call stream_backup_to_storage, error: %s',
+                      err)
+
+
+def stream_restore_from_storage(runner_cls, storage):
+    lsn = ""
+    if storage.is_incremental_backup(CONF.restore_from):
+        lsn = storage.get_backup_lsn(CONF.restore_from)
+
+    try:
+        runner = runner_cls(storage=storage, location=CONF.restore_from,
+                            checksum=CONF.restore_checksum, lsn=lsn)
+        restore_size = runner.restore()
+        LOG.info('Restore successfully, restore_size: %s', restore_size)
+    except Exception as err:
+        LOG.exception('Failed to call stream_restore_from_storage, error: %s',
+                      err)
+
+
+def main():
+    CONF.register_cli_opts(cli_opts)
+    logging.register_options(CONF)
+    CONF(sys.argv[1:], project='trove-backup')
+    logging.setup(CONF, 'trove-backup')
+
+    runner_cls = importutils.import_class(driver_mapping[CONF.driver])
+    storage = importutils.import_class(storage_mapping[CONF.storage_driver])()
+
+    if CONF.backup:
+        if CONF.incremental:
+            runner_cls = importutils.import_class(
+                driver_mapping['%s_inc' % CONF.driver])
+
+        LOG.info('Starting backup database to %s, backup ID %s',
+                 CONF.storage_driver, CONF.backup_id)
+        stream_backup_to_storage(runner_cls, storage)
+    else:
+        if storage.is_incremental_backup(CONF.restore_from):
+            LOG.debug('Restore from incremental backup')
+            runner_cls = importutils.import_class(
+                driver_mapping['%s_inc' % CONF.driver])
+
+        LOG.info('Starting restore database from %s, location: %s',
+                 CONF.storage_driver, CONF.restore_from)
+
+        stream_restore_from_storage(runner_cls, storage)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

backup/requirements.txt

@@ -0,0 +1,6 @@
+oslo.config!=4.3.0,!=4.4.0;python_version>='3.0'  # Apache-2.0
+oslo.log;python_version>='3.0'  # Apache-2.0
+oslo.utils!=3.39.1,!=3.40.0,!=3.40.1;python_version>='3.0'  # Apache-2.0
+oslo.concurrency;python_version>='3.0'  # Apache-2.0
+keystoneauth1  # Apache-2.0
+python-swiftclient  # Apache-2.0

backup/storage/base.py

@@ -0,0 +1,48 @@
+# Copyright 2020 Catalyst Cloud
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import abc
+
+
+class Storage(object):
+    """Base class for Storage driver implementation."""
+
+    @abc.abstractmethod
+    def save(self, stream, metadata=None, **kwargs):
+        """Persist information from the stream.
+
+        Should return the new backup checkshum and location.
+        """
+
+    @abc.abstractmethod
+    def load(self, location, backup_checksum, **kwargs):
+        """Load a stream from the data location.
+
+        Should return an object that provides "read" method.
+        """
+
+    def load_metadata(self, parent_location, parent_checksum):
+        """Load metadata for a parent backup.
+
+        It's up to the storage driver to decide how to implement this function.
+        """
+        return {}
+
+    def is_incremental_backup(self, location):
+        """Check if the location is an incremental backup."""
+        return False
+
+    @abc.abstractmethod
+    def get_backup_lsn(self, location):
+        """Get the backup LSN."""

backup/storage/swift.py

@@ -0,0 +1,294 @@
+# Copyright 2020 Catalyst Cloud
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import hashlib
+import json
+
+from keystoneauth1 import session
+from keystoneauth1.identity import v3
+from oslo_config import cfg
+from oslo_log import log as logging
+import swiftclient
+
+from backup.storage import base
+
+LOG = logging.getLogger(__name__)
+CONF = cfg.CONF
+
+
+def _get_user_keystone_session(auth_url, token, tenant_id):
+    auth = v3.Token(
+        auth_url=auth_url, token=token,
+        project_domain_name="Default",
+        project_id=tenant_id
+    )
+    return session.Session(auth=auth, verify=False)
+
+
+def _get_service_client(auth_url, token, tenant_id):
+    sess = _get_user_keystone_session(auth_url, token, tenant_id)
+    return swiftclient.Connection(session=sess)
+
+
+def _set_attr(original):
+    """Return a swift friendly header key."""
+    key = original.replace('_', '-')
+    return 'X-Object-Meta-%s' % key
+
+
+def _get_attr(original):
+    """Get a friendly name from an object header key."""
+    key = original.replace('-', '_')
+    key = key.replace('x_object_meta_', '')
+    return key
+
+
+class StreamReader(object):
+    """Wrap the stream from the backup process and chunk it into segements."""
+
+    def __init__(self, stream, container, filename, max_file_size):
+        self.stream = stream
+        self.container = container
+        self.filename = filename
+        self.max_file_size = max_file_size
+        self.segment_length = 0
+        self.process = None
+        self.file_number = 0
+        self.end_of_file = False
+        self.end_of_segment = False
+        self.segment_checksum = hashlib.md5()
+
+    @property
+    def base_filename(self):
+        """Filename with extensions removed."""
+        return self.filename.split('.')[0]
+
+    @property
+    def segment(self):
+        return '%s_%08d' % (self.base_filename, self.file_number)
+
+    @property
+    def first_segment(self):
+        return '%s_%08d' % (self.base_filename, 0)
+
+    @property
+    def segment_path(self):
+        return '%s/%s' % (self.container, self.segment)
+
+    def read(self, chunk_size=2 ** 16):
+        if self.end_of_segment:
+            self.segment_length = 0
+            self.segment_checksum = hashlib.md5()
+            self.end_of_segment = False
+
+        # Upload to a new file if we are starting or too large
+        if self.segment_length > (self.max_file_size - chunk_size):
+            self.file_number += 1
+            self.end_of_segment = True
+            return ''
+
+        chunk = self.stream.read(chunk_size)
+        if not chunk:
+            self.end_of_file = True
+            return ''
+
+        self.segment_checksum.update(chunk)
+        self.segment_length += len(chunk)
+        return chunk
+
+
+class SwiftStorage(base.Storage):
+    def __init__(self):
+        self.client = _get_service_client(CONF.os_auth_url, CONF.os_token,
+                                          CONF.os_tenant_id)
+
+    def save(self, stream, metadata=None, container='database_backups'):
+        """Persist data from the stream to swift.
+
+        * Read data from stream, upload to swift
+        * Update the new object metadata, stream provides method to get
+          metadata.
+
+        :returns the new object checkshum and swift full URL.
+        """
+        filename = stream.manifest
+        LOG.info('Saving %(filename)s to %(container)s in swift.',
+                 {'filename': filename, 'container': container})
+
+        # Create the container if it doesn't already exist
+        LOG.debug('Ensuring container %s', container)
+        self.client.put_container(container)
+
+        # Swift Checksum is the checksum of the concatenated segment checksums
+        swift_checksum = hashlib.md5()
+        # Wrap the output of the backup process to segment it for swift
+        stream_reader = StreamReader(stream, container, filename,
+                                     2 * (1024 ** 3))
+
+        url = self.client.url
+        # Full location where the backup manifest is stored
+        location = "%s/%s/%s" % (url, container, filename)
+        LOG.info('Uploading to %s', location)
+
+        # Information about each segment upload job
+        segment_results = []
+
+        # Read from the stream and write to the container in swift
+        while not stream_reader.end_of_file:
+            LOG.debug('Uploading segment %s.', stream_reader.segment)
+            path = stream_reader.segment_path
+            etag = self.client.put_object(container,
+                                          stream_reader.segment,
+                                          stream_reader)
+
+            segment_checksum = stream_reader.segment_checksum.hexdigest()
+
+            # Check each segment MD5 hash against swift etag
+            if etag != segment_checksum:
+                msg = ('Failed to upload data segment to swift. ETAG: %(tag)s '
+                       'Segment MD5: %(checksum)s.' %
+                       {'tag': etag, 'checksum': segment_checksum})
+                raise Exception(msg)
+
+            segment_results.append({
+                'path': path,
+                'etag': etag,
+                'size_bytes': stream_reader.segment_length
+            })
+
+            swift_checksum.update(segment_checksum.encode())
+
+        # All segments uploaded.
+        num_segments = len(segment_results)
+        LOG.debug('File uploaded in %s segments.', num_segments)
+
+        # An SLO will be generated if the backup was more than one segment in
+        # length.
+        large_object = num_segments > 1
+
+        # Meta data is stored as headers
+        if metadata is None:
+            metadata = {}
+        metadata.update(stream.get_metadata())
+        headers = {}
+        for key, value in metadata.items():
+            headers[_set_attr(key)] = value
+
+        LOG.debug('Metadata headers: %s', headers)
+        if large_object:
+            manifest_data = json.dumps(segment_results)
+            LOG.info('Creating the SLO manifest file, manifest content: %s',
+                     manifest_data)
+            # The etag returned from the manifest PUT is the checksum of the
+            # manifest object (which is empty); this is not the checksum we
+            # want.
+            self.client.put_object(container,
+                                   filename,
+                                   manifest_data,
+                                   query_string='multipart-manifest=put')
+
+            # Validation checksum is the Swift Checksum
+            final_swift_checksum = swift_checksum.hexdigest()
+        else:
+            LOG.info('Moving segment %(segment)s to %(filename)s.',
+                     {'segment': stream_reader.first_segment,
+                      'filename': filename})
+            segment_result = segment_results[0]
+            # Just rename it via a special put copy.
+            headers['X-Copy-From'] = segment_result['path']
+            self.client.put_object(container,
+                                   filename, '',
+                                   headers=headers)
+
+            # Delete the old segment file that was copied
+            LOG.debug('Deleting the old segment file %s.',
+                      stream_reader.first_segment)
+            self.client.delete_object(container,
+                                      stream_reader.first_segment)
+
+            final_swift_checksum = segment_result['etag']
+
+        # Validate the object by comparing checksums
+        resp = self.client.head_object(container, filename)
+        # swift returns etag in double quotes
+        # e.g. '"dc3b0827f276d8d78312992cc60c2c3f"'
+        etag = resp['etag'].strip('"')
+
+        # Raise an error and mark backup as failed
+        if etag != final_swift_checksum:
+            msg = ('Failed to upload data to swift. Manifest ETAG: %(tag)s '
+                   'Swift MD5: %(checksum)s' %
+                   {'tag': etag, 'checksum': final_swift_checksum})
+            raise Exception(msg)
+
+        return (final_swift_checksum, location)
+
+    def _explodeLocation(self, location):
+        storage_url = "/".join(location.split('/')[:-2])
+        container = location.split('/')[-2]
+        filename = location.split('/')[-1]
+        return storage_url, container, filename
+
+    def _verify_checksum(self, etag, checksum):
+        etag_checksum = etag.strip('"')
+        if etag_checksum != checksum:
+            msg = ('Checksum validation failure, actual: %s, expected: %s' %
+                   (etag_checksum, checksum))
+            raise Exception(msg)
+
+    def load(self, location, backup_checksum):
+        """Get object from the location."""
+        storage_url, container, filename = self._explodeLocation(location)
+
+        headers, contents = self.client.get_object(container, filename,
+                                                   resp_chunk_size=2 ** 16)
+
+        if backup_checksum:
+            self._verify_checksum(headers.get('etag', ''), backup_checksum)
+
+        return contents
+
+    def load_metadata(self, parent_location, parent_checksum):
+        """Load metadata from swift."""
+        if not parent_location:
+            return {}
+
+        _, container, filename = self._explodeLocation(parent_location)
+        headers = self.client.head_object(container, filename)
+
+        if parent_checksum:
+            self._verify_checksum(headers.get('etag', ''), parent_checksum)
+
+        _meta = {}
+        for key, value in headers.items():
+            if key.startswith('x-object-meta'):
+                _meta[_get_attr(key)] = value
+
+        return _meta
+
+    def is_incremental_backup(self, location):
+        """Check if the location is an incremental backup."""
+        _, container, filename = self._explodeLocation(location)
+        headers = self.client.head_object(container, filename)
+
+        if 'x-object-meta-parent-location' in headers:
+            return True
+
+        return False
+
+    def get_backup_lsn(self, location):
+        """Get the backup LSN."""
+        _, container, filename = self._explodeLocation(location)
+        headers = self.client.head_object(container, filename)
+        return headers.get('x-object-meta-lsn')

devstack/plugin.sh

@@ -362,7 +362,7 @@ function create_subnet_v6 {
     if [[ -n "$IPV6_PRIVATE_NETWORK_GATEWAY" ]]; then
         subnet_params+="--gateway $IPV6_PRIVATE_NETWORK_GATEWAY "
     fi
-    if [ -n $SUBNETPOOL_V6_ID ]; then
+    if [[ -n $SUBNETPOOL_V6_ID ]]; then
         subnet_params+="--subnet-pool $SUBNETPOOL_V6_ID "
     else
         subnet_params+="--subnet-range $FIXED_RANGE_V6 $ipv6_modes} "
@@ -447,27 +447,26 @@ function create_guest_image {
         return 0
     fi
 
-    image_name="trove-datastore-${TROVE_IMAGE_OS}-${TROVE_IMAGE_OS_RELEASE}-${TROVE_DATASTORE_TYPE}"
+    image_name="trove-guest-${TROVE_IMAGE_OS}-${TROVE_IMAGE_OS_RELEASE}"
-    image_url_var="TROVE_NON_DEV_IMAGE_URL_${TROVE_DATASTORE_TYPE^^}"
-    image_url=`eval echo '$'"$image_url_var"`
     mkdir -p $HOME/images
     image_file=$HOME/images/${image_name}.qcow2
 
-    if [[ -n ${image_url} ]]; then
+    if [[ -n ${TROVE_NON_DEV_IMAGE_URL} ]]; then
-        echo "Downloading guest image from ${image_url}"
+        echo "Downloading guest image from ${TROVE_NON_DEV_IMAGE_URL}"
-        curl -sSL ${image_url} -o ${image_file}
+        curl -sSL ${TROVE_NON_DEV_IMAGE_URL} -o ${image_file}
     else
         echo "Starting to create guest image"
 
-        TROVE_BRANCH=${TROVE_BRANCH} $DEST/trove/integration/scripts/trovestack \
+        $DEST/trove/integration/scripts/trovestack \
           build-image \
-          ${TROVE_DATASTORE_TYPE} \
           ${TROVE_IMAGE_OS} \
           ${TROVE_IMAGE_OS_RELEASE} \
-          true
+          true \
+          ${TROVE_IMAGE_OS} \
+          ${image_file}
     fi
 
-    if [ ! -f ${image_file} ]; then
+    if [[ ! -f ${image_file} ]]; then
         echo "Image file was not found at ${image_file}"
         exit 1
     fi
@@ -485,7 +484,7 @@ function create_guest_image {
     $TROVE_MANAGE datastore_update $TROVE_DATASTORE_TYPE $TROVE_DATASTORE_VERSION
 
     echo "Add parameter validation rules if available"
-    if [ -f $DEST/trove/trove/templates/$TROVE_DATASTORE_TYPE/validation-rules.json ]; then
+    if [[ -f $DEST/trove/trove/templates/$TROVE_DATASTORE_TYPE/validation-rules.json ]]; then
         $TROVE_MANAGE db_load_datastore_config_parameters "$TROVE_DATASTORE_TYPE" "$TROVE_DATASTORE_VERSION" \
             $DEST/trove/trove/templates/$TROVE_DATASTORE_TYPE/validation-rules.json
     fi

devstack/settings

@@ -29,15 +29,9 @@ TROVE_LOCAL_API_PASTE_INI=${TROVE_LOCAL_API_PASTE_INI:-${TROVE_LOCAL_CONF_DIR}/a
 TROVE_LOCAL_POLICY_JSON=${TROVE_LOCAL_POLICY_JSON:-${TROVE_LOCAL_CONF_DIR}/policy.json}
 
 TROVE_IMAGE_OS=${TROVE_IMAGE_OS:-"ubuntu"}
-TROVE_IMAGE_OS_RELEASE=${TROVE_IMAGE_OS_RELEASE:-"xenial"}
+TROVE_IMAGE_OS_RELEASE=${TROVE_IMAGE_OS_RELEASE:-"bionic"}
 TROVE_DATASTORE_TYPE=${TROVE_DATASTORE_TYPE:-"mysql"}
-if [[ "$DISTRO" == "xenial" || "$DISTRO" == "bionic" ]]; then
+TROVE_DATASTORE_VERSION=${TROVE_DATASTORE_VERSION:-"5.7.29"}
-    TROVE_DATASTORE_VERSION=${TROVE_DATASTORE_VERSION:-"5.7"}
-    TROVE_DATASTORE_PACKAGE=${TROVE_DATASTORE_PACKAGE:-"mysql-server-5.7"}
-else
-    TROVE_DATASTORE_VERSION=${TROVE_DATASTORE_VERSION:-"5.6"}
-    TROVE_DATASTORE_PACKAGE=${TROVE_DATASTORE_PACKAGE:-"mysql-server-5.6"}
-fi
 
 # Configuration values listed here for reference
 TROVE_MAX_ACCEPTED_VOLUME_SIZE=${TROVE_MAX_ACCEPTED_VOLUME_SIZE}
@@ -46,8 +40,8 @@ TROVE_MAX_VOLUMES_PER_TENANT=${TROVE_MAX_VOLUMES_PER_TENANT}
 TROVE_AGENT_CALL_LOW_TIMEOUT=${TROVE_AGENT_CALL_LOW_TIMEOUT}
 TROVE_AGENT_CALL_HIGH_TIMEOUT=${TROVE_AGENT_CALL_HIGH_TIMEOUT:-1200}
 TROVE_RESIZE_TIME_OUT=${TROVE_RESIZE_TIME_OUT}
-TROVE_USAGE_TIMEOUT=${TROVE_USAGE_TIMEOUT:-900}
+TROVE_USAGE_TIMEOUT=${TROVE_USAGE_TIMEOUT:-1800}
-TROVE_STATE_CHANGE_WAIT_TIME=${TROVE_STATE_CHANGE_WAIT_TIME}
+TROVE_STATE_CHANGE_WAIT_TIME=${TROVE_STATE_CHANGE_WAIT_TIME:-600}
 TROVE_COMMAND_PROCESS_TIMEOUT=${TROVE_COMMAND_PROCESS_TIMEOUT:-60}
 
 # Set up the host gateway
@@ -90,4 +84,4 @@ CELLSV2_SETUP=singleconductor
 
 # Enable or disable the Trove guest image build during devstack installation.
 TROVE_ENABLE_IMAGE_BUILD=${TROVE_ENABLE_IMAGE_BUILD:-"true"}
-TROVE_NON_DEV_IMAGE_URL_MYSQL=${TROVE_NON_DEV_IMAGE_URL_MYSQL:-""}
+TROVE_NON_DEV_IMAGE_URL=${TROVE_NON_DEV_IMAGE_URL:-""}

doc/source/admin/building_guest_images.rst

@@ -26,54 +26,42 @@ stored in Glance. This document shows you the steps to build the guest images.
     periodically built and published in
     http://tarballs.openstack.org/trove/images/ in Trove upstream CI.
 
-    Additionally, if you install Trove in devstack environment, a MySQL image
+    Additionally, if you install Trove in devstack environment, the guest image
     is created and registered in Glance automatically, unless it's disabled by
     setting ``TROVE_ENABLE_IMAGE_BUILD=false`` in devstack local.conf file.
 
 High Level Overview of a Trove Guest Instance
 =============================================
 
-At the most basic level, a Trove Guest Instance is a Nova instance
+At the most basic level, a Trove Guest Instance is a Nova instance launched by
-launched by Trove in response to a create command. For most of this
+Trove in response to a create command. This section describes the various
-document, we will confine ourselves to single instance databases; in
+components of a Trove Guest Instance.
-other words, without the additional complexity of replication or
-mirroring. Guest instances and Guest images for replicated and
-mirrored database instances will be addressed specifically in later
-sections of this document.
 
-This section describes the various components of a Trove Guest
+----------------
-Instance.
+Operating System
+----------------
 
------------------------------
+The officially supported operating system is Ubuntu, based on which the
-Operating System and Database
+functional tests are running.
------------------------------
 
-A Trove Guest Instance contains at least a functioning Operating
+------
-System and the database software that the instance wishes to provide
+Docker
-(as a Service). For example, if your chosen operating system is Ubuntu
+------
-and you wish to deliver MySQL version 5.7, then your guest instance is
+
-a Nova instance running the Ubuntu operating system and will have
+Since Vitoria release, all the datastore services are installed by docker
-MySQL version 5.7 installed on it.
+container inside the Trove instance, so docker should be installed when
+building the guest image.
 
 -----------------
 Trove Guest Agent
 -----------------
 
-Trove supports multiple databases, some of them are relational (RDBMS)
+The guest agent runs inside the Nova instances that are used to run the
-and some are non-relational (NoSQL). In order to provide a common
+database engines. The agent listens to the messaging bus for the topic and is
-management interface to all of these, the Trove Guest Instance has on
+responsible for actually translating and executing the commands that are sent
-it a 'Guest Agent'. The Trove Guest Agent is a component of the
+to it by the task manager component for the particular datastore.
-Trove system that is specific to the database running on that Guest
-Instance.
 
-The purpose of the Trove Guest Agent is to implement the Trove Guest
+Trove guest agent is responsible for datastore docker container management.
-Agent API for the specific database. This includes such things as the
-implementation of the database 'start' and 'stop' commands. The Trove
-Guest Agent API is the common API used by Trove to communicate with
-any guest database, and the Guest Agent is the implementation of that
-API for the specific database.
-
-The Trove Guest Agent runs inside the Trove Guest Instance.
 
 ------------------------------------------
 Injected Configuration for the Guest Agent
@@ -104,44 +92,45 @@ services(e.g. the message queue).
 Building Guest Images
 =====================
 
+Since Victoria release, a single trove guest image can be used for different
+datastores, it's unnecessary to maintain different images for differnt
+datastores.
+
 -----------------------------
 Build images using trovestack
 -----------------------------
 
 ``trovestack`` is the recommended tooling provided by Trove community to build
-the guest images. Before running ``trovestack`` command, go to the scripts
+the guest images. Before running ``trovestack`` command:
-folder:
 
 .. code-block:: console
 
     git clone https://opendev.org/openstack/trove
     cd trove/integration/scripts
 
-The trove guest agent image could be created by running the following command:
+The trove guest image could be created by running the following command:
 
 .. code-block:: console
 
     $ ./trovestack build-image \
-        ${datastore_type} \
         ${guest_os} \
         ${guest_os_release} \
         ${dev_mode} \
         ${guest_username} \
-        ${imagepath}
+        ${output_image_path}
 
-* Currently, only ``guest_os=ubuntu`` and ``guest_os_release=xenial`` are fully
+* Currently, only ``guest_os=ubuntu`` and ``guest_os_release=bionic`` are fully
   tested and supported.
 
 * Default input values:
 
   .. code-block:: ini
 
-      datastore_type=mysql
       guest_os=ubuntu
-      guest_os_release=xenial
+      guest_os_release=bionic
       dev_mode=true
       guest_username=ubuntu
-      imagepath=$HOME/images/trove-${guest_os}-${guest_os_release}-${datastore_type}
+      output_image_path=$HOME/images/trove-guest--${guest_os}-${guest_os_release}-dev
 
 * ``dev_mode=true`` is mainly for testing purpose for trove developers and it's
   necessary to build the image on the trove controller host, because the host
@@ -159,31 +148,27 @@ The trove guest agent image could be created by running the following command:
   * ``HOST_SCP_USERNAME``: Only used in dev mode, this is the user name used by
     guest agent to connect to the controller host, e.g. in devstack
     environment, it should be the ``stack`` user.
-  * ``GUEST_WORKING_DIR``: The place to save the guest image, default value is
-    ``$HOME/images``.
-  * ``TROVE_BRANCH``: Only used in dev mode. The branch name of Trove code
-    repository, by default it's master, use other branches as needed such as
-    stable/train.
 
-For example, in order to build a MySQL image for Ubuntu Xenial operating
+For example, in order to build a guest image for Ubuntu Bionic operating
 system in development mode:
 
 .. code-block:: console
 
-    $ ./trovestack build-image mysql ubuntu xenial true
+    $ ./trovestack build-image ubuntu bionic true ubuntu
 
 Once the image build is finished, the cloud administrator needs to register the
 image in Glance and register a new datastore or version in Trove using
-``trove-manage`` command, e.g. after building an image for MySQL 5.7.1:
+``trove-manage`` command, e.g. after building an image for MySQL 5.7.29:
 
 .. code-block:: console
 
-    $ openstack image create ubuntu-mysql-5.7.1-dev \
+    $ openstack image create trove-guest-ubuntu-bionic \
-      --public \
+      --private \
       --disk-format qcow2 \
       --container-format bare \
-      --file ~/images/ubuntu-xenial-mysql.qcow2
+      --file ~/images/trove-guest-ubuntu-bionic-dev.qcow2
-    $ trove-manage datastore_version_update mysql 5.7.1 mysql $image_id "" 1
+    $ trove-manage datastore_version_update mysql 5.7.29 mysql $image_id "" 1
+    $ trove-manage db_load_datastore_config_parameters mysql 5.7.29 ${trove_repo_dir}/trove/templates/mysql/validation-rules.json
 
 If you see anything error or need help for the image creation, please ask help
 either in ``#openstack-trove`` IRC channel or sending emails to

doc/source/admin/run_trove_in_production.rst

@@ -322,7 +322,7 @@ Command examples:
     # Use 5.7.29 as the default datastore version for 'mysql'
     trove-manage datastore_update mysql 5.7.29
     # Register configuration parameters for 5.7.29 version of datastore 'mysql'
-    trove-manage db_load_datastore_config_parameters mysql 5.7.29 $workdir/trove/trove/templates/mysql/validation-rules.json
+    trove-manage db_load_datastore_config_parameters mysql 5.7.29 ${trove_repo_dir}}/trove/templates/mysql/validation-rules.json
 
 
 Quota Management

etc/trove/cloudinit/README

@@ -1,3 +0,0 @@
-These cloudinit scripts will used as userdata on instance create
-File names should match pattern: service_type.cloudinit
-For example: mysql.cloudinit

etc/trove/conf.d/README

@@ -1,4 +0,0 @@
-These conf files are read and used by the guest to provide extra
-information to the guest. The first example of this is the
-guest_info.conf which will have the uuid of the instance so that
-the guest can report back things to the infra.

etc/trove/conf.d/guest_info.conf

@@ -1 +0,0 @@
-# Guest-specific information injected by the taskmanager

etc/trove/trove-guestagent.conf.sample

@@ -1,166 +0,0 @@
-[DEFAULT]
-
-
-#=========== RPC Configuration ======================
-
-# URL representing the messaging driver to use and its full configuration.
-# If not set, we fall back to the 'rpc_backend' option and driver specific
-# configuration.
-#transport_url=<None>
-
-# The messaging driver to use. Options include rabbit, qpid and zmq.
-# Default is rabbit. (string value)
-#rpc_backend=rabbit
-
-# The default exchange under which topics are scoped. May be
-# overridden by an exchange name specified in the 'transport_url option.
-control_exchange = trove
-
-# ========== Configuration options for Swift ==========
-
-# The swift_url can be specified directly or fetched from Keystone catalog.
-
-# To fetch from Keystone, comment out swift_url, and uncomment the others.
-# swift_url = http://10.0.0.1:8080/v1/AUTH_
-# Region name of this node. Default value is None.
-# os_region_name = RegionOne
-# Service type to use when searching catalog.
-# swift_service_type = object-store
-
-
-# ========== Datastore Manager Configurations ==========
-
-# Datastore manager implementations.
-# Format: list of 'datastore-type:datastore.manager.implementation.module'
-# datastore_registry_ext = mysql:trove.guestagent.datastore.mysql.manager.Manager, percona:trove.guestagent.datastore.mysql.manager.Manager
-
-
-# ========== Default Users / DBs Configuration ==========
-
-# Permissions to grant "root" user by default
-root_grant = ALL
-root_grant_option = True
-# root_grant = ALTER ROUTINE, CREATE, ALTER, CREATE ROUTINE, CREATE TEMPORARY TABLES, CREATE VIEW, CREATE USER, DELETE, DROP, EVENT, EXECUTE, INDEX, INSERT, LOCK TABLES, PROCESS, REFERENCES, SELECT, SHOW DATABASES, SHOW VIEW, TRIGGER, UPDATE, USAGE
-# root_grant_option = False
-
-# Default password Length for root password
-# default_password_length = 36
-
-
-# ========== Default Storage Options for backup ==========
-
-# Default configuration for storage strategy and storage options
-# for backups
-
-# For storage to Swift, use the following as defaults:
-# storage_strategy = SwiftStorage
-# storage_namespace = trove.common.strategies.storage.swift
-
-# Default config options for storing backups to swift
-# backup_swift_container = database_backups
-# backup_use_gzip_compression = True
-# backup_use_openssl_encryption = True
-# backup_aes_cbc_key = "default_aes_cbc_key"
-# backup_use_snet = False
-# backup_chunk_size = 65536
-# backup_segment_max_size = 2147483648
-
-
-# ========== Sample Logging Configuration ==========
-
-# Show debugging output in logs (sets DEBUG log level output)
-# debug = True
-
-# Directory and path for log files
-log_dir = /var/log/trove/
-log_file = logfile.txt
-log_config_append = /etc/trove/trove-logging-guestagent.conf
-
-[profiler]
-# If False fully disable profiling feature.
-#enabled = False
-# If False doesn't trace SQL requests.
-#trace_sqlalchemy = True
-
-[oslo_messaging_notifications]
-
-#
-# From oslo.messaging
-#
-
-# The Driver(s) to handle sending notifications. Possible
-# values are messaging, messagingv2, routing, log, test, noop
-# (multi valued)
-# Deprecated group/name - [DEFAULT]/notification_driver
-#driver =
-
-# A URL representing the messaging driver to use for
-# notifications. If not set, we fall back to the same
-# configuration used for RPC. (string value)
-# Deprecated group/name - [DEFAULT]/notification_transport_url
-#transport_url = <None>
-
-# AMQP topic used for OpenStack notifications. (list value)
-# Deprecated group/name - [rpc_notifier2]/topics
-# Deprecated group/name - [DEFAULT]/notification_topics
-#topics = notifications
-
-# The maximum number of attempts to re-send a notification
-# message which failed to be delivered due to a recoverable
-# error. 0 - No retry, -1 - indefinite (integer value)
-#retry = -1
-
-# ========== Datastore Specific Configuration Options ==========
-
-[mysql]
-# For mysql, the following are the defaults for backup, and restore:
-# backup_strategy = InnoBackupEx
-# backup_namespace = trove.guestagent.strategies.backup.mysql_impl
-# restore_namespace = trove.guestagent.strategies.restore.mysql_impl
-# Default configuration for mysql replication
-# replication_strategy = MysqlBinlogReplication
-# replication_namespace = trove.guestagent.strategies.replication.mysql_binlog
-# replication_user = slave_user
-# replication_password = slave_password
-
-# Users to ignore for user create/list/delete operations
-# ignore_users = os_admin
-
-# Databases to ignore for db create/list/delete operations
-# ignore_dbs = mysql, information_schema, performance_schema
-
-[vertica]
-# For vertica, following are the defaults needed:
-# mount_point = /var/lib/vertica
-# readahead_size = 2048
-# guestagent_strategy = trove.common.strategies.cluster.experimental.vertica.guestagent.VerticaGuestAgentStrategy
-
-[redis]
-# For redis, the following are the defaults for backup, and restore:
-# backup_strategy = RedisBackup
-# backup_namespace = trove.guestagent.strategies.backup.experimental.redis_impl
-# restore_namespace = trove.guestagent.strategies.restore.experimental.redis_impl
-
-[percona]
-backup_namespace = trove.guestagent.strategies.backup.mysql_impl
-restore_namespace = trove.guestagent.strategies.restore.mysql_impl
-
-[couchbase]
-backup_namespace = trove.guestagent.strategies.backup.experimental.couchbase_impl
-restore_namespace = trove.guestagent.strategies.restore.experimental.couchbase_impl
-
-[cassandra]
-backup_namespace = trove.guestagent.strategies.backup.experimental.cassandra_impl
-restore_namespace = trove.guestagent.strategies.restore.experimental.cassandra_impl
-
-[db2]
-# For db2, the following are the defaults for backup, and restore:
-# backup_strategy = DB2OfflineBackup
-# backup_namespace = trove.guestagent.strategies.backup.experimental.db2_impl
-# restore_namespace = trove.guestagent.strategies.restore.experimental.db2_impl
-
-[couchdb]
-#For CouchDB, the following are the defaults for backup and restore:
-# backup_strategy = CouchDBBackup
-# backup_namespace = trove.guestagent.strategies.backup.experimental.couchdb_impl
-# restore_namespace = trove.guestagent.strategies.restore.experimental.couchdb_impl

etc/trove/trove-workbook.yaml

@@ -1,19 +0,0 @@
----
-version: '2.0'
-
-name: trove
-
-description: Trove Workflows
-
-workflows:
-
-  backup_create:
-    input: [instance, name, description, incremental]
-    output:
-      status: <% $.message %>
-
-    tasks:
-      backup_create:
-        action: trove.backups_create instance=<% $.instance %> name=<% $.name %> description=<% $.description %> incremental=<% $.incremental %>
-        publish:
-          message: <% 'Backup complete' %>

etc/trove/trove.conf.sample

@@ -1,311 +0,0 @@
-[DEFAULT]
-# Show debugging output in logs (sets DEBUG log level output)
-debug = True
-
-# Address to bind the API server
-bind_host = 0.0.0.0
-
-# Port the bind the API server to
-bind_port = 8779
-
-# Number of workers for the API service. The default will
-# be the number of CPUs available. (integer value)
-#trove_api_workers=None
-
-#===================== RPC Configuration =================================
-
-# URL representing the messaging driver to use and its full configuration.
-# If not set, we fall back to the 'rpc_backend' option and driver specific
-# configuration.
-#transport_url=<None>
-
-# The messaging driver to use. Options include rabbit, qpid and zmq.
-# Default is rabbit. (string value)
-#rpc_backend=rabbit
-
-# The default exchange under which topics are scoped. May be
-# overridden by an exchange name specified in the 'transport_url option.
-control_exchange = trove
-
-# Maximum line size of message headers to be accepted.
-# max_header_line may need to be increased when using large tokens
-# (typically those generated by the Keystone v3 API with big service
-# catalogs)
-# max_header_line = 16384
-
-#DB Api Implementation
-db_api_implementation = "trove.db.sqlalchemy.api"
-
-# Configuration options for talking to nova via the novaclient.
-trove_auth_url = http://0.0.0.0/identity/v2.0
-#nova_compute_url = http://localhost:8774/v2
-#cinder_url = http://localhost:8776/v1
-#swift_url = http://localhost:8080/v1/AUTH_
-#neutron_url = http://localhost:9696/
-
-# nova_compute_url, cinder_url, swift_url, and can all be fetched
-# from Keystone. To fetch from Keystone, comment out nova_compute_url,
-# cinder_url, swift_url, and and optionally uncomment the lines below.
-
-# Region name of this node. Used when searching catalog. Default value is None.
-#os_region_name = RegionOne
-# Service type to use when searching catalog.
-#nova_compute_service_type = compute
-# Service type to use when searching catalog.
-#cinder_service_type = volumev2
-# Service type to use when searching catalog.
-#swift_service_type = object-store
-# Service type to use when searching catalog.
-#neutron_service_type = network
-
-#ip_regex = ^(15.|123.)
-#black_list_regex = ^10.0.0.
-
-# Config options for enabling volume service
-trove_volume_support = True
-block_device_mapping = vdb
-device_path = /dev/vdb
-# Maximum volume size for an instance
-max_accepted_volume_size = 10
-max_instances_per_tenant = 5
-# Maximum volume capacity (in GB) spanning across all trove volumes per tenant
-max_volumes_per_tenant = 100
-max_backups_per_tenant = 5
-volume_time_out=30
-
-# Config options for rate limits
-http_get_rate = 200
-http_post_rate = 200
-http_put_rate = 200
-http_delete_rate = 200
-http_mgmt_post_rate = 200
-
-# Trove DNS
-trove_dns_support = False
-dns_account_id = 123456
-dns_auth_url = http://127.0.0.1/identity/v2.0
-dns_username = user
-dns_passkey = password
-dns_ttl = 3600
-dns_domain_name = 'trove.com.'
-dns_domain_id = 11111111-1111-1111-1111-111111111111
-dns_driver = trove.dns.designate.driver.DesignateDriver
-dns_instance_entry_factory = trove.dns.designate.driver.DesignateInstanceEntryFactory
-dns_endpoint_url = http://127.0.0.1/v1/
-dns_service_type = dns
-
-# Neutron
-network_driver = trove.network.nova.NovaNetwork
-management_networks =
-
-
-# Taskmanager queue name
-taskmanager_queue = taskmanager
-
-# Auth
-admin_roles = admin
-
-# Guest related conf
-agent_heartbeat_time = 10
-agent_call_low_timeout = 5
-agent_call_high_timeout = 150
-
-# Reboot time out for instances
-reboot_time_out = 60
-
-# Trove api-paste file name
-api_paste_config = api-paste.ini
-
-
-# ============ Notification System configuration ===========================
-
-# Sets the notification driver used by oslo.messaging. Options include
-# messaging, messagingv2, log and routing. Default is 'noop'
-# notification_driver=noop
-
-# Topics used for OpenStack notifications, list value. Default is 'notifications'.
-# notification_topics=notifications
-
-# ============ Logging information =============================
-#log_dir = /integration/report
-#log_file = trove-api.log
-
-
-[database]
-
-# SQLAlchemy connection string for the reference implementation
-# registry server. Any valid SQLAlchemy connection string is fine.
-# See: http://www.sqlalchemy.org/docs/05/reference/sqlalchemy/connections.html#sqlalchemy.create_engine
-# connection = sqlite:///trove_test.sqlite
-connection = mysql+pymysql://root:e1a2c042c828d3566d0a@localhost/trove
-#connection = postgresql://trove:trove@localhost/trove
-
-# Period in seconds after which SQLAlchemy should reestablish its connection
-# to the database.
-#
-# MySQL uses a default `wait_timeout` of 8 hours, after which it will drop
-# idle connections. This can result in 'MySQL Gone Away' exceptions. If you
-# notice this, you can lower this value to ensure that SQLAlchemy reconnects
-# before MySQL can drop the connection.
-idle_timeout = 3600
-
-
-
-# ============ SSL configuration (and enablement) =============================
-# In order to enable SSL for the trove api server, uncomment
-# the cert_file and key_file - and of course have those files
-# accessible. The existence of those setting and files will
-# enable SSL.
-
-[profiler]
-# If False fully disable profiling feature.
-#enabled = False
-# If False doesn't trace SQL requests.
-#trace_sqlalchemy = True
-
-[ssl]
-
-#cert_file = /path/to/server.crt
-#key_file = /path/to/server.key
-#optional:
-#ca_file = /path/to/ca_file
-
-[oslo_messaging_notifications]
-
-#
-# From oslo.messaging
-#
-
-# The Driver(s) to handle sending notifications. Possible
-# values are messaging, messagingv2, routing, log, test, noop
-# (multi valued)
-# Deprecated group/name - [DEFAULT]/notification_driver
-#driver =
-
-# A URL representing the messaging driver to use for
-# notifications. If not set, we fall back to the same
-# configuration used for RPC. (string value)
-# Deprecated group/name - [DEFAULT]/notification_transport_url
-#transport_url = <None>
-
-# AMQP topic used for OpenStack notifications. (list value)
-# Deprecated group/name - [rpc_notifier2]/topics
-# Deprecated group/name - [DEFAULT]/notification_topics
-#topics = notifications
-
-# The maximum number of attempts to re-send a notification
-# message which failed to be delivered due to a recoverable
-# error. 0 - No retry, -1 - indefinite (integer value)
-#retry = -1
-
-[mysql]
-root_on_create = False
-# Format (single port or port range): A, B-C
-# where C greater than B
-tcp_ports = 3306
-volume_support = True
-device_path = /dev/vdb
-
-# Users to ignore for user create/list/delete operations
-ignore_users = os_admin, root
-ignore_dbs = mysql, information_schema, performance_schema
-
-
-[redis]
-tcp_ports = 6379, 16379
-volume_support = True
-device_path = /dev/vdb
-
-[cassandra]
-tcp_ports = 7000, 7001, 9042, 9160
-volume_support = True
-device_path = /dev/vdb
-
-[couchbase]
-tcp_ports = 8091, 8092, 4369, 11209-11211, 21100-21199
-volume_support = True
-device_path = /dev/vdb
-
-[mongodb]
-tcp_ports = 2500, 27017, 27019
-volume_support = True
-device_path = /dev/vdb
-num_config_servers_per_cluster = 1
-num_query_routers_per_cluster = 1
-
-[vertica]
-tcp_ports = 5433, 5434, 22, 5444, 5450, 4803
-udp_ports = 5433, 4803, 4804, 6453
-volume_support = True
-device_path = /dev/vdb
-cluster_support = True
-cluster_member_count = 3
-api_strategy = trove.common.strategies.cluster.experimental.vertica.api.VerticaAPIStrategy
-
-
-# ============ CORS configuration =============================
-
-[cors]
-
-#
-# From oslo.middleware.cors
-#
-
-# Indicate whether this resource may be shared with the domain received in the
-# requests "origin" header. (list value)
-#allowed_origin = <None>
-
-# Indicate that the actual request can include user credentials (boolean value)
-#allow_credentials = true
-
-# Indicate which headers are safe to expose to the API. Defaults to HTTP Simple
-# Headers. (list value)
-#expose_headers = X-Auth-Token, X-Subject-Token, X-Service-Token, X-OpenStack-Request-ID
-
-# Maximum cache age of CORS preflight requests. (integer value)
-#max_age = 3600
-
-# Indicate which methods can be used during the actual request. (list value)
-#allow_methods = GET,PUT,POST,DELETE,PATCH
-
-# Indicate which header field names may be used during the actual request.
-# (list value)
-#allow_headers = X-Auth-Token, X-Identity-Status, X-Roles, X-Service-Catalog, X-User-Id, X-Tenant-Id, X-OpenStack-Request-ID
-
-
-[cors.subdomain]
-
-#
-# From oslo.middleware.cors
-#
-
-# Indicate whether this resource may be shared with the domain received in the
-# requests "origin" header. (list value)
-#allowed_origin = <None>
-
-# Indicate that the actual request can include user credentials (boolean value)
-#allow_credentials = true
-
-# Indicate which headers are safe to expose to the API. Defaults to HTTP Simple
-# Headers. (list value)
-#expose_headers = X-Auth-Token, X-Subject-Token, X-Service-Token, X-OpenStack-Request-ID
-
-# Maximum cache age of CORS preflight requests. (integer value)
-#max_age = 3600
-
-# Indicate which methods can be used during the actual request. (list value)
-#allow_methods = GET,PUT,POST,DELETE,PATCH
-
-# Indicate which header field names may be used during the actual request.
-# (list value)
-#allow_headers = X-Auth-Token, X-Identity-Status, X-Roles, X-Service-Catalog, X-User-Id, X-Tenant-Id, X-OpenStack-Request-ID
-
-[oslo_middleware]
-
-#
-# From oslo.middleware
-#
-
-# Whether the application is behind a proxy or not. This determines if the
-# middleware should parse the headers or not. (boolean value)
-#enable_proxy_headers_parsing = false

integration/scripts/conf.json.example

@@ -1,12 +0,0 @@
-{
-    "devstack":null,
-    "glance":null,
-    "horizon":null,
-    "keystone":null,
-    "nova":null,
-    "python_openstackclient":null,
-    "python_novaclient":null,
-    "trove":null,
-    "python_troveclient":null,
-    "tempest":null
-}

integration/scripts/files/elements/apt-conf-dir/README.rst

@@ -1,16 +0,0 @@
-============
-apt-conf-dir
-============
-
-This element overrides the default apt.conf.d directory for APT based systems.
-
-Environment Variables
----------------------
-
-DIB_APT_CONF_DIR:
-   :Required: No
-   :Default: None
-   :Description: To override `DIB_APT_CONF_DIR`, set it to the path to your
-                 apt.conf.d. The new apt.conf.d will take effect at build time
-                 and run time.
-   :Example: ``DIB_APT_CONF_DIR=/etc/apt/apt.conf``

integration/scripts/files/elements/apt-conf-dir/extra-data.d/99-use-host-apt-confd

@@ -1,21 +0,0 @@
-#!/bin/bash
-# Override the default /etc/apt/apt.conf.d directory with $DIB_APT_CONF_DIR
-
-if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
-    set -x
-fi
-set -eu
-set -o pipefail
-
-# exit directly if DIB_APT_CONF_DIR is not defined properly
-if [ -z "${DIB_APT_CONF_DIR:-}" ] ; then
-    echo "DIB_APT_CONF_DIR is not set - no apt.conf.d will be copied in"
-    exit 0
-elif [ ! -d "$DIB_APT_CONF_DIR" ] ; then
-    echo "$DIB_APT_CONF_DIR is not a valid apt.conf.d directory."
-    echo "You should assign a proper apt.conf.d directory in DIB_APT_CONF_DIR"
-    exit 1
-fi
-
-# copy the apt.conf to cloudimg
-sudo cp -L -f -R $DIB_APT_CONF_DIR $TMP_MOUNT_PATH/etc/apt

integration/scripts/files/elements/guest-agent/element-deps

@@ -4,3 +4,4 @@ pkg-map
 source-repositories
 svc-map
 pip-and-virtualenv
+ubuntu-docker

integration/scripts/files/elements/guest-agent/environment.d/99-reliable-apt-key-importing.bash

@@ -1,34 +0,0 @@
-# sometimes the primary key server is unavailable and we should try an
-# alternate.  see
-# https://bugs.launchpad.net/percona-server/+bug/907789.  Disable
-# shell errexit so we can interrogate the exit code and take action
-# based on the exit code. We will reenable it later.
-#
-# NOTE(zhaochao): we still have this problem from time to time, so it's
-# better use more reliable keyservers and just retry on that(for now, 3
-# tries should be fine).
-# According to:
-# [1] https://www.gnupg.org/faq/gnupg-faq.html#new_user_default_keyserver
-# [2] https://sks-keyservers.net/overview-of-pools.php
-# we'll just the primary suggested pool: pool.sks-keyservers.net.
-function get_key_robust() {
-    KEY=$1
-    set +e
-
-    tries=1
-    while [ $tries -le 3 ]; do
-        if [ $tries -eq 3 ]; then
-            set -e
-        fi
-
-        echo "Importing the key, try: $tries"
-        # Behind a firewall should use the port 80 instead of the default port 11371
-        apt-key adv --keyserver hkp://pool.sks-keyservers.net:80 --recv-keys ${KEY} && break
-
-        tries=$((tries+1))
-    done
-
-    set -e
-}
-
-export -f get_key_robust

integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/31-guest-agent-install

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+DEV_MODE=${DEV_MODE:-"true"}
+SCRIPTDIR=$(dirname $0)
+GUEST_USERNAME=${GUEST_USERNAME:-"ubuntu"}
+GUEST_VENV=/opt/guest-agent-venv
+
+for folder in "/var/lib/trove" "/etc/trove" "/etc/trove/certs" "/etc/trove/conf.d" "/var/log/trove" "/opt/trove-guestagent"; do
+    mkdir -p ${folder}
+    chown -R ${GUEST_USERNAME}:root ${folder}
+done
+
+install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.logrotate /etc/logrotate.d/guest-agent
+
+# Create a virtual environment (with dependencies installed) for guest agent service
+${DIB_PYTHON} -m virtualenv ${GUEST_VENV}
+${GUEST_VENV}/bin/pip install pip --upgrade
+${GUEST_VENV}/bin/pip install -U -c /opt/upper-constraints.txt /opt/guest-agent
+chown -R ${GUEST_USERNAME}:root ${GUEST_VENV}
+
+if [[ ${DEV_MODE} == "true" ]]; then
+    [[ -n "${HOST_SCP_USERNAME}" ]] || die "HOST_SCP_USERNAME needs to be set to the trovestack host user"
+    [[ -n "${ESCAPED_PATH_TROVE}" ]] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the trovestack host"
+
+    sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${SCRIPTDIR}/guest-agent-dev.service > /etc/systemd/system/guest-agent.service
+else
+    # Link the trove-guestagent out to /usr/local/bin where the startup scripts look for
+    ln -s ${GUEST_VENV}/bin/trove-guestagent /usr/local/bin/guest-agent || true
+
+    case "$DIB_INIT_SYSTEM" in
+        systemd)
+            sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g" ${SCRIPTDIR}/guest-agent.service > /etc/systemd/system/guest-agent.service
+            ;;
+        upstart)
+            install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.conf /etc/init/guest-agent.conf
+            ;;
+        sysv)
+            install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.init /etc/init.d/guest-agent.init
+            ;;
+        *)
+            echo "Unsupported init system"
+            exit 1
+            ;;
+    esac
+fi

integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/75-guest-agent-install

@@ -1,45 +0,0 @@
-#!/bin/bash
-
-if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
-    set -x
-fi
-set -eu
-set -o pipefail
-
-SCRIPTDIR=$(dirname $0)
-GUEST_VENV=/opt/guest-agent-venv
-GUEST_USERNAME=${GUEST_USERNAME:-"ubuntu"}
-
-# Create a virtual environment for guest agent
-${DIB_PYTHON} -m virtualenv ${GUEST_VENV}
-${GUEST_VENV}/bin/pip install pip --upgrade
-${GUEST_VENV}/bin/pip install -U -c /opt/upper-constraints.txt /opt/guest-agent
-chown -R ${GUEST_USERNAME}:root ${GUEST_VENV}
-
-# Link the trove-guestagent out to /usr/local/bin where the startup scripts look for
-ln -s ${GUEST_VENV}/bin/trove-guestagent /usr/local/bin/guest-agent || true
-
-for folder in "/var/lib/trove" "/etc/trove" "/etc/trove/certs" "/etc/trove/conf.d" "/var/log/trove"; do
-    mkdir -p ${folder}
-    chown -R ${GUEST_USERNAME}:root ${folder}
-done
-
-install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.logrotate /etc/logrotate.d/guest-agent
-
-case "$DIB_INIT_SYSTEM" in
-    systemd)
-        mkdir -p /usr/lib/systemd/system
-        touch /usr/lib/systemd/system/guest-agent.service
-        sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g" ${SCRIPTDIR}/guest-agent.service > /usr/lib/systemd/system/guest-agent.service
-        ;;
-    upstart)
-        install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.conf /etc/init/guest-agent.conf
-        ;;
-    sysv)
-        install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.init /etc/init.d/guest-agent.init
-        ;;
-    *)
-        echo "Unsupported init system"
-        exit 1
-        ;;
-esac

integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-agent-dev.service

@@ -0,0 +1,31 @@
+[Unit]
+Description=OpenStack Trove Guest Agent Service for Development
+After=syslog.target network.target
+
+[Install]
+WantedBy=multi-user.target
+
+[Service]
+Type=simple
+User=GUEST_USERNAME
+Group=GUEST_USERNAME
+
+# This script is only for testing purpose for dev_mode=true, the controller
+# IP address should be defined in /etc/trove/controller.conf, e.g.
+# CONTROLLER=192.168.32.151
+EnvironmentFile=/etc/trove/controller.conf
+
+# If ~/trove-installed does not exist, copy the trove source from
+# the user's development environment, then touch the sentinel file
+ExecStartPre=/bin/bash -c "test -e /home/GUEST_USERNAME/trove-installed || sudo rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i /home/GUEST_USERNAME/.ssh/id_rsa' -az --exclude='.*' HOST_SCP_USERNAME@$CONTROLLER:PATH_TROVE/ /home/GUEST_USERNAME/trove && touch /home/GUEST_USERNAME/trove-installed"
+
+ExecStartPre=/bin/bash -c "sudo chown -R GUEST_USERNAME:root /etc/trove /var/log/trove/ /home/GUEST_USERNAME/trove"
+
+# Take care of the changes in requirements.txt
+ExecStartPre=/bin/bash -c "sudo /opt/guest-agent-venv/bin/pip install -r /home/GUEST_USERNAME/trove/requirements.txt -c /opt/upper-constraints.txt"
+
+# Start guest-agent.service in virtual environment
+ExecStart=/bin/bash -c "/opt/guest-agent-venv/bin/python /home/GUEST_USERNAME/trove/contrib/trove-guestagent --config-dir=/etc/trove/conf.d"
+
+TimeoutSec=300
+Restart=on-failure

integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-agent.service

@@ -1,15 +1,16 @@
 [Unit]
-Description=OpenStack Trove Guest Agent
+Description=OpenStack Trove Guest Agent Service
 After=network.target syslog.service
 Wants=syslog.service
 
 [Service]
 User=GUEST_USERNAME
 Group=GUEST_USERNAME
-ExecStartPre=/bin/bash -c "sudo chown -R GUEST_USERNAME:root /etc/trove/conf.d"
-ExecStart=/usr/local/bin/guest-agent --config-dir=/etc/trove/conf.d
 KillMode=mixed
 Restart=always
 
+ExecStartPre=/bin/bash -c "sudo chown -R GUEST_USERNAME:root /etc/trove /var/log/trove/"
+ExecStart=/usr/local/bin/guest-agent --config-dir=/etc/trove/conf.d
+
 [Install]
 WantedBy=multi-user.target

integration/scripts/files/elements/guest-agent/package-installs.yaml

@@ -1,53 +1,15 @@
 guest-agent:
   installtype: package
 build-essential:
-  installtype: source
+python3-all:
+python3-all-dev:
+python3-pip:
+python3-sqlalchemy:
+libxml2-dev:
+libxslt1-dev:
 libffi-dev:
-  installtype: source
 libssl-dev:
-  installtype: source
+libyaml-dev:
-python-dev:
-  installtype: source
-
-acl:
-acpid:
-apparmor:
-apparmor-utils:
-apt-transport-https:
-at:
-bash-completion:
-cloud-guest-utils:
-cloud-init:
-cron:
-curl:
-dbus:
-dkms:
-dmeventd:
-ethtool:
-gawk:
-ifenslave:
-ifupdown:
-iptables:
-iputils-tracepath:
-irqbalance:
-isc-dhcp-client:
-less:
-logrotate:
-lsof:
-net-tools:
-netbase:
-netcat-openbsd:
-open-vm-tools:
-  arch: i386, amd64
 openssh-client:
 openssh-server:
-pollinate:
+rsync:
-psmisc:
-rsyslog:
-socat:
-tcpdump:
-ubuntu-cloudimage-keyring:
-ureadahead:
-uuid-runtime:
-vim-tiny:
-vlan:

integration/scripts/files/elements/no-resolvconf/README.rst

@@ -1,8 +0,0 @@
-This element clears out /etc/resolv.conf and prevents dhclient from populating
-it with data from DHCP. This means that DNS resolution will not work from the
-guest. This is OK because all outbound connections from the guest will
-be based using raw IP addresses.
-
-In addition we remove dns from the nsswitch.conf hosts setting.
-
-This means that the guest never waits for DNS timeouts to occur.

integration/scripts/files/elements/no-resolvconf/finalise.d/99-disable-resolv-conf

@@ -1,19 +0,0 @@
-#!/bin/bash
-echo "" > /etc/resolv.conf
-echo "" > /etc/resolv.conf.ORIG
-if [ -d /etc/dhcp/dhclient-enter-hooks.d ]; then
-    # Debian/Ubuntu
-    echo "#!/bin/sh
-make_resolv_conf() { : ; }" > /etc/dhcp/dhclient-enter-hooks.d/noresolvconf
-    chmod +x /etc/dhcp/dhclient-enter-hooks.d/noresolvconf
-    rm -f /etc/dhcp/dhclient-enter-hooks.d/resolvconf
-else
-    # RHEL/CentOS/Fedora
-    echo "#!/bin/sh
-make_resolv_conf() { : ; }" > /etc/dhclient-enter-hooks
-    chmod +x /etc/dhclient-enter-hooks
-fi
-
-if [ -e /etc/nsswitch.conf ]; then
-    sed -i -e "/hosts:/ s/dns//g" /etc/nsswitch.conf
-fi

integration/scripts/files/elements/ubuntu-docker/element-deps

@@ -0,0 +1 @@
+ubuntu-guest

integration/scripts/files/elements/ubuntu-docker/install.d/21-docker

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+GUEST_USERNAME=${GUEST_USERNAME:-"ubuntu"}
+
+echo "Installing docker"
+export DEBIAN_FRONTEND=noninteractive
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu ${DIB_RELEASE} stable"
+apt-get update
+apt-get install -y -qq docker-ce >/dev/null
+
+echo "Adding ${GUEST_USERNAME} user to docker group"
+usermod -aG docker ${GUEST_USERNAME}

integration/scripts/files/elements/ubuntu-guest/environment.d/99-reliable-apt-key-importing.bash

@@ -1,34 +0,0 @@
-# sometimes the primary key server is unavailable and we should try an
-# alternate.  see
-# https://bugs.launchpad.net/percona-server/+bug/907789.  Disable
-# shell errexit so we can interrogate the exit code and take action
-# based on the exit code. We will reenable it later.
-#
-# NOTE(zhaochao): we still have this problem from time to time, so it's
-# better use more reliable keyservers and just retry on that(for now, 3
-# tries should be fine).
-# According to:
-# [1] https://www.gnupg.org/faq/gnupg-faq.html#new_user_default_keyserver
-# [2] https://sks-keyservers.net/overview-of-pools.php
-# we'll just the primary suggested pool: pool.sks-keyservers.net.
-function get_key_robust() {
-    KEY=$1
-    set +e
-
-    tries=1
-    while [ $tries -le 3 ]; do
-        if [ $tries -eq 3 ]; then
-            set -e
-        fi
-
-        echo "Importing the key, try: $tries"
-        # Behind a firewall should use the port 80 instead of the default port 11371
-        apt-key adv --keyserver hkp://pool.sks-keyservers.net:80 --recv-keys ${KEY} && break
-
-        tries=$((tries+1))
-    done
-
-    set -e
-}
-
-export -f get_key_robust

integration/scripts/files/elements/ubuntu-guest/extra-data.d/11-ssh-key-dev

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+source $_LIB/die
+
+[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
+
+# Guest agent needs to ssh into the controller to download code in dev mode.
+if [[ ${DEV_MODE} == "true" && -e ${SSH_DIR}/id_rsa ]]; then
+   sudo -Hiu ${HOST_SCP_USERNAME} dd if=${SSH_DIR}/id_rsa of=${TMP_HOOKS_PATH}/id_rsa
+   sudo -Hiu ${HOST_SCP_USERNAME} dd if=${SSH_DIR}/id_rsa.pub of=${TMP_HOOKS_PATH}/id_rsa.pub
+fi

integration/scripts/files/elements/ubuntu-guest/extra-data.d/15-trove-dep

@@ -1,28 +0,0 @@
-#!/bin/bash
-
-set -e
-set -o xtrace
-
-# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER
-# PURPOSE: Setup the requirements file for use by 15-reddwarf-dep
-
-source $_LIB/die
-
-TROVE_BRANCH=${TROVE_BRANCH:-'master'}
-REQUIREMENTS_FILE=${TROVESTACK_SCRIPTS}/../../requirements.txt
-
-[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
-[ -e ${REQUIREMENTS_FILE} ] || die "Requirements not found"
-[ -n "$HOST_USERNAME" ] || die "HOST_USERNAME not set"
-
-sudo -Hiu ${HOST_USERNAME} dd if=${REQUIREMENTS_FILE} of=${TMP_HOOKS_PATH}/requirements.txt
-
-UC_FILE=upper-constraints.txt
-UC_DIR=$(pwd)
-UC_BRANCH=${TROVE_BRANCH##stable/}
-
-curl -L -o "${UC_DIR}/${UC_FILE}" "https://releases.openstack.org/constraints/upper/${UC_BRANCH}"
-if [ -f "${UC_DIR}/${UC_FILE}" ]; then
-    sudo -Hiu ${HOST_USERNAME} dd if="${UC_DIR}/${UC_FILE}" of=${TMP_HOOKS_PATH}/${UC_FILE}
-    rm -f "${UC_DIR}/${UC_FILE}"
-fi

integration/scripts/files/elements/ubuntu-guest/extra-data.d/62-ssh-key

@@ -1,27 +0,0 @@
-#!/bin/bash
-
-set -e
-set -o xtrace
-
-# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER
-# PURPOSE: creates the SSH key on the host if it doesn't exist.  Then this copies the keys over to a staging area where
-# they will be duplicated in the guest VM.
-# This process allows the host to log into the guest but more importantly the guest phones home to get the trove
-# source
-
-source $_LIB/die
-
-[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
-[ -n "${HOST_USERNAME}" ] || die "HOST_USERNAME needs to be set to the user for the current user on the host"
-
-if [ `whoami` = "root" ]; then
-    die "This should not be run as root"
-fi
-
-# Guest agent needs to ssh into the controller to download code in dev mode.
-if [ -e ${SSH_DIR}/id_rsa ]; then
-   sudo -Hiu ${HOST_USERNAME} dd if=${SSH_DIR}/id_rsa of=${TMP_HOOKS_PATH}/id_rsa
-   sudo -Hiu ${HOST_USERNAME} dd if=${SSH_DIR}/id_rsa.pub of=${TMP_HOOKS_PATH}/id_rsa.pub
-else
-    die "SSH keys must exist"
-fi

integration/scripts/files/elements/ubuntu-guest/install.d/05-base-apps

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-# CONTEXT: GUEST during CONSTRUCTION as ROOT
-# PURPOSE: Install basic services and applications
-
-set -e
-set -o xtrace
-
-export DEBIAN_FRONTEND=noninteractive
-apt-get --allow-unauthenticated -y install ntp apparmor-utils

integration/scripts/files/elements/ubuntu-guest/install.d/12-ssh-key-dev

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+GUEST_SSH_DIR="/home/${GUEST_USERNAME}/.ssh"
+TMP_HOOKS_DIR="/tmp/in_target.d"
+
+if [ ! -e ${GUEST_SSH_DIR} ]; then
+    # this method worked more reliable in vmware fusion over doing sudo -Hiu ${GUEST_USERNAME}
+    mkdir ${GUEST_SSH_DIR}
+    chown -R ${GUEST_USERNAME}:${GUEST_USERNAME} ${GUEST_SSH_DIR}
+fi
+
+if [[ ${DEV_MODE} == "true" && -e "${TMP_HOOKS_DIR}/id_rsa" ]]; then
+    sudo -Hiu ${GUEST_USERNAME} dd of=${GUEST_SSH_DIR}/id_rsa.pub if=${TMP_HOOKS_DIR}/id_rsa.pub
+    sudo -Hiu ${GUEST_USERNAME} dd of=${GUEST_SSH_DIR}/id_rsa if=${TMP_HOOKS_DIR}/id_rsa
+    sudo -Hiu ${GUEST_USERNAME} chmod 600 ${GUEST_SSH_DIR}/id_rsa
+fi

integration/scripts/files/elements/ubuntu-guest/install.d/15-trove-dep

@@ -1,37 +0,0 @@
-#!/bin/bash
-
-# CONTEXT: GUEST during CONSTRUCTION as ROOT
-# PURPOSE: Install trove guest python dependencies - see trovestack functions_qemu
-
-set -e
-set -o xtrace
-
-export DEBIAN_FRONTEND=noninteractive
-apt-get --allow-unauthenticated -y install \
-    libxml2-dev libxslt1-dev libffi-dev libssl-dev libyaml-dev \
-    python3-pip python3-sqlalchemy python3-setuptools
-
-# Install python 3.7, some python lib (e.g. oslo.concurrency>4.0.0) requries
-# Python 3.7
-add-apt-repository --yes ppa:deadsnakes/ppa
-apt update
-apt install -y python3.7 python3.7-dev
-
-update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 10
-python3.5 -m pip install pip==9.0.3
-python3.5 -m pip install -U wheel setuptools
-
-TMP_HOOKS_DIR="/tmp/in_target.d"
-
-UPPER_CONSTRAINTS=
-if [ -f ${TMP_HOOKS_DIR}/upper-constraints.txt ]; then
-    UPPER_CONSTRAINTS=" -c ${TMP_HOOKS_DIR}/upper-constraints.txt"
-fi
-
-python3.7 -m pip install pip==9.0.3
-python3.7 -m pip install -U wheel setuptools
-python3.7 -m pip install --upgrade -r ${TMP_HOOKS_DIR}/requirements.txt ${UPPER_CONSTRAINTS}
-
-echo "diagnostic pip freeze output follows"
-python3.7 -m pip freeze
-echo "diagnostic pip freeze output above"

integration/scripts/files/elements/ubuntu-guest/install.d/50-user

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-# CONTEXT: GUEST during CONSTRUCTION as ROOT
-# PURPOSE: Add the guest image user that will own the trove agent source...if the user does not already exist
-
-set -e
-set -o xtrace
-
-if ! id -u ${GUEST_USERNAME} >/dev/null 2>&1; then
-    echo "Adding ${GUEST_USERNAME} user"
-    useradd -G sudo -m ${GUEST_USERNAME} -s /bin/bash
-    chown ${GUEST_USERNAME}:${GUEST_USERNAME} /home/${GUEST_USERNAME}
-    passwd ${GUEST_USERNAME} <<_EOF_
-${GUEST_USERNAME}
-${GUEST_USERNAME}
-_EOF_
-fi
-

integration/scripts/files/elements/ubuntu-guest/install.d/62-ssh-key

@@ -1,25 +0,0 @@
-#!/bin/bash
-
-# PURPOSE: take "staged" ssh keys (see extra-data.d/62-ssh-key) and put them in the GUEST_USERS home directory
-# In future, this should be removed and use Nova keypair to inject ssh keys.
-
-set -e
-set -o xtrace
-
-SSH_DIR="/home/${GUEST_USERNAME}/.ssh"
-TMP_HOOKS_DIR="/tmp/in_target.d"
-
-if [ ! -e ${SSH_DIR} ]; then
-    # this method worked more reliable in vmware fusion over doing sudo -Hiu ${GUEST_USERNAME}
-    mkdir ${SSH_DIR}
-    chown ${GUEST_USERNAME}:${GUEST_USERNAME} ${SSH_DIR}
-fi
-
-if [ -e "${TMP_HOOKS_DIR}/id_rsa" ]; then
-    sudo -Hiu ${GUEST_USERNAME} dd of=${SSH_DIR}/id_rsa.pub if=${TMP_HOOKS_DIR}/id_rsa.pub
-    sudo -Hiu ${GUEST_USERNAME} dd of=${SSH_DIR}/id_rsa if=${TMP_HOOKS_DIR}/id_rsa
-    sudo -Hiu ${GUEST_USERNAME} chmod 600 ${SSH_DIR}/id_rsa
-else
-    echo "SSH Keys were not staged by host"
-    exit -1
-fi

integration/scripts/files/elements/ubuntu-guest/install.d/98-ssh

@@ -1,8 +0,0 @@
-#!/bin/bash
-# Regenerate host keys now. XXX: Really should be a cloud-init task, should get
-# that working.
-
-set -e
-set -o xtrace
-
-dpkg-reconfigure openssh-server

integration/scripts/files/elements/ubuntu-guest/install.d/99-clean-apt

@@ -1,11 +0,0 @@
-#!/bin/sh
-
-# CONTEXT: GUEST during CONSTRUCTION as ROOT
-# PURPOSE: Delete contents of apt cache on guest (saves image disk space)
-
-set -e
-set -o xtrace
-
-apt-get clean
-
-

integration/scripts/files/elements/ubuntu-guest/pre-install.d/04-baseline-tools

@@ -1,7 +0,0 @@
-#!/bin/bash
-# Install baseline packages and tools.
-
-set -e
-set -o xtrace
-
-apt-get --allow-unauthenticated install -y language-pack-en python-software-properties software-properties-common

integration/scripts/files/elements/ubuntu-guest/pre-install.d/11-baseline-tools

@@ -0,0 +1,7 @@
+#!/bin/bash
+# Install baseline packages and tools.
+
+set -e
+set -o xtrace
+
+DEBIAN_FRONTEND=noninteractive apt-get --allow-unauthenticated install -y -qq software-properties-common apt-transport-https ca-certificates ntp >/dev/null

integration/scripts/files/elements/ubuntu-mariadb/README.md

@@ -1,3 +0,0 @@
-Sets up a MariaDB server install in the image.
-
-TODO: auto-tune settings based on host resources or metadata service.

integration/scripts/files/elements/ubuntu-mariadb/pre-install.d/20-apparmor-mysql-local

@@ -1,11 +0,0 @@
-#!/bin/sh
-
-set -e
-
-#CONTEXT: chroot on host
-#PURPOSE: Allows mysqld to create temporary files when restoring backups
-
-cat <<EOF >>/etc/apparmor.d/local/usr.sbin.mysqld
-  /tmp/ rw,
-  /tmp/** rwk,
-EOF

integration/scripts/files/elements/ubuntu-mysql/pre-install.d/20-apparmor-mysql-local

@@ -1,12 +0,0 @@
-#!/bin/sh
-
-set -e
-
-#CONTEXT: chroot on host
-#PURPOSE: Allows mysqld to create temporary files when restoring backups
-
-mkdir -p /etc/apparmor.d/local/
-cat <<EOF >>/etc/apparmor.d/local/usr.sbin.mysqld
-  /tmp/ rw,
-  /tmp/** rwk,
-EOF

integration/scripts/files/elements/ubuntu-postgresql/install.d/30-postgresql

@@ -1,25 +0,0 @@
-#!/bin/sh
-
-set -e
-set -o xtrace
-
-export DEBIAN_FRONTEND=noninteractive
-
-cat > "/etc/sysctl.d/10-postgresql-performance.conf" << _EOF_
-# See 'http://www.postgresql.org/docs/9.6/static/kernel-resources.html'
-# for best practices.
-# It is recommended to disable memory overcommit,
-# but the Python interpreter may require it on smaller flavors.
-# We therefore stick with the heuristic overcommit setting.
-vm.overcommit_memory=0
-_EOF_
-
-apt-get --allow-unauthenticated -y install libpq-dev postgresql-12 postgresql-server-dev-12 postgresql-client-12
-
-pgsql_conf=/etc/postgresql/12/main/postgresql.conf
-sed -i "/listen_addresses/c listen_addresses = '*'" ${pgsql_conf}
-
-systemctl restart postgresql
-
-# Install the native Python client.
-pip3 install psycopg2

integration/scripts/files/elements/ubuntu-postgresql/pre-install.d/10-postgresql-repo

@@ -1,14 +0,0 @@
-#!/bin/sh
-
-set -e
-set -o xtrace
-
-[ -n "${DIB_RELEASE}" ] || die "RELEASE must be set to a valid Ubuntu release (e.g. trusty)"
-
-cat <<EOL > /etc/apt/sources.list.d/postgresql.list
-deb http://apt.postgresql.org/pub/repos/apt/ ${DIB_RELEASE}-pgdg main
-EOL
-
-wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-
-apt-get update

integration/scripts/files/elements/ubuntu-xenial-guest/element-deps

@@ -1 +0,0 @@
-ubuntu-guest

integration/scripts/files/elements/ubuntu-xenial-guest/extra-data.d/20-guest-systemd

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-set -e
-set -o xtrace
-
-# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER
-# PURPOSE: stages the bootstrap file and upstart conf file while replacing variables so that guest image is properly
-# configured
-
-source $_LIB/die
-
-[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
-
-[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image"
-[ -n "${HOST_SCP_USERNAME}" ] || die "HOST_SCP_USERNAME needs to be set to the user for the host instance"
-[ -n "${ESCAPED_PATH_TROVE}" ] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the trovestack host"
-[ -n "${TROVESTACK_SCRIPTS}" ] || die "TROVESTACK_SCRIPTS needs to be set to the trove/integration/scripts dir"
-[ -n "${ESCAPED_GUEST_LOGDIR}" ] || die "ESCAPED_GUEST_LOGDIR must be set to the escaped guest log dir"
-
-sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/GUEST_LOGDIR/${ESCAPED_GUEST_LOGDIR}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${TROVESTACK_SCRIPTS}/files/trove-guest.systemd.conf > ${TMP_HOOKS_PATH}/trove-guest.service
-

integration/scripts/files/elements/ubuntu-xenial-guest/install.d/20-etc

@@ -1,8 +0,0 @@
-#!/bin/sh
-
-# CONTEXT: GUEST during CONSTRUCTION as ROOT
-# PURPOSE: take "staged" trove-guest.conf file and put it in the init directory on guest image
-
-dd if=/tmp/in_target.d/trove-guest.service of=/etc/systemd/system/trove-guest.service
-
-systemctl enable trove-guest.service

integration/scripts/files/elements/ubuntu-xenial-guest/install.d/21-use-ubuntu-certificates

@@ -1,12 +0,0 @@
-#!/bin/sh
-
-# CONTEXT: GUEST during CONSTRUCTION as ROOT
-# PURPOSE: configure trove-guest service to use system store of trusted certificates
-
-GUEST_UNIT_DROPINS="/etc/systemd/system/trove-guest.service.d"
-
-mkdir -v -p ${GUEST_UNIT_DROPINS}
-cat <<EOF > ${GUEST_UNIT_DROPINS}/30-use-system-certificates.conf
-[Service]
-Environment=REQUESTS_CA_BUNDLE=/etc/ssl/certs
-EOF

integration/scripts/files/elements/ubuntu-xenial-guest/post-install.d/91-hwe-kernel

@@ -1,29 +0,0 @@
-#!/bin/bash
-
-set -e
-set -o xtrace
-
-# The HWE stack must be installed for nested virtualization on ppc64el.  This
-# environment variable is set automatically by trovestack, but it may also be
-# set by the user when manually invoking disk-image-create.
-
-case "$DIB_USE_HWE_KERNEL" in
-    true|True|TRUE|yes|Yes|YES)
-        DIB_USE_HWE_KERNEL=true
-        ;;
-    *)
-        DIB_USE_HWE_KERNEL=false
-        ;;
-esac
-
-if [ "$DIB_USE_HWE_KERNEL" == "true" ]; then
-    export DEBIAN_FRONTEND=noninteractive
-
-    PKG_ARCH=$(dpkg --print-architecture)
-
-    case "$PKG_ARCH" in
-        amd64|arm64|ppc64el|s390x)
-            apt-get --allow-unauthenticated install -y linux-generic-hwe-16.04
-            ;;
-    esac
-fi

integration/scripts/files/elements/ubuntu-xenial-guest/pre-install.d/01-trim-pkgs

@@ -1,90 +0,0 @@
-#!/bin/bash
-
-# CONTEXT: GUEST during CONSTRUCTION as ROOT
-# PURPOSE: Install basic services and applications
-
-set -e
-set -o xtrace
-
-export DEBIAN_FRONTEND=noninteractive
-apt-get -y purge acpid\
-                 apport\
-                 apport-symptoms\
-                 apt-transport-https\
-                 aptitude\
-                 at\
-                 bash-completion\
-                 bc\
-                 bind9-host\
-                 bsdmainutils\
-                 busybox-static\
-                 byobu\
-                 command-not-found\
-                 command-not-found-data\
-                 curl\
-                 dbus\
-                 dmidecode\
-                 dosfstools\
-                 ed\
-                 fonts-ubuntu-font-family-console\
-                 friendly-recovery\
-                 ftp\
-                 fuse\
-                 geoip-database\
-                 groff-base\
-                 hdparm\
-                 info\
-                 install-info\
-                 iptables\
-                 iputils-tracepath\
-                 irqbalance\
-                 language-selector-common\
-                 libaccountsservice0\
-                 libevent-2.0-5\
-                 libgeoip1\
-                 libnfnetlink0\
-                 libpcap0.8\
-                 libpci3\
-                 libpipeline1\
-                 libpolkit-gobject-1-0\
-                 libsasl2-modules\
-                 libusb-1.0-0\
-                 lshw\
-                 lsof\
-                 ltrace\
-                 man-db\
-                 mlocate\
-                 mtr-tiny\
-                 nano\
-                 ntfs-3g\
-                 parted\
-                 patch\
-                 plymouth-theme-ubuntu-text\
-                 popularity-contest\
-                 powermgmt-base\
-                 ppp\
-                 screen\
-                 shared-mime-info\
-                 strace\
-                 tcpdump\
-                 telnet\
-                 time\
-                 tmux\
-                 ubuntu-standard\
-                 ufw\
-                 update-manager-core\
-                 update-notifier-common\
-                 usbutils\
-                 uuid-runtime\
-
-# The following packages cannot be removed as they cause cloud-init to be
-# uninstalled in Ubuntu 14.04
-#                 gir1.2-glib-2.0
-#                 libdbus-glib-1-2
-#                 libgirepository-1.0-1
-#                 python-chardet
-#                 python-serial
-#                 xz-utils
-
-apt-get -y autoremove
-

integration/scripts/files/elements/ubuntu-xenial-mariadb/element-deps

@@ -1 +0,0 @@
-ubuntu-mariadb

integration/scripts/files/elements/ubuntu-xenial-mariadb/install.d/30-mariadb

@@ -1,39 +0,0 @@
-#!/bin/bash
-
-# CONTEXT: GUEST during CONSTRUCTION as ROOT
-# PURPOSE: Install controller base required packages
-# Refer to https://mariadb.com/kb/en/library/installing-mariadb-deb-files
-
-set -e
-set -o xtrace
-
-export DEBIAN_FRONTEND=noninteractive
-
-# These GPG key IDs are used to fetch keys from a keyserver on Ubuntu & Debian
-apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8
-curl -sS https://downloads.mariadb.com/MariaDB/mariadb_repo_setup |
-  bash -s -- --mariadb-server-version="mariadb-10.4" --skip-key-import --skip-maxscale
-
-apt-get install -y -qq apt-transport-https ca-certificates gnupg2
-
-# NOTE(lxkong): Refer to https://www.percona.com/doc/percona-xtrabackup/2.4/installation/apt_repo.html
-wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
-dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
-
-# Disable password prompt
-debconf-set-selections <<< "mariadb-server mysql-server/root_password password ''"
-debconf-set-selections <<< "mariadb-server mysql-server/root_password_again password ''"
-
-apt-get update -qq
-apt-get install -y -qq --allow-unauthenticated mariadb-server mariadb-client galera-4 libmariadb3 mariadb-backup mariadb-common
-
-cat <<EOF >/etc/mysql/conf.d/no_perf_schema.cnf
-[mysqld]
-performance_schema = off
-EOF
-
-chown mysql:mysql /etc/mysql/my.cnf
-rm -f /etc/init.d/mysql
-
-systemctl daemon-reload
-systemctl enable mariadb

integration/scripts/files/elements/ubuntu-xenial-mysql/element-deps

@@ -1 +0,0 @@
-ubuntu-mysql

integration/scripts/files/elements/ubuntu-xenial-mysql/install.d/30-mysql

@@ -1,39 +0,0 @@
-#!/bin/bash
-
-# CONTEXT: GUEST during CONSTRUCTION as ROOT
-# PURPOSE: Install controller base required packages
-
-set -e
-set -o xtrace
-
-export DEBIAN_FRONTEND=noninteractive
-
-apt-get --allow-unauthenticated -y install mysql-client mysql-server gnupg2
-
-# NOTE(lxkong): Refer to https://www.percona.com/doc/percona-xtrabackup/2.4/installation/apt_repo.html
-wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
-dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
-apt-get update
-
-# Xenial provides mysql 5.7 which requires percona-xtrabackup-24
-PXB_VERSION_OVERRIDE=24
-apt-get --allow-unauthenticated -y install percona-xtrabackup-${PXB_VERSION_OVERRIDE}
-
-cat >/etc/mysql/conf.d/no_perf_schema.cnf <<_EOF_
-[mysqld]
-performance_schema = off
-show_compatibility_56 = on
-_EOF_
-
-mv /etc/mysql/my.cnf.fallback /etc/mysql/my.cnf
-chown mysql:mysql /etc/mysql/my.cnf
-cat >/etc/mysql/my.cnf <<_EOF_
-[mysql]
-!includedir /etc/mysql/conf.d/
-_EOF_
-
-if [ -e /etc/init/mysql.conf ]; then
-    rm -f /etc/init/mysql.conf
-fi
-
-systemctl enable mysql

integration/scripts/files/elements/ubuntu-xenial-postgresql/element-deps

@@ -1 +0,0 @@
-ubuntu-postgresql

integration/scripts/functions_qemu

@@ -3,44 +3,36 @@
 # Additional functions that would mostly just pertain to a Ubuntu + Qemu setup
 #
 
-function build_vm() {
+function build_guest_image() {
-    exclaim "Actually building the image, this can take up to 15 minutes"
+    exclaim "Actually building the image, params: $@"
-    rm -rf ~/.cache/image-create
 
-    local datastore_type=$1
+    local guest_os=$1
-    local guest_os=$2
+    local guest_release=$2
-    local guest_release=$3
+    local dev_mode=$3
-    local dev_mode=$4
+    local guest_username=$4
-    local guest_username=$5
+    local image_output=$5
-    local image_output=$6
 
     local elementes="base vm"
     local trove_elements_path=${PATH_TROVE}/integration/scripts/files/elements
-    local GUEST_IMAGETYPE=${GUEST_IMAGETYPE:-"qcow2"}
+    local GUEST_IMAGESIZE=${GUEST_IMAGESIZE:-3}
-    local GUEST_IMAGESIZE=${GUEST_IMAGESIZE:-4}
     local GUEST_CACHEDIR=${GUEST_CACHEDIR:-"$HOME/.cache/image-create"}
+    rm -rf ${GUEST_CACHEDIR}
     local working_dir=$(dirname ${image_output})
 
     export GUEST_USERNAME=${guest_username}
+    export HOST_SCP_USERNAME=${HOST_SCP_USERNAME:-$(whoami)}
+    export ESCAPED_PATH_TROVE=$(echo ${PATH_TROVE} | sed 's/\//\\\//g')
+    export DEV_MODE=${dev_mode,,}
 
     # In dev mode, the trove guest agent needs to download trove code from
     # trove-taskmanager host during service initialization.
-    if [[ "${dev_mode,,}" == "true" ]]; then
+    if [[ "${DEV_MODE}" == "true" ]]; then
-        export PATH_TROVE=${PATH_TROVE}
-        export ESCAPED_PATH_TROVE=$(echo ${PATH_TROVE} | sed 's/\//\\\//g')
-        export GUEST_LOGDIR=${GUEST_LOGDIR:-"/var/log/trove/"}
-        export ESCAPED_GUEST_LOGDIR=$(echo ${GUEST_LOGDIR} | sed 's/\//\\\//g')
-        export TROVESTACK_SCRIPTS=${TROVESTACK_SCRIPTS}
-        export HOST_SCP_USERNAME=${HOST_SCP_USERNAME:-$(whoami)}
-        export HOST_USERNAME=${HOST_SCP_USERNAME}
         export SSH_DIR=${SSH_DIR:-"$HOME/.ssh"}
-        export DEST=${DEST:-'/opt/stack'}
-        export TROVE_BRANCH=${TROVE_BRANCH:-'master'}
         manage_ssh_keys
     fi
 
     # For system-wide installs, DIB will automatically find the elements, so we only check local path
-    if [ "${DIB_LOCAL_ELEMENTS_PATH}" ]; then
+    if [[ "${DIB_LOCAL_ELEMENTS_PATH}" ]]; then
         export ELEMENTS_PATH=${trove_elements_path}:${DIB_LOCAL_ELEMENTS_PATH}
     else
         export ELEMENTS_PATH=${trove_elements_path}
@@ -50,36 +42,26 @@ function build_vm() {
     export DIB_CLOUD_INIT_DATASOURCES="ConfigDrive"
 
     # https://cloud-images.ubuntu.com/releases is more stable than the daily
-    # builds(https://cloud-images.ubuntu.com/xenial/current/),
+    # builds (https://cloud-images.ubuntu.com/xenial/current/),
-    # e.g. sometimes SHA256SUMS file is missing in the daily builds
+    # e.g. sometimes SHA256SUMS file is missing in the daily builds website.
-    declare -A releasemapping=( ["xenial"]="16.04" ["bionic"]="18.04")
+    # Ref: diskimage_builder/elements/ubuntu/root.d/10-cache-ubuntu-tarball
+    declare -A image_file_mapping=( ["xenial"]="ubuntu-16.04-server-cloudimg-amd64-root.tar.gz" ["bionic"]="ubuntu-18.04-server-cloudimg-amd64.squashfs" )
     export DIB_CLOUD_IMAGES="https://cloud-images.ubuntu.com/releases/${DIB_RELEASE}/release/"
-    export BASE_IMAGE_FILE="ubuntu-${releasemapping[${DIB_RELEASE}]}-server-cloudimg-amd64-root.tar.gz"
+    export BASE_IMAGE_FILE=${image_file_mapping[${DIB_RELEASE}]}
 
     TEMP=$(mktemp -d ${working_dir}/diskimage-create.XXXXXXX)
     pushd $TEMP > /dev/null
 
     elementes="$elementes ${guest_os}"
-
+    elementes="$elementes pip-and-virtualenv"
-    if [[ "${dev_mode,,}" == "false" ]]; then
+    elementes="$elementes pip-cache"
-        elementes="$elementes pip-and-virtualenv"
+    elementes="$elementes guest-agent"
-        elementes="$elementes pip-cache"
+    elementes="$elementes ${guest_os}-docker"
-        elementes="$elementes guest-agent"
-    else
-        # Install guest agent dependencies, user, etc.
-        elementes="$elementes ${guest_os}-guest"
-        # Install guest agent service
-        elementes="$elementes ${guest_os}-${guest_release}-guest"
-    fi
-
-    elementes="$elementes ${guest_os}-${datastore_type}"
-    elementes="$elementes ${guest_os}-${guest_release}-${datastore_type}"
 
     # Build the image
     disk-image-create -x \
       -a amd64 \
       -o ${image_output} \
-      -t ${GUEST_IMAGETYPE} \
       --image-size ${GUEST_IMAGESIZE} \
       --image-cache ${GUEST_CACHEDIR} \
       $elementes
@@ -91,25 +73,6 @@ function build_vm() {
     exclaim "Image ${image_output} was built successfully."
 }
 
-function build_guest_image() {
-    exclaim "Params for build_guest_image function: $@"
-
-    local datastore_type=${1:-"mysql"}
-    local guest_os=${2:-"ubuntu"}
-    local guest_release=${3:-"xenial"}
-    local dev_mode=${4:-"true"}
-    local guest_username=${5:-"ubuntu"}
-    local output=$6
-
-    VALID_SERVICES='mysql percona mariadb redis cassandra couchbase mongodb postgresql couchdb vertica db2 pxc'
-    if ! [[ " $VALID_SERVICES " =~ " $datastore_type " ]]; then
-        exclaim "You did not pass in a valid datastore type. Valid types are:" $VALID_SERVICES
-        exit 1
-    fi
-
-    build_vm ${datastore_type} ${guest_os} ${guest_release} ${dev_mode} ${guest_username} ${output}
-}
-
 function clean_instances() {
     LIST=`virsh -q list|awk '{print $1}'`
     for i in $LIST; do sudo virsh destroy $i; done
@@ -117,6 +80,8 @@ function clean_instances() {
 
 # In dev mode, guest agent needs to ssh into the controller to download code.
 function manage_ssh_keys() {
+    SSH_DIR=${SSH_DIR:-"$HOME/.ssh"}
+
     if [ -d ${SSH_DIR} ]; then
         echo "${SSH_DIR} already exists"
     else

integration/scripts/local.conf.d/ceilometer_cinder.conf.rc

@@ -1,3 +0,0 @@
-[[post-config|\$CINDER_CONF]]
-[DEFAULT]
-notification_driver = messagingv2

integration/scripts/local.conf.d/ceilometer_nova.conf.rc

@@ -1,3 +0,0 @@
-[[post-config|\$NOVA_CONF]]
-[DEFAULT]
-instance_usage_audit = True

integration/scripts/local.conf.d/ceilometer_services.conf.rc

@@ -1,3 +0,0 @@
-[[post-config|\$CEILOMETER_CONF]]
-[notification]
-store_events = True

integration/scripts/local.conf.d/sample.rc

@@ -1,42 +0,0 @@
-#
-# Files in this directory are automatically added to the devstack
-# local.conf file, between a specific set of tags.
-#
-# Filenames must end with '.rc' to be recognized; sample.rc is
-# ignored.
-#
-# A '\' is required in front of any devstack variables since all
-# .rc files are parsed first (using eval).
-#
-# Meta section headings must be included in each file, such as:
-# [[local|localrc]]
-# as the order of inserting the files is not guaranteed.
-#
-# All files are inherently included by default - to exclude a file,
-# add a variable 'FILENAME_IN_UPPERCASE_MINUS_RC=false' in trovestack.rc
-# For Example: USING_VAGRANT=false (for the using_vagrant.rc file).
-#
-# Symbolic links are followed, so additional files can be loaded
-# by placing them in an external directory and linking it in
-# local.conf.d (this should allow complete flexibility in setting
-# up testing options).
-# For Example:
-#   cd /path/to/trove/integration/scripts/local.conf.d
-#   ln -s $HOME/local.conf.d local.conf.d
-#   cp /path/to/my_conf.rc $HOME/local.conf.d
-
-
-[[local|localrc]]
-# Put regular devstack variables under this meta section heading.
-# This section is written out to a file and sourced by devstack,
-# so it can contain logic as well.
-
-# The following section types should only contain ini file style
-# section headings and name=value pairs
-[[post-config|\$TROVE_CONF]]
-
-[[post-config|\$TROVE_TASKMANAGER_CONF]]
-
-[[post-config|\$TROVE_CONDUCTOR_CONF]]
-
-[[post-config|\$TROVE_API_PASTE_INI]]

integration/scripts/local.conf.d/trove_services.conf.rc

@@ -1,24 +0,0 @@
-[[post-config|\$TROVE_CONF]]
-[profiler]
-enabled = $ENABLE_PROFILER
-trace_sqlalchemy = $PROFILER_TRACE_SQL
-
-[[post-config|\$TROVE_TASKMANAGER_CONF]]
-[profiler]
-enabled = $ENABLE_PROFILER
-trace_sqlalchemy = $PROFILER_TRACE_SQL
-
-[[post-config|\$TROVE_CONDUCTOR_CONF]]
-[profiler]
-enabled = $ENABLE_PROFILER
-trace_sqlalchemy = $PROFILER_TRACE_SQL
-
-[[post-config|\$TROVE_GUESTAGENT_CONF]]
-[profiler]
-enabled = $ENABLE_PROFILER
-trace_sqlalchemy = $PROFILER_TRACE_SQL
-
-[[post-config|\$TROVE_API_PASTE_INI]]
-[filter:osprofiler]
-enabled = $ENABLE_PROFILER
-hmac_keys = $PROFILER_HMAC_KEYS

integration/scripts/local.conf.d/use_kvm.rc

@@ -1,4 +0,0 @@
-[[local|localrc]]
-
-# force kvm as the libvirt type.
-LIBVIRT_TYPE=kvm

integration/scripts/local.conf.d/use_uuid_token.rc

@@ -1,3 +0,0 @@
-[[local|localrc]]
-
-KEYSTONE_TOKEN_FORMAT=UUID

integration/scripts/local.conf.d/using_vagrant.rc

@@ -1,9 +0,0 @@
-[[local|localrc]]
-
-# This is similar to code found at
-# https://github.com/bcwaldon/vagrant_devstack/blob/master/Vagrantfile
-# and seems to make instances ping'able in VirtualBox.
-FLAT_INTERFACE=eth1
-PUBLIC_INTERFACE=eth1
-FLOATING_RANGE=`ip_chunk eth0 1`.`ip_chunk eth0 2`.`ip_chunk eth0 3`.128/28
-HOST_IP=`ip_chunk eth0 1`.`ip_chunk eth0 2`.`ip_chunk eth0 3`.`ip_chunk eth0 4`

integration/scripts/local.conf.rc

@@ -1,37 +0,0 @@
-$TROVE_PRESENT_TAG
-# Set some arguments for devstack.
-#
-# Note: This file contains autogenerated parts.
-#       All lines are removed from between the tag/end of tag
-#       markers (lines with '$MARKER_TOKEN' at beginning and end) and
-#       are replaced by trovestack.
-#       Edits to these sections will not persist.
-#
-#       See the '$USER_OPTS_TAG' section
-#       for ways to insert user args into this file.
-#
-
-#
-# This section is for things that belong in localrc
-# It comes from $DEFAULT_LOCALRC
-#
-[[local|localrc]]
-
-$LOCALRC_OPTS_TAG
-$LOCALRC_OPTS_TAG_END
-
-#
-# User options here were inserted from the file USER_LOCAL_CONF
-# (defaults to $USERHOME/.$LOCAL_CONF)
-#
-
-$USER_OPTS_TAG
-$USER_OPTS_TAG_END
-
-#
-# Additional options here were inserted by trovestack
-# automatically from files in $LOCAL_CONF_D
-#
-
-$ADD_OPTS_TAG
-$ADD_OPTS_TAG_END

integration/scripts/trovestack

@@ -124,7 +124,7 @@ if is_fedora; then
 else
     PKG_INSTALL_OPTS="DEBIAN_FRONTEND=noninteractive"
     PKG_MGR=apt-get
-    PKG_GET_ARGS="-y --allow-unauthenticated --force-yes"
+    PKG_GET_ARGS="-y --allow-unauthenticated --force-yes -qq"
 fi
 PKG_INSTALL_ARG="install"
 PKG_UPDATE_ARG="update"
@@ -522,57 +522,15 @@ function set_bin_path() {
 
 function cmd_set_datastore() {
     local IMAGEID=$1
-    local DATASTORE_TYPE=$2
-
-    # rd_manage datastore_update <datastore_name> <default_version>
-    rd_manage datastore_update "$DATASTORE_TYPE" ""
-    PACKAGES=${PACKAGES:-""}
-
-    if [ "$DATASTORE_TYPE" == "mysql" ]; then
-        VERSION="5.7"
-    elif [ "$DATASTORE_TYPE" == "percona" ]; then
-        PACKAGES=${PACKAGES:-"percona-server-server-5.6"}
-        VERSION="5.6"
-    elif [ "$DATASTORE_TYPE" == "pxc" ]; then
-        PACKAGES=${PACKAGES:-"percona-xtradb-cluster-server-5.6"}
-        VERSION="5.6"
-    elif [ "$DATASTORE_TYPE" == "mariadb" ]; then
-        VERSION="10.4"
-    elif [ "$DATASTORE_TYPE" == "mongodb" ]; then
-        PACKAGES=${PACKAGES:-"mongodb-org"}
-        VERSION="3.2"
-    elif [ "$DATASTORE_TYPE" == "redis" ]; then
-        PACKAGES=${PACKAGES:-""}
-        VERSION="3.2.6"
-    elif [ "$DATASTORE_TYPE" == "cassandra" ]; then
-        PACKAGES=${PACKAGES:-"cassandra"}
-        VERSION="2.1.0"
-    elif [ "$DATASTORE_TYPE" == "couchbase" ]; then
-        PACKAGES=${PACKAGES:-"couchbase-server"}
-        VERSION="2.2.0"
-    elif [ "$DATASTORE_TYPE" == "postgresql" ]; then
-        VERSION="9.6"
-    elif [ "$DATASTORE_TYPE" == "couchdb" ]; then
-        PACKAGES=${PACKAGES:-"couchdb"}
-        VERSION="1.6.1"
-    elif [ "$DATASTORE_TYPE" == "vertica" ]; then
-        PACKAGES=${PACKAGES:-"vertica"}
-        VERSION="9.0.1"
-    elif [ "$DATASTORE_TYPE" == "db2" ]; then
-        PACKAGES=${PACKAGES:-""}
-        VERSION="11.1"
-    else
-        echo "Unrecognized datastore type. ($DATASTORE_TYPE)"
-        exit 1
-    fi
 
+    rd_manage datastore_update "$datastore" ""
     # trove-manage datastore_version_update <datastore_name> <version_name> <datastore_manager> <image_id> <packages> <active>
-    rd_manage datastore_version_update "$DATASTORE_TYPE" "$VERSION" "$DATASTORE_TYPE" $IMAGEID "$PACKAGES" 1
+    rd_manage datastore_version_update "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" "${DATASTORE_TYPE}" $IMAGEID "" 1
-    rd_manage datastore_update "$DATASTORE_TYPE" "$VERSION"
+    rd_manage datastore_update "${DATASTORE_TYPE}" "${DATASTORE_VERSION}"
 
-    if [ -f "$PATH_TROVE"/trove/templates/$DATASTORE_TYPE/validation-rules.json ]; then
+    if [[ -f "$PATH_TROVE"/trove/templates/${DATASTORE_TYPE}/validation-rules.json ]]; then
         # add the configuration parameters to the database for the kick-start datastore
-        rd_manage db_load_datastore_config_parameters "$DATASTORE_TYPE" "$VERSION" "$PATH_TROVE"/trove/templates/$DATASTORE_TYPE/validation-rules.json
+        rd_manage db_load_datastore_config_parameters "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" "$PATH_TROVE"/trove/templates/${DATASTORE_TYPE}/validation-rules.json
     fi
 }
 
@@ -627,8 +585,8 @@ function install_test_packages() {
     DATASTORE_TYPE=$1
 
     sudo -H $HTTP_PROXY pip install openstack.nose_plugin proboscis pexpect
-    if [ "$DATASTORE_TYPE" = "couchbase" ]; then
+    if [[ "$DATASTORE_TYPE" = "couchbase" ]]; then
-        if [ "$DISTRO" == "ubuntu" ]; then
+        if [[ "$DISTRO" == "ubuntu" ]]; then
 	        # Install Couchbase SDK for scenario tests.
 	        sudo -H $HTTP_PROXY curl http://packages.couchbase.com/ubuntu/couchbase.key | sudo apt-key add -
 	        echo "deb http://packages.couchbase.com/ubuntu trusty trusty/main" | sudo tee /etc/apt/sources.list.d/couchbase-csdk.list
@@ -649,12 +607,12 @@ function mod_confs() {
     TROVE_REPORT_DIR=${TROVE_REPORT_DIR:=$TROVESTACK_SCRIPTS/../report/}
 
     EXTRA_CONF=$TROVESTACK_SCRIPTS/conf/test.extra.conf
-    if [ -e $EXTRA_CONF ]; then
+    if [[ -e $EXTRA_CONF ]]; then
         cat $EXTRA_CONF >> $TEST_CONF
     fi
     # Append datastore specific configuration file
     DATASTORE_CONF=$TROVESTACK_SCRIPTS/conf/$DATASTORE_TYPE.conf
-    if [ ! -f $DATASTORE_CONF ]; then
+    if [[ ! -f $DATASTORE_CONF ]]; then
         exclaim "Datastore configuration file ${DATASTORE_CONF} not found"
         exit 1
     fi
@@ -695,14 +653,14 @@ function mod_confs() {
         sed -i "/%shared_network_subnet%/d" $TEST_CONF
     fi
 
-    if [ "$DATASTORE_TYPE" = "vertica" ]; then
+    if [[ "$DATASTORE_TYPE" = "vertica" ]]; then
         # Vertica needs more time than mysql for its boot/start/stop operations.
         setup_cluster_configs cluster_member_count 3
-    elif [ "$DATASTORE_TYPE" = "pxc" ]; then
+    elif [[ "$DATASTORE_TYPE" = "pxc" ]]; then
         setup_cluster_configs min_cluster_member_count 2
-    elif [ "$DATASTORE_TYPE" = "cassandra" ]; then
+    elif [[ "$DATASTORE_TYPE" = "cassandra" ]]; then
         setup_cluster_configs cluster_member_count 2
-    elif [ "$DATASTORE_TYPE" = "mongodb" ]; then
+    elif [[ "$DATASTORE_TYPE" = "mongodb" ]]; then
         setup_cluster_configs cluster_member_count 2
         # Decrease the number of required config servers per cluster to save resources.
         iniset $TROVE_CONF $DATASTORE_TYPE num_config_servers_per_cluster 1
@@ -747,7 +705,7 @@ function cmd_test_init() {
     local DATASTORE_TYPE=$1
     local DATASTORE_VERSION=$2
 
-    if [ -z "${DATASTORE_TYPE}" ]; then
+    if [[ -z "${DATASTORE_TYPE}" ]]; then
         exclaim "${COLOR_RED}Datastore argument was not specified.${COLOR_NONE}"
         exit 1
     fi
@@ -768,73 +726,60 @@ function cmd_test_init() {
     pip3 install -U git+https://opendev.org/openstack/python-troveclient@master#egg=python-troveclient
 }
 
+# Build trove guest image
 function cmd_build_image() {
     exclaim "Params for cmd_build_image function: $@"
 
-    local IMAGE_DATASTORE_TYPE=${1:-'mysql'}
+    local image_guest_os=${1:-'ubuntu'}
-    local IMAGE_GUEST_OS=${2:-'ubuntu'}
+    local image_guest_release=${2:-'bionic'}
-    local IMAGE_GUEST_RELEASE=${3:-'xenial'}
+    local dev_mode=${3:-'true'}
-    local DEV_MODE=${4:-'true'}
+    local guest_username=${4:-'ubuntu'}
-    local guest_username=${5:-'ubuntu'}
+    local output=$5
-    local output=$6
 
     if [[ -z "$output" ]]; then
-        image_name="trove-datastore-${IMAGE_GUEST_OS}-${IMAGE_GUEST_RELEASE}-${IMAGE_DATASTORE_TYPE}"
+        image_name="trove-guest-${image_guest_os}-${image_guest_release}"
+        if [[ ${dev_mode} == "true" ]]; then
+            image_name="${image_name}-dev"
+        fi
         image_folder=$HOME/images
-        output="${image_folder}/${image_name}"
+        output="${image_folder}/${image_name}.qcow2"
     fi
+
     # Always rebuild the image.
-    sudo rm -f $output
+    sudo rm -rf ${output}
-    sudo mkdir -p $(dirname $output); sudo chmod 777 -R $(dirname $output)
+    sudo mkdir -p $(dirname ${output}); sudo chmod 777 -R $(dirname ${output})
 
     echo "Ensuring we have all packages needed to build image."
     sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS update
-    sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS install qemu git kpartx debootstrap
+    sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS install qemu git kpartx debootstrap squashfs-tools
     sudo -H $HTTP_PROXY pip install diskimage-builder
 
-    exclaim "Use diskimage-builder to actually build the Trove Guest Agent Image."
+    build_guest_image ${image_guest_os} ${image_guest_release} ${dev_mode} ${guest_username} ${output}
-    build_guest_image $IMAGE_DATASTORE_TYPE $IMAGE_GUEST_OS $IMAGE_GUEST_RELEASE $DEV_MODE ${guest_username} $output
 }
 
 # Build guest image and upload to Glance, register the datastore and configuration parameters.
-# We could skip the image build and upload by:
-#   1. MYSQL_IMAGE_ID is passed, or
-#   2. There is an image in Glance contains the datastore name
 function cmd_build_and_upload_image() {
-    local datastore_type=$1
+    local guest_os=${1:-"ubuntu"}
-    local guest_os=${2:-"ubuntu"}
+    local guest_release=${2:-"bionic"}
-    local guest_release=${3:-"xenial"}
+    local dev_mode=${3:-"true"}
-    local dev_mode=${4:-"true"}
+    local guest_username=${4:-"ubuntu"}
-    local guest_username=${5:-"ubuntu"}
+    local output_dir=${5:-"$HOME/images"}
-    local output_dir=${6:-"$HOME/images"}
 
-    if [ -z "${datastore_type}" ]; then
+    name=trove-guest-${guest_os}-${guest_release}
-        exclaim "${COLOR_RED}Datastore argument was not specified.${COLOR_NONE}"
+    glance_imageid=$(openstack ${CLOUD_ADMIN_ARG} image list --name $name -f value -c ID)
-        exit 1
+    if [[ -z ${glance_imageid} ]]; then
-    fi
+        mkdir -p ${output_dir}
+        output=${output_dir}/${name}
+        cmd_build_image ${guest_os} ${guest_release} ${dev_mode} ${guest_username} ${output}
 
-    image_var="${datastore_type^^}_IMAGE_ID"
+        glance_imageid=$(openstack ${CLOUD_ADMIN_ARG} image create ${name} --public --disk-format qcow2 --container-format bare --file ${output} --property hw_rng_model='virtio' --tag trove -c id -f value)
-    glance_imageid=`eval echo '$'"$image_var"`
+        [[ -z "$glance_imageid" ]] && echo "Glance upload failed!" && exit 1
-
-    if [[ -z $glance_imageid ]]; then
-      # Find the first image id with the name contains datastore_type.
-      glance_imageid=$(openstack $CLOUD_ADMIN_ARG image list | grep "$datastore_type" | awk 'NR==1 {print}' | awk '{print $2}')
-
-      if [[ -z $glance_imageid ]]; then
-          mkdir -p ${output_dir}
-          name=trove-datastore-${guest_os}-${guest_release}-${datastore_type}
-          output=${output_dir}/$name.qcow2
-          cmd_build_image ${datastore_type} ${guest_os} ${guest_release} ${dev_mode} ${guest_username} $output
-
-          glance_imageid=$(openstack ${CLOUD_ADMIN_ARG} image create $name --public --disk-format qcow2 --container-format bare --file $output --property hw_rng_model='virtio' -c id -f value)
-          [[ -z "$glance_imageid" ]] && echo "Glance upload failed!" && exit 1
-      fi
     fi
 
     exclaim "Using Glance image ID: $glance_imageid"
 
     exclaim "Updating Datastores"
-    cmd_set_datastore "${glance_imageid}" "${datastore_type}"
+    cmd_set_datastore "${glance_imageid}"
 }
 
 
@@ -991,11 +936,11 @@ function cmd_stop() {
 
 function cmd_int_tests() {
     exclaim "Running Trove Integration Tests..."
-    if [ ! $USAGE_ENDPOINT ]; then
+    if [[ ! $USAGE_ENDPOINT ]]; then
         export USAGE_ENDPOINT=trove.tests.util.usage.FakeVerifier
     fi
     cd $TROVESTACK_SCRIPTS
-    if [ $# -lt 1 ]; then
+    if [[ $# -lt 1 ]]; then
         args="--group=mysql"
     else
         args="$@"
@@ -1203,7 +1148,7 @@ function cmd_kick_start() {
     local DATASTORE_TYPE=$1
     local DATASTORE_VERSION=$2
 
-    if [ -z "${DATASTORE_TYPE}" ]; then
+    if [[ -z "${DATASTORE_TYPE}" ]]; then
         exclaim "${COLOR_RED}Datastore argument was not specified.${COLOR_NONE}"
         exit 1
     fi
@@ -1220,10 +1165,13 @@ function cmd_kick_start() {
 function cmd_gate_tests() {
     local DATASTORE_TYPE=${1:-'mysql'}
     local TEST_GROUP=${2:-${DATASTORE_TYPE}}
-    local DATASTORE_VERSION=${3:-'5.7'}
+    local DATASTORE_VERSION=${3:-'5.7.29'}
     local HOST_SCP_USERNAME=${4:-$(whoami)}
     local GUEST_USERNAME=${5:-'ubuntu'}
 
+    export DATASTORE_TYPE=${DATASTORE_TYPE}
+    export DATASTORE_VERSION=${DATASTORE_VERSION}
+
     exclaim "Running cmd_gate_tests ..."
 
     export REPORT_DIRECTORY=${REPORT_DIRECTORY:=$HOME/gate-tests-report/}
@@ -1238,7 +1186,7 @@ function cmd_gate_tests() {
     cd $TROVESTACK_SCRIPTS
 
     # Build and upload guest image, register datastore version.
-    cmd_build_and_upload_image ${DATASTORE_TYPE}
+    cmd_build_and_upload_image
 
     cmd_kick_start "${DATASTORE_TYPE}" "${DATASTORE_VERSION}"
 

lower-constraints.txt

@@ -25,6 +25,7 @@ deprecation==2.0
 diskimage-builder==1.1.2
 doc8==0.6.0
 docutils==0.14
+docker==4.2.0
 dogpile.cache==0.6.5
 dulwich==0.19.0
 enum34===1.0.4

playbooks/image-build/run.yaml

@@ -7,12 +7,11 @@
     - name: Build Trove guest image
       shell: >-
         ./trovestack build-image \
-          {{ datastore_type }} \
           {{ guest_os }} \
           {{ guest_os_release }} \
           {{ dev_mode }} \
           {{ guest_username }} \
-          {{ ansible_user_dir }}/images/trove-{{ branch }}-{{ datastore_type }}-{{ guest_os }}-{{ guest_os_release }}{{ image_suffix }}
+          {{ ansible_user_dir }}/images/trove-{{ branch }}-guest-{{ guest_os }}-{{ guest_os_release }}{{ image_suffix }}.qcow2
       args:
         chdir: "{{ ansible_user_dir }}/src/opendev.org/openstack/trove/integration/scripts"
       tags:

requirements.txt

@@ -48,3 +48,4 @@ xmltodict>=0.10.1 # MIT
 cryptography>=2.1.4  # BSD/Apache-2.0
 oslo.policy>=1.30.0 # Apache-2.0
 diskimage-builder!=1.6.0,!=1.7.0,!=1.7.1,>=1.1.2 # Apache-2.0
+docker>=4.2.0 # Apache-2.0

roles/trove-devstack/defaults/main.yml

@@ -1,5 +1,5 @@
 devstack_base_dir: /opt/stack
 trove_test_datastore: 'mysql'
 trove_test_group: 'mysql'
-trove_test_datastore_version: '5.7'
+trove_test_datastore_version: '5.7.29'
 trove_resize_time_out: ''

tools/trove-pylint.config

@@ -1497,6 +1497,12 @@
       "Instance of 'FreshInstance' has no 'get_replication_master_snapshot' member",
       "Manager._create_replication_slave"
     ],
+    [
+      "trove/taskmanager/manager.py",
+      "E1136",
+      "Value 'snapshot' is unsubscriptable",
+      "Manager._create_replication_slave"
+    ],
     [
       "trove/taskmanager/manager.py",
       "E1101",

tox.ini

@@ -53,9 +53,7 @@ ignore-path = .venv,.tox,.git,dist,doc,*egg-info,tools,etc,build,*.po,*.pot,inte
 
 [flake8]
 show-source = True
-# H301 is ignored on purpose.
+ignore = E125,E129,E402,E731,F601,F821,H301,H306,H404,H405,H501,W503,W504,W605
-# The rest of the ignores are TODOs.
-ignore = E402,E731,F601,F821,H301,H404,H405,H501,W503,W504,W605
 enable-extensions = H203,H106
 builtins = _
 # add *.yaml for playbooks/trove-devstack-base.yaml, as it will be matched by
@@ -68,7 +66,7 @@ import_exceptions = trove.common.i18n
 
 [flake8:local-plugins]
 extension =
-    T103= checks:check_raised_localized_exceptions
+    # T103= checks:check_raised_localized_exceptions
     T104 = checks:check_no_basestring
     T105 = checks:no_translate_logs
     N335 = checks:assert_raises_regexp

trove/backup/models.py

@@ -268,7 +268,7 @@ class Backup(object):
             try:
                 cls.delete(context, child.id)
             except exception.NotFound:
-                LOG.exception("Backup %s cannot be found.", backup_id)
+                LOG.warning("Backup %s cannot be found.", backup_id)
 
         def _delete_resources():
             backup = cls.get_by_id(context, backup_id)

trove/cmd/guest.py

@@ -12,7 +12,6 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
-
 import sys
 
 from oslo_config import cfg as openstack_cfg
@@ -23,6 +22,7 @@ from trove.common import cfg
 from trove.common import debug_utils
 from trove.common.i18n import _
 from trove.guestagent import api as guest_api
+from trove.guestagent.common import operating_system
 
 CONF = cfg.CONF
 # The guest_id opt definition must match the one in common/cfg.py
@@ -31,9 +31,18 @@ CONF.register_opts([openstack_cfg.StrOpt('guest_id', default=None,
                     openstack_cfg.StrOpt('instance_rpc_encr_key',
                                          help=('Key (OpenSSL aes_cbc) for '
                                                'instance RPC encryption.'))])
+LOG = logging.getLogger(__name__)
 
 
 def main():
+    log_levels = [
+        'docker=WARN',
+    ]
+    default_log_levels = logging.get_default_log_levels()
+    default_log_levels.extend(log_levels)
+    logging.set_defaults(default_log_levels=default_log_levels)
+    logging.register_options(CONF)
+
     cfg.parse_args(sys.argv)
     logging.setup(CONF, None)
     debug_utils.setup()
@@ -50,6 +59,11 @@ def main():
                "was not injected into the guest or not read by guestagent"))
         raise RuntimeError(msg)
 
+    # Create user and group for running docker container.
+    LOG.info('Creating user and group for database service')
+    uid = cfg.get_configuration_property('database_service_uid')
+    operating_system.create_user('database', uid)
+
     # rpc module must be loaded after decision about thread monkeypatching
     # because if thread module is not monkeypatched we can't use eventlet
     # executor from oslo_messaging library.

trove/common/cfg.py

@@ -175,10 +175,10 @@ common_opts = [
                help="Maximum time (in seconds) to wait for Guest Agent "
                     "'quick' requests (such as retrieving a list of "
                     "users or databases)."),
-    cfg.IntOpt('agent_call_high_timeout', default=60 * 5,
+    cfg.IntOpt('agent_call_high_timeout', default=60 * 3,
                help="Maximum time (in seconds) to wait for Guest Agent 'slow' "
                     "requests (such as restarting the database)."),
-    cfg.IntOpt('agent_replication_snapshot_timeout', default=36000,
+    cfg.IntOpt('agent_replication_snapshot_timeout', default=60 * 30,
                help='Maximum time (in seconds) to wait for taking a Guest '
                     'Agent replication snapshot.'),
     cfg.IntOpt('command_process_timeout', default=30,
@@ -186,8 +186,9 @@ common_opts = [
                     'commands to complete.'),
     # The guest_id opt definition must match the one in cmd/guest.py
     cfg.StrOpt('guest_id', default=None, help="ID of the Guest Instance."),
-    cfg.IntOpt('state_change_wait_time', default=60 * 10,
+    cfg.IntOpt('state_change_wait_time', default=180,
-               help='Maximum time (in seconds) to wait for a state change.'),
+               help='Maximum time (in seconds) to wait for database state '
+                    'change.'),
     cfg.IntOpt('state_change_poll_time', default=3,
                help='Interval between state change poll requests (seconds).'),
     cfg.IntOpt('agent_heartbeat_time', default=10,
@@ -293,9 +294,11 @@ common_opts = [
                help='The region this service is located.'),
     cfg.StrOpt('backup_runner',
                default='trove.guestagent.backup.backup_types.InnoBackupEx',
-               help='Runner to use for backups.'),
+               help='Runner to use for backups.',
+               deprecated_for_removal=True),
     cfg.DictOpt('backup_runner_options', default={},
-                help='Additional options to be passed to the backup runner.'),
+                help='Additional options to be passed to the backup runner.',
+                deprecated_for_removal=True),
     cfg.BoolOpt('verify_swift_checksum_on_restore', default=True,
                 help='Enable verification of Swift checksum before starting '
                 'restore. Makes sure the checksum of original backup matches '
@@ -304,11 +307,12 @@ common_opts = [
                 help='Require the replica volume size to be greater than '
                 'or equal to the size of the master volume '
                 'during replica creation.'),
-    cfg.StrOpt('storage_strategy', default='SwiftStorage',
+    cfg.StrOpt('storage_strategy', default='swift',
                help="Default strategy to store backups."),
     cfg.StrOpt('storage_namespace',
                default='trove.common.strategies.storage.swift',
-               help='Namespace to load the default storage strategy from.'),
+               help='Namespace to load the default storage strategy from.',
+               deprecated_for_removal=True),
     cfg.StrOpt('backup_swift_container', default='database_backups',
                help='Swift container to put backups in.'),
     cfg.BoolOpt('backup_use_gzip_compression', default=True,
@@ -429,15 +433,12 @@ common_opts = [
     cfg.IntOpt('usage_timeout', default=60 * 30,
                help='Maximum time (in seconds) to wait for a Guest to become '
                     'active.'),
-    cfg.IntOpt('restore_usage_timeout', default=36000,
+    cfg.IntOpt('restore_usage_timeout', default=60 * 60,
                help='Maximum time (in seconds) to wait for a Guest instance '
                     'restored from a backup to become active.'),
     cfg.IntOpt('cluster_usage_timeout', default=36000,
                help='Maximum time (in seconds) to wait for a cluster to '
                     'become active.'),
-    cfg.IntOpt('timeout_wait_for_service', default=120,
-               help='Maximum time (in seconds) to wait for a service to '
-                    'become alive.'),
     cfg.StrOpt('module_aes_cbc_key', default='module_aes_cbc_key',
                help='OpenSSL aes_cbc key for module encryption.'),
     cfg.ListOpt('module_types', default=['ping', 'new_relic_license'],
@@ -466,6 +467,10 @@ common_opts = [
                help='Key (OpenSSL aes_cbc) to encrypt instance keys in DB.'),
     cfg.StrOpt('instance_rpc_encr_key',
                help='Key (OpenSSL aes_cbc) for instance RPC encryption.'),
+    cfg.StrOpt('database_service_uid', default='1001',
+               help='The UID(GID) of database service user.'),
+    cfg.StrOpt('backup_docker_image', default='openstacktrove/db-backup:1.0.0',
+               help='The docker image used for backup and restore.'),
 ]
 
 
@@ -544,7 +549,7 @@ mysql_opts = [
                 help='List of UDP ports and/or port ranges to open '
                      'in the security group (only applicable '
                      'if trove_security_groups_support is True).'),
-    cfg.StrOpt('backup_strategy', default='InnoBackupEx',
+    cfg.StrOpt('backup_strategy', default='innobackupex',
                help='Default strategy to perform backups.',
                deprecated_name='backup_strategy',
                deprecated_group='DEFAULT'),
@@ -564,28 +569,10 @@ mysql_opts = [
     cfg.IntOpt('usage_timeout', default=400,
                help='Maximum time (in seconds) to wait for a Guest to become '
                     'active.'),
-    cfg.StrOpt('backup_namespace',
-               default='trove.guestagent.strategies.backup.mysql_impl',
-               help='Namespace to load backup strategies from.',
-               deprecated_name='backup_namespace',
-               deprecated_group='DEFAULT'),
-    cfg.StrOpt('restore_namespace',
-               default='trove.guestagent.strategies.restore.mysql_impl',
-               help='Namespace to load restore strategies from.',
-               deprecated_name='restore_namespace',
-               deprecated_group='DEFAULT'),
     cfg.BoolOpt('volume_support', default=True,
                 help='Whether to provision a Cinder volume for datadir.'),
     cfg.StrOpt('device_path', default='/dev/vdb',
                help='Device path for volume if volume support is enabled.'),
-    cfg.DictOpt('backup_incremental_strategy',
-                default={'InnoBackupEx': 'InnoBackupExIncremental'},
-                help='Incremental Backup Runner based on the default '
-                'strategy. For strategies that do not implement an '
-                'incremental backup, the runner will use the default full '
-                'backup.',
-                deprecated_name='backup_incremental_strategy',
-                deprecated_group='DEFAULT'),
     cfg.StrOpt('root_controller',
                default='trove.extensions.common.service.DefaultRootController',
                help='Root controller implementation for mysql.'),
@@ -611,6 +598,10 @@ mysql_opts = [
                help='Character length of generated passwords.',
                deprecated_name='default_password_length',
                deprecated_group='DEFAULT'),
+    cfg.StrOpt(
+        'docker_image', default='mysql',
+        help='Database docker image.'
+    )
 ]
 
 # Percona
@@ -653,28 +644,10 @@ percona_opts = [
     cfg.IntOpt('usage_timeout', default=450,
                help='Maximum time (in seconds) to wait for a Guest to become '
                     'active.'),
-    cfg.StrOpt('backup_namespace',
-               default='trove.guestagent.strategies.backup.mysql_impl',
-               help='Namespace to load backup strategies from.',
-               deprecated_name='backup_namespace',
-               deprecated_group='DEFAULT'),
-    cfg.StrOpt('restore_namespace',
-               default='trove.guestagent.strategies.restore.mysql_impl',
-               help='Namespace to load restore strategies from.',
-               deprecated_name='restore_namespace',
-               deprecated_group='DEFAULT'),
     cfg.BoolOpt('volume_support', default=True,
                 help='Whether to provision a Cinder volume for datadir.'),
     cfg.StrOpt('device_path', default='/dev/vdb',
                help='Device path for volume if volume support is enabled.'),
-    cfg.DictOpt('backup_incremental_strategy',
-                default={'InnoBackupEx': 'InnoBackupExIncremental'},
-                help='Incremental Backup Runner based on the default '
-                'strategy. For strategies that do not implement an '
-                'incremental backup, the runner will use the default full '
-                'backup.',
-                deprecated_name='backup_incremental_strategy',
-                deprecated_group='DEFAULT'),
     cfg.StrOpt('root_controller',
                default='trove.extensions.common.service.DefaultRootController',
                help='Root controller implementation for percona.'),
@@ -739,22 +712,10 @@ pxc_opts = [
     cfg.IntOpt('usage_timeout', default=450,
                help='Maximum time (in seconds) to wait for a Guest to become '
                     'active.'),
-    cfg.StrOpt('backup_namespace',
-               default='trove.guestagent.strategies.backup.mysql_impl',
-               help='Namespace to load backup strategies from.'),
-    cfg.StrOpt('restore_namespace',
-               default='trove.guestagent.strategies.restore.mysql_impl',
-               help='Namespace to load restore strategies from.'),
     cfg.BoolOpt('volume_support', default=True,
                 help='Whether to provision a Cinder volume for datadir.'),
     cfg.StrOpt('device_path', default='/dev/vdb',
                help='Device path for volume if volume support is enabled.'),
-    cfg.DictOpt('backup_incremental_strategy',
-                default={'InnoBackupEx': 'InnoBackupExIncremental'},
-                help='Incremental Backup Runner based on the default '
-                'strategy. For strategies that do not implement an '
-                'incremental backup, the runner will use the default full '
-                'backup.'),
     cfg.ListOpt('ignore_users', default=['os_admin', 'root', 'clusterrepuser'],
                 help='Users to exclude when listing users.'),
     cfg.ListOpt('ignore_dbs',
@@ -818,12 +779,6 @@ redis_opts = [
                help='Default strategy to perform backups.',
                deprecated_name='backup_strategy',
                deprecated_group='DEFAULT'),
-    cfg.DictOpt('backup_incremental_strategy', default={},
-                help='Incremental Backup Runner based on the default '
-                'strategy. For strategies that do not implement an '
-                'incremental, the runner will use the default full backup.',
-                deprecated_name='backup_incremental_strategy',
-                deprecated_group='DEFAULT'),
     cfg.StrOpt('replication_strategy', default='RedisSyncReplication',
                help='Default strategy for replication.'),
     cfg.StrOpt('replication_namespace',
@@ -837,18 +792,6 @@ redis_opts = [
                 help='Whether to provision a Cinder volume for datadir.'),
     cfg.StrOpt('device_path', default='/dev/vdb',
                help='Device path for volume if volume support is enabled.'),
-    cfg.StrOpt('backup_namespace',
-               default="trove.guestagent.strategies.backup.experimental."
-                       "redis_impl",
-               help='Namespace to load backup strategies from.',
-               deprecated_name='backup_namespace',
-               deprecated_group='DEFAULT'),
-    cfg.StrOpt('restore_namespace',
-               default="trove.guestagent.strategies.restore.experimental."
-                       "redis_impl",
-               help='Namespace to load restore strategies from.',
-               deprecated_name='restore_namespace',
-               deprecated_group='DEFAULT'),
     cfg.BoolOpt('cluster_support', default=True,
                 help='Enable clusters to be created and managed.'),
     cfg.StrOpt('api_strategy',
@@ -893,12 +836,6 @@ cassandra_opts = [
                 help='List of UDP ports and/or port ranges to open '
                      'in the security group (only applicable '
                      'if trove_security_groups_support is True).'),
-    cfg.DictOpt('backup_incremental_strategy', default={},
-                help='Incremental strategy based on the default backup '
-                'strategy. For strategies that do not implement incremental '
-                'backups, the runner performs full backup instead.',
-                deprecated_name='backup_incremental_strategy',
-                deprecated_group='DEFAULT'),
     cfg.StrOpt('backup_strategy', default="NodetoolSnapshot",
                help='Default strategy to perform backups.',
                deprecated_name='backup_strategy',
@@ -912,18 +849,6 @@ cassandra_opts = [
                 help='Whether to provision a Cinder volume for datadir.'),
     cfg.StrOpt('device_path', default='/dev/vdb',
                help='Device path for volume if volume support is enabled.'),
-    cfg.StrOpt('backup_namespace',
-               default="trove.guestagent.strategies.backup.experimental."
-               "cassandra_impl",
-               help='Namespace to load backup strategies from.',
-               deprecated_name='backup_namespace',
-               deprecated_group='DEFAULT'),
-    cfg.StrOpt('restore_namespace',
-               default="trove.guestagent.strategies.restore.experimental."
-               "cassandra_impl",
-               help='Namespace to load restore strategies from.',
-               deprecated_name='restore_namespace',
-               deprecated_group='DEFAULT'),
     cfg.StrOpt('root_controller',
                default='trove.extensions.common.service.DefaultRootController',
                help='Root controller implementation for Cassandra.'),
@@ -1002,12 +927,6 @@ couchbase_opts = [
                help='Default strategy to perform backups.',
                deprecated_name='backup_strategy',
                deprecated_group='DEFAULT'),
-    cfg.DictOpt('backup_incremental_strategy', default={},
-                help='Incremental Backup Runner based on the default '
-                'strategy. For strategies that do not implement an '
-                'incremental, the runner will use the default full backup.',
-                deprecated_name='backup_incremental_strategy',
-                deprecated_group='DEFAULT'),
     cfg.StrOpt('replication_strategy', default=None,
                help='Default strategy for replication.'),
     cfg.StrOpt('mount_point', default='/var/lib/couchbase',
@@ -1018,18 +937,6 @@ couchbase_opts = [
                 'service during instance-create. The generated password for '
                 'the root user is immediately returned in the response of '
                 "instance-create as the 'password' field."),
-    cfg.StrOpt('backup_namespace',
-               default='trove.guestagent.strategies.backup.experimental.'
-               'couchbase_impl',
-               help='Namespace to load backup strategies from.',
-               deprecated_name='backup_namespace',
-               deprecated_group='DEFAULT'),
-    cfg.StrOpt('restore_namespace',
-               default='trove.guestagent.strategies.restore.experimental.'
-               'couchbase_impl',
-               help='Namespace to load restore strategies from.',
-               deprecated_name='restore_namespace',
-               deprecated_group='DEFAULT'),
     cfg.BoolOpt('volume_support', default=True,
                 help='Whether to provision a Cinder volume for datadir.'),
     cfg.StrOpt('device_path', default='/dev/vdb',
@@ -1066,12 +973,6 @@ mongodb_opts = [
                help='Default strategy to perform backups.',
                deprecated_name='backup_strategy',
                deprecated_group='DEFAULT'),
-    cfg.DictOpt('backup_incremental_strategy', default={},
-                help='Incremental Backup Runner based on the default '
-                'strategy. For strategies that do not implement an '
-                'incremental, the runner will use the default full backup.',
-                deprecated_name='backup_incremental_strategy',
-                deprecated_group='DEFAULT'),
     cfg.StrOpt('replication_strategy', default=None,
                help='Default strategy for replication.'),
     cfg.StrOpt('mount_point', default='/var/lib/mongodb',
@@ -1109,18 +1010,6 @@ mongodb_opts = [
                'mongodb.guestagent.MongoDbGuestAgentStrategy',
                help='Class that implements datastore-specific Guest Agent API '
                     'logic.'),
-    cfg.StrOpt('backup_namespace',
-               default='trove.guestagent.strategies.backup.experimental.'
-                       'mongo_impl',
-               help='Namespace to load backup strategies from.',
-               deprecated_name='backup_namespace',
-               deprecated_group='DEFAULT'),
-    cfg.StrOpt('restore_namespace',
-               default='trove.guestagent.strategies.restore.experimental.'
-                       'mongo_impl',
-               help='Namespace to load restore strategies from.',
-               deprecated_name='restore_namespace',
-               deprecated_group='DEFAULT'),
     cfg.PortOpt('mongodb_port', default=27017,
                 help='Port for mongod and mongos instances.'),
     cfg.PortOpt('configsvr_port', default=27019,
@@ -1164,11 +1053,6 @@ postgresql_opts = [
                 help='The TCP port the server listens on.'),
     cfg.StrOpt('backup_strategy', default='PgBaseBackup',
                help='Default strategy to perform backups.'),
-    cfg.DictOpt('backup_incremental_strategy',
-                default={'PgBaseBackup': 'PgBaseBackupIncremental'},
-                help='Incremental Backup Runner based on the default '
-                'strategy. For strategies that do not implement an '
-                'incremental, the runner will use the default full backup.'),
     cfg.StrOpt('replication_strategy',
                default='PostgresqlReplicationStreaming',
                help='Default strategy for replication.'),
@@ -1188,14 +1072,6 @@ postgresql_opts = [
                 'service during instance-create. The generated password for '
                 'the root user is immediately returned in the response of '
                 "instance-create as the 'password' field."),
-    cfg.StrOpt('backup_namespace',
-               default='trove.guestagent.strategies.backup.experimental.'
-               'postgresql_impl',
-               help='Namespace to load backup strategies from.'),
-    cfg.StrOpt('restore_namespace',
-               default='trove.guestagent.strategies.restore.experimental.'
-               'postgresql_impl',
-               help='Namespace to load restore strategies from.'),
     cfg.BoolOpt('volume_support', default=True,
                 help='Whether to provision a Cinder volume for datadir.'),
     cfg.StrOpt('device_path', default='/dev/vdb'),
@@ -1248,16 +1124,6 @@ couchdb_opts = [
                help='Default strategy to perform backups.'),
     cfg.StrOpt('replication_strategy', default=None,
                help='Default strategy for replication.'),
-    cfg.StrOpt('backup_namespace', default='trove.guestagent.strategies'
-               '.backup.experimental.couchdb_impl',
-               help='Namespace to load backup strategies from.'),
-    cfg.StrOpt('restore_namespace', default='trove.guestagent.strategies'
-               '.restore.experimental.couchdb_impl',
-               help='Namespace to load restore strategies from.'),
-    cfg.DictOpt('backup_incremental_strategy', default={},
-                help='Incremental Backup Runner based on the default '
-                'strategy. For strategies that do not implement an '
-                'incremental, the runner will use the default full backup.'),
     cfg.BoolOpt('root_on_create', default=False,
                 help='Enable the automatic creation of the root user for the '
                 'service during instance-create. The generated password for '
@@ -1303,10 +1169,6 @@ vertica_opts = [
                      'if trove_security_groups_support is True).'),
     cfg.StrOpt('backup_strategy', default=None,
                help='Default strategy to perform backups.'),
-    cfg.DictOpt('backup_incremental_strategy', default={},
-                help='Incremental Backup Runner based on the default '
-                'strategy. For strategies that do not implement an '
-                'incremental, the runner will use the default full backup.'),
     cfg.StrOpt('replication_strategy', default=None,
                help='Default strategy for replication.'),
     cfg.StrOpt('mount_point', default='/var/lib/vertica',
@@ -1317,9 +1179,11 @@ vertica_opts = [
     cfg.StrOpt('device_path', default='/dev/vdb',
                help='Device path for volume if volume support is enabled.'),
     cfg.StrOpt('backup_namespace', default=None,
-               help='Namespace to load backup strategies from.'),
+               help='Namespace to load backup strategies from.',
+               deprecated_for_removal=True),
     cfg.StrOpt('restore_namespace', default=None,
-               help='Namespace to load restore strategies from.'),
+               help='Namespace to load restore strategies from.',
+               deprecated_for_removal=True),
     cfg.IntOpt('readahead_size', default=2048,
                help='Size(MB) to be set as readahead_size for data volume'),
     cfg.BoolOpt('cluster_support', default=True,
@@ -1387,22 +1251,6 @@ db2_opts = [
                 'service during instance-create. The generated password for '
                 'the root user is immediately returned in the response of '
                 "instance-create as the 'password' field."),
-    cfg.StrOpt('backup_namespace',
-               default='trove.guestagent.strategies.backup.experimental.'
-                       'db2_impl',
-               help='Namespace to load backup strategies from.',
-               deprecated_name='backup_namespace',
-               deprecated_group='DEFAULT'),
-    cfg.StrOpt('restore_namespace',
-               default='trove.guestagent.strategies.restore.experimental.'
-                       'db2_impl',
-               help='Namespace to load restore strategies from.',
-               deprecated_name='restore_namespace',
-               deprecated_group='DEFAULT'),
-    cfg.DictOpt('backup_incremental_strategy', default={},
-                help='Incremental Backup Runner based on the default '
-                'strategy. For strategies that do not implement an '
-                'incremental, the runner will use the default full backup.'),
     cfg.ListOpt('ignore_users', default=['PUBLIC', 'DB2INST1']),
     cfg.StrOpt('root_controller',
                default='trove.extensions.common.service.DefaultRootController',
@@ -1432,21 +1280,14 @@ mariadb_opts = [
                 help='List of UDP ports and/or port ranges to open '
                      'in the security group (only applicable '
                      'if trove_security_groups_support is True).'),
-    cfg.StrOpt('backup_namespace',
+    cfg.StrOpt('backup_strategy', default='mariabackup',
-               default='trove.guestagent.strategies.backup.experimental'
-                       '.mariadb_impl',
-               help='Namespace to load backup strategies from.',
-               deprecated_name='backup_namespace',
-               deprecated_group='DEFAULT'),
-    cfg.StrOpt('backup_strategy', default='MariaBackup',
                help='Default strategy to perform backups.',
                deprecated_name='backup_strategy',
                deprecated_group='DEFAULT'),
     cfg.StrOpt('replication_strategy', default='MariaDBGTIDReplication',
                help='Default strategy for replication.'),
     cfg.StrOpt('replication_namespace',
-               default='trove.guestagent.strategies.replication.experimental'
+               default='trove.guestagent.strategies.replication.mariadb_gtid',
-               '.mariadb_gtid',
                help='Namespace to load replication strategies from.'),
     cfg.StrOpt('mount_point', default='/var/lib/mysql',
                help="Filesystem path for mounting "
@@ -1459,25 +1300,10 @@ mariadb_opts = [
     cfg.IntOpt('usage_timeout', default=400,
                help='Maximum time (in seconds) to wait for a Guest to become '
                     'active.'),
-    cfg.StrOpt('restore_namespace',
-               default='trove.guestagent.strategies.restore.experimental'
-                       '.mariadb_impl',
-               help='Namespace to load restore strategies from.',
-               deprecated_name='restore_namespace',
-               deprecated_group='DEFAULT'),
     cfg.BoolOpt('volume_support', default=True,
                 help='Whether to provision a Cinder volume for datadir.'),
     cfg.StrOpt('device_path', default='/dev/vdb',
                help='Device path for volume if volume support is enabled.'),
-    cfg.DictOpt('backup_incremental_strategy',
-                default={'MariaBackup':
-                         'MariaBackupIncremental'},
-                help='Incremental Backup Runner based on the default '
-                'strategy. For strategies that do not implement an '
-                'incremental backup, the runner will use the default full '
-                'backup.',
-                deprecated_name='backup_incremental_strategy',
-                deprecated_group='DEFAULT'),
     cfg.StrOpt('root_controller',
                default='trove.extensions.common.service.DefaultRootController',
                help='Root controller implementation for mysql.'),
@@ -1521,6 +1347,10 @@ mariadb_opts = [
                help='Character length of generated passwords.',
                deprecated_name='default_password_length',
                deprecated_group='DEFAULT'),
+    cfg.StrOpt(
+        'docker_image', default='mariadb',
+        help='Database docker image.'
+    )
 ]
 
 # RPC version groups

trove/common/exception.py

@@ -307,7 +307,7 @@ class VolumeAttachmentsNotFound(NotFound):
 
 class VolumeCreationFailure(TroveError):
 
-    message = _("Failed to create a volume in Nova.")
+    message = _("Failed to create volume.")
 
 
 class VolumeSizeNotSpecified(BadRequest):
@@ -341,6 +341,16 @@ class ReplicationSlaveAttachError(TroveError):
     message = _("Exception encountered attaching slave to new replica source.")
 
 
+class SlaveOperationNotSupported(TroveError):
+    message = _("The '%(operation)s' operation is not supported for slaves in "
+                "replication.")
+
+
+class UnableToDetermineLastMasterGTID(TroveError):
+    message = _("Unable to determine last GTID executed on master "
+                "(from file %(binlog_file)s).")
+
+
 class TaskManagerError(TroveError):
 
     message = _("An error occurred communicating with the task manager: "
@@ -688,9 +698,3 @@ class LogAccessForbidden(Forbidden):
 class LogsNotAvailable(Forbidden):
 
     message = _("Log actions are not supported.")
-
-
-class SlaveOperationNotSupported(TroveError):
-
-    message = _("The '%(operation)s' operation is not supported for slaves in "
-                "replication.")

trove/common/schemas/atom-link.rng

@@ -1,141 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  -*- rnc -*-
-  RELAX NG Compact Syntax Grammar for the
-  Atom Format Specification Version 11
--->
-<grammar xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:s="http://www.ascc.net/xml/schematron" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
-  <start>
-    <choice>
-      <ref name="atomLink"/>
-    </choice>
-  </start>
-  <!-- Common attributes -->
-  <define name="atomCommonAttributes">
-    <optional>
-      <attribute name="xml:base">
-        <ref name="atomUri"/>
-      </attribute>
-    </optional>
-    <optional>
-      <attribute name="xml:lang">
-        <ref name="atomLanguageTag"/>
-      </attribute>
-    </optional>
-    <zeroOrMore>
-      <ref name="undefinedAttribute"/>
-    </zeroOrMore>
-  </define>
-  <!-- atom:link -->
-  <define name="atomLink">
-    <element name="atom:link">
-      <ref name="atomCommonAttributes"/>
-      <attribute name="href">
-        <ref name="atomUri"/>
-      </attribute>
-      <optional>
-        <attribute name="rel">
-          <choice>
-            <ref name="atomNCName"/>
-            <ref name="atomUri"/>
-          </choice>
-        </attribute>
-      </optional>
-      <optional>
-        <attribute name="type">
-          <ref name="atomMediaType"/>
-        </attribute>
-      </optional>
-      <optional>
-        <attribute name="hreflang">
-          <ref name="atomLanguageTag"/>
-        </attribute>
-      </optional>
-      <optional>
-        <attribute name="title"/>
-      </optional>
-      <optional>
-        <attribute name="length"/>
-      </optional>
-      <ref name="undefinedContent"/>
-    </element>
-  </define>
-  <!-- Low-level simple types -->
-  <define name="atomNCName">
-    <data type="string">
-      <param name="minLength">1</param>
-      <param name="pattern">[^:]*</param>
-    </data>
-  </define>
-  <!-- Whatever a media type is, it contains at least one slash -->
-  <define name="atomMediaType">
-    <data type="string">
-      <param name="pattern">.+/.+</param>
-    </data>
-  </define>
-  <!-- As defined in RFC 3066 -->
-  <define name="atomLanguageTag">
-    <data type="string">
-      <param name="pattern">[A-Za-z]{1,8}(-[A-Za-z0-9]{1,8})*</param>
-    </data>
-  </define>
-  <!--
-    Unconstrained; it's not entirely clear how IRI fit into
-    xsd:anyURI so let's not try to constrain it here
-  -->
-  <define name="atomUri">
-    <text/>
-  </define>
-  <!-- Other Extensibility -->
-  <define name="undefinedAttribute">
-    <attribute>
-      <anyName>
-        <except>
-          <name>xml:base</name>
-          <name>xml:lang</name>
-          <nsName ns=""/>
-        </except>
-      </anyName>
-    </attribute>
-  </define>
-  <define name="undefinedContent">
-    <zeroOrMore>
-      <choice>
-        <text/>
-        <ref name="anyForeignElement"/>
-      </choice>
-    </zeroOrMore>
-  </define>
-  <define name="anyElement">
-    <element>
-      <anyName/>
-      <zeroOrMore>
-        <choice>
-          <attribute>
-            <anyName/>
-          </attribute>
-          <text/>
-          <ref name="anyElement"/>
-        </choice>
-      </zeroOrMore>
-    </element>
-  </define>
-  <define name="anyForeignElement">
-    <element>
-      <anyName>
-        <except>
-          <nsName ns="http://www.w3.org/2005/Atom"/>
-        </except>
-      </anyName>
-      <zeroOrMore>
-        <choice>
-          <attribute>
-            <anyName/>
-          </attribute>
-          <text/>
-          <ref name="anyElement"/>
-        </choice>
-      </zeroOrMore>
-    </element>
-  </define>
-</grammar>

trove/common/schemas/atom.rng

@@ -1,597 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  -*- rnc -*-
-  RELAX NG Compact Syntax Grammar for the
-  Atom Format Specification Version 11
--->
-<grammar xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:s="http://www.ascc.net/xml/schematron" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
-  <start>
-    <choice>
-      <ref name="atomFeed"/>
-      <ref name="atomEntry"/>
-    </choice>
-  </start>
-  <!-- Common attributes -->
-  <define name="atomCommonAttributes">
-    <optional>
-      <attribute name="xml:base">
-        <ref name="atomUri"/>
-      </attribute>
-    </optional>
-    <optional>
-      <attribute name="xml:lang">
-        <ref name="atomLanguageTag"/>
-      </attribute>
-    </optional>
-    <zeroOrMore>
-      <ref name="undefinedAttribute"/>
-    </zeroOrMore>
-  </define>
-  <!-- Text Constructs -->
-  <define name="atomPlainTextConstruct">
-    <ref name="atomCommonAttributes"/>
-    <optional>
-      <attribute name="type">
-        <choice>
-          <value>text</value>
-          <value>html</value>
-        </choice>
-      </attribute>
-    </optional>
-    <text/>
-  </define>
-  <define name="atomXHTMLTextConstruct">
-    <ref name="atomCommonAttributes"/>
-    <attribute name="type">
-      <value>xhtml</value>
-    </attribute>
-    <ref name="xhtmlDiv"/>
-  </define>
-  <define name="atomTextConstruct">
-    <choice>
-      <ref name="atomPlainTextConstruct"/>
-      <ref name="atomXHTMLTextConstruct"/>
-    </choice>
-  </define>
-  <!-- Person Construct -->
-  <define name="atomPersonConstruct">
-    <ref name="atomCommonAttributes"/>
-    <interleave>
-      <element name="atom:name">
-        <text/>
-      </element>
-      <optional>
-        <element name="atom:uri">
-          <ref name="atomUri"/>
-        </element>
-      </optional>
-      <optional>
-        <element name="atom:email">
-          <ref name="atomEmailAddress"/>
-        </element>
-      </optional>
-      <zeroOrMore>
-        <ref name="extensionElement"/>
-      </zeroOrMore>
-    </interleave>
-  </define>
-  <!-- Date Construct -->
-  <define name="atomDateConstruct">
-    <ref name="atomCommonAttributes"/>
-    <data type="dateTime"/>
-  </define>
-  <!-- atom:feed -->
-  <define name="atomFeed">
-    <element name="atom:feed">
-      <s:rule context="atom:feed">
-        <s:assert test="atom:author or not(atom:entry[not(atom:author)])">An atom:feed must have an atom:author unless all of its atom:entry children have an atom:author.</s:assert>
-      </s:rule>
-      <ref name="atomCommonAttributes"/>
-      <interleave>
-        <zeroOrMore>
-          <ref name="atomAuthor"/>
-        </zeroOrMore>
-        <zeroOrMore>
-          <ref name="atomCategory"/>
-        </zeroOrMore>
-        <zeroOrMore>
-          <ref name="atomContributor"/>
-        </zeroOrMore>
-        <optional>
-          <ref name="atomGenerator"/>
-        </optional>
-        <optional>
-          <ref name="atomIcon"/>
-        </optional>
-        <ref name="atomId"/>
-        <zeroOrMore>
-          <ref name="atomLink"/>
-        </zeroOrMore>
-        <optional>
-          <ref name="atomLogo"/>
-        </optional>
-        <optional>
-          <ref name="atomRights"/>
-        </optional>
-        <optional>
-          <ref name="atomSubtitle"/>
-        </optional>
-        <ref name="atomTitle"/>
-        <ref name="atomUpdated"/>
-        <zeroOrMore>
-          <ref name="extensionElement"/>
-        </zeroOrMore>
-      </interleave>
-      <zeroOrMore>
-        <ref name="atomEntry"/>
-      </zeroOrMore>
-    </element>
-  </define>
-  <!-- atom:entry -->
-  <define name="atomEntry">
-    <element name="atom:entry">
-      <s:rule context="atom:entry">
-        <s:assert test="atom:link[@rel='alternate'] or atom:link[not(@rel)] or atom:content">An atom:entry must have at least one atom:link element with a rel attribute of 'alternate' or an atom:content.</s:assert>
-      </s:rule>
-      <s:rule context="atom:entry">
-        <s:assert test="atom:author or ../atom:author or atom:source/atom:author">An atom:entry must have an atom:author if its feed does not.</s:assert>
-      </s:rule>
-      <ref name="atomCommonAttributes"/>
-      <interleave>
-        <zeroOrMore>
-          <ref name="atomAuthor"/>
-        </zeroOrMore>
-        <zeroOrMore>
-          <ref name="atomCategory"/>
-        </zeroOrMore>
-        <optional>
-          <ref name="atomContent"/>
-        </optional>
-        <zeroOrMore>
-          <ref name="atomContributor"/>
-        </zeroOrMore>
-        <ref name="atomId"/>
-        <zeroOrMore>
-          <ref name="atomLink"/>
-        </zeroOrMore>
-        <optional>
-          <ref name="atomPublished"/>
-        </optional>
-        <optional>
-          <ref name="atomRights"/>
-        </optional>
-        <optional>
-          <ref name="atomSource"/>
-        </optional>
-        <optional>
-          <ref name="atomSummary"/>
-        </optional>
-        <ref name="atomTitle"/>
-        <ref name="atomUpdated"/>
-        <zeroOrMore>
-          <ref name="extensionElement"/>
-        </zeroOrMore>
-      </interleave>
-    </element>
-  </define>
-  <!-- atom:content -->
-  <define name="atomInlineTextContent">
-    <element name="atom:content">
-      <ref name="atomCommonAttributes"/>
-      <optional>
-        <attribute name="type">
-          <choice>
-            <value>text</value>
-            <value>html</value>
-          </choice>
-        </attribute>
-      </optional>
-      <zeroOrMore>
-        <text/>
-      </zeroOrMore>
-    </element>
-  </define>
-  <define name="atomInlineXHTMLContent">
-    <element name="atom:content">
-      <ref name="atomCommonAttributes"/>
-      <attribute name="type">
-        <value>xhtml</value>
-      </attribute>
-      <ref name="xhtmlDiv"/>
-    </element>
-  </define>
-  <define name="atomInlineOtherContent">
-    <element name="atom:content">
-      <ref name="atomCommonAttributes"/>
-      <optional>
-        <attribute name="type">
-          <ref name="atomMediaType"/>
-        </attribute>
-      </optional>
-      <zeroOrMore>
-        <choice>
-          <text/>
-          <ref name="anyElement"/>
-        </choice>
-      </zeroOrMore>
-    </element>
-  </define>
-  <define name="atomOutOfLineContent">
-    <element name="atom:content">
-      <ref name="atomCommonAttributes"/>
-      <optional>
-        <attribute name="type">
-          <ref name="atomMediaType"/>
-        </attribute>
-      </optional>
-      <attribute name="src">
-        <ref name="atomUri"/>
-      </attribute>
-      <empty/>
-    </element>
-  </define>
-  <define name="atomContent">
-    <choice>
-      <ref name="atomInlineTextContent"/>
-      <ref name="atomInlineXHTMLContent"/>
-      <ref name="atomInlineOtherContent"/>
-      <ref name="atomOutOfLineContent"/>
-    </choice>
-  </define>
-  <!-- atom:author -->
-  <define name="atomAuthor">
-    <element name="atom:author">
-      <ref name="atomPersonConstruct"/>
-    </element>
-  </define>
-  <!-- atom:category -->
-  <define name="atomCategory">
-    <element name="atom:category">
-      <ref name="atomCommonAttributes"/>
-      <attribute name="term"/>
-      <optional>
-        <attribute name="scheme">
-          <ref name="atomUri"/>
-        </attribute>
-      </optional>
-      <optional>
-        <attribute name="label"/>
-      </optional>
-      <ref name="undefinedContent"/>
-    </element>
-  </define>
-  <!-- atom:contributor -->
-  <define name="atomContributor">
-    <element name="atom:contributor">
-      <ref name="atomPersonConstruct"/>
-    </element>
-  </define>
-  <!-- atom:generator -->
-  <define name="atomGenerator">
-    <element name="atom:generator">
-      <ref name="atomCommonAttributes"/>
-      <optional>
-        <attribute name="uri">
-          <ref name="atomUri"/>
-        </attribute>
-      </optional>
-      <optional>
-        <attribute name="version"/>
-      </optional>
-      <text/>
-    </element>
-  </define>
-  <!-- atom:icon -->
-  <define name="atomIcon">
-    <element name="atom:icon">
-      <ref name="atomCommonAttributes"/>
-      <ref name="atomUri"/>
-    </element>
-  </define>
-  <!-- atom:id -->
-  <define name="atomId">
-    <element name="atom:id">
-      <ref name="atomCommonAttributes"/>
-      <ref name="atomUri"/>
-    </element>
-  </define>
-  <!-- atom:logo -->
-  <define name="atomLogo">
-    <element name="atom:logo">
-      <ref name="atomCommonAttributes"/>
-      <ref name="atomUri"/>
-    </element>
-  </define>
-  <!-- atom:link -->
-  <define name="atomLink">
-    <element name="atom:link">
-      <ref name="atomCommonAttributes"/>
-      <attribute name="href">
-        <ref name="atomUri"/>
-      </attribute>
-      <optional>
-        <attribute name="rel">
-          <choice>
-            <ref name="atomNCName"/>
-            <ref name="atomUri"/>
-          </choice>
-        </attribute>
-      </optional>
-      <optional>
-        <attribute name="type">
-          <ref name="atomMediaType"/>
-        </attribute>
-      </optional>
-      <optional>
-        <attribute name="hreflang">
-          <ref name="atomLanguageTag"/>
-        </attribute>
-      </optional>
-      <optional>
-        <attribute name="title"/>
-      </optional>
-      <optional>
-        <attribute name="length"/>
-      </optional>
-      <ref name="undefinedContent"/>
-    </element>
-  </define>
-  <!-- atom:published -->
-  <define name="atomPublished">
-    <element name="atom:published">
-      <ref name="atomDateConstruct"/>
-    </element>
-  </define>
-  <!-- atom:rights -->
-  <define name="atomRights">
-    <element name="atom:rights">
-      <ref name="atomTextConstruct"/>
-    </element>
-  </define>
-  <!-- atom:source -->
-  <define name="atomSource">
-    <element name="atom:source">
-      <ref name="atomCommonAttributes"/>
-      <interleave>
-        <zeroOrMore>
-          <ref name="atomAuthor"/>
-        </zeroOrMore>
-        <zeroOrMore>
-          <ref name="atomCategory"/>
-        </zeroOrMore>
-        <zeroOrMore>
-          <ref name="atomContributor"/>
-        </zeroOrMore>
-        <optional>
-          <ref name="atomGenerator"/>
-        </optional>
-        <optional>
-          <ref name="atomIcon"/>
-        </optional>
-        <optional>
-          <ref name="atomId"/>
-        </optional>
-        <zeroOrMore>
-          <ref name="atomLink"/>
-        </zeroOrMore>
-        <optional>
-          <ref name="atomLogo"/>
-        </optional>
-        <optional>
-          <ref name="atomRights"/>
-        </optional>
-        <optional>
-          <ref name="atomSubtitle"/>
-        </optional>
-        <optional>
-          <ref name="atomTitle"/>
-        </optional>
-        <optional>
-          <ref name="atomUpdated"/>
-        </optional>
-        <zeroOrMore>
-          <ref name="extensionElement"/>
-        </zeroOrMore>
-      </interleave>
-    </element>
-  </define>
-  <!-- atom:subtitle -->
-  <define name="atomSubtitle">
-    <element name="atom:subtitle">
-      <ref name="atomTextConstruct"/>
-    </element>
-  </define>
-  <!-- atom:summary -->
-  <define name="atomSummary">
-    <element name="atom:summary">
-      <ref name="atomTextConstruct"/>
-    </element>
-  </define>
-  <!-- atom:title -->
-  <define name="atomTitle">
-    <element name="atom:title">
-      <ref name="atomTextConstruct"/>
-    </element>
-  </define>
-  <!-- atom:updated -->
-  <define name="atomUpdated">
-    <element name="atom:updated">
-      <ref name="atomDateConstruct"/>
-    </element>
-  </define>
-  <!-- Low-level simple types -->
-  <define name="atomNCName">
-    <data type="string">
-      <param name="minLength">1</param>
-      <param name="pattern">[^:]*</param>
-    </data>
-  </define>
-  <!-- Whatever a media type is, it contains at least one slash -->
-  <define name="atomMediaType">
-    <data type="string">
-      <param name="pattern">.+/.+</param>
-    </data>
-  </define>
-  <!-- As defined in RFC 3066 -->
-  <define name="atomLanguageTag">
-    <data type="string">
-      <param name="pattern">[A-Za-z]{1,8}(-[A-Za-z0-9]{1,8})*</param>
-    </data>
-  </define>
-  <!--
-    Unconstrained; it's not entirely clear how IRI fit into
-    xsd:anyURI so let's not try to constrain it here
-  -->
-  <define name="atomUri">
-    <text/>
-  </define>
-  <!-- Whatever an email address is, it contains at least one @ -->
-  <define name="atomEmailAddress">
-    <data type="string">
-      <param name="pattern">.+@.+</param>
-    </data>
-  </define>
-  <!-- Simple Extension -->
-  <define name="simpleExtensionElement">
-    <element>
-      <anyName>
-        <except>
-          <nsName ns="http://www.w3.org/2005/Atom"/>
-        </except>
-      </anyName>
-      <text/>
-    </element>
-  </define>
-  <!-- Structured Extension -->
-  <define name="structuredExtensionElement">
-    <element>
-      <anyName>
-        <except>
-          <nsName ns="http://www.w3.org/2005/Atom"/>
-        </except>
-      </anyName>
-      <choice>
-        <group>
-          <oneOrMore>
-            <attribute>
-              <anyName/>
-            </attribute>
-          </oneOrMore>
-          <zeroOrMore>
-            <choice>
-              <text/>
-              <ref name="anyElement"/>
-            </choice>
-          </zeroOrMore>
-        </group>
-        <group>
-          <zeroOrMore>
-            <attribute>
-              <anyName/>
-            </attribute>
-          </zeroOrMore>
-          <group>
-            <optional>
-              <text/>
-            </optional>
-            <oneOrMore>
-              <ref name="anyElement"/>
-            </oneOrMore>
-            <zeroOrMore>
-              <choice>
-                <text/>
-                <ref name="anyElement"/>
-              </choice>
-            </zeroOrMore>
-          </group>
-        </group>
-      </choice>
-    </element>
-  </define>
-  <!-- Other Extensibility -->
-  <define name="extensionElement">
-    <choice>
-      <ref name="simpleExtensionElement"/>
-      <ref name="structuredExtensionElement"/>
-    </choice>
-  </define>
-  <define name="undefinedAttribute">
-    <attribute>
-      <anyName>
-        <except>
-          <name>xml:base</name>
-          <name>xml:lang</name>
-          <nsName ns=""/>
-        </except>
-      </anyName>
-    </attribute>
-  </define>
-  <define name="undefinedContent">
-    <zeroOrMore>
-      <choice>
-        <text/>
-        <ref name="anyForeignElement"/>
-      </choice>
-    </zeroOrMore>
-  </define>
-  <define name="anyElement">
-    <element>
-      <anyName/>
-      <zeroOrMore>
-        <choice>
-          <attribute>
-            <anyName/>
-          </attribute>
-          <text/>
-          <ref name="anyElement"/>
-        </choice>
-      </zeroOrMore>
-    </element>
-  </define>
-  <define name="anyForeignElement">
-    <element>
-      <anyName>
-        <except>
-          <nsName ns="http://www.w3.org/2005/Atom"/>
-        </except>
-      </anyName>
-      <zeroOrMore>
-        <choice>
-          <attribute>
-            <anyName/>
-          </attribute>
-          <text/>
-          <ref name="anyElement"/>
-        </choice>
-      </zeroOrMore>
-    </element>
-  </define>
-  <!-- XHTML -->
-  <define name="anyXHTML">
-    <element>
-      <nsName ns="http://www.w3.org/1999/xhtml"/>
-      <zeroOrMore>
-        <choice>
-          <attribute>
-            <anyName/>
-          </attribute>
-          <text/>
-          <ref name="anyXHTML"/>
-        </choice>
-      </zeroOrMore>
-    </element>
-  </define>
-  <define name="xhtmlDiv">
-    <element name="xhtml:div">
-      <zeroOrMore>
-        <choice>
-          <attribute>
-            <anyName/>
-          </attribute>
-          <text/>
-          <ref name="anyXHTML"/>
-        </choice>
-      </zeroOrMore>
-    </element>
-  </define>
-</grammar>

trove/common/schemas/v1.1/limits.rng

@@ -1,28 +0,0 @@
-<element name="limits" ns="http://docs.openstack.org/common/api/v1.0"
-  xmlns="http://relaxng.org/ns/structure/1.0">
-  <element name="rates">
-    <zeroOrMore>
-      <element name="rate">
-        <attribute name="uri"> <text/> </attribute>
-        <attribute name="regex"> <text/> </attribute>
-        <zeroOrMore>
-          <element name="limit">
-            <attribute name="value"> <text/> </attribute>
-            <attribute name="verb"> <text/> </attribute>
-            <attribute name="remaining"> <text/> </attribute>
-            <attribute name="unit"> <text/> </attribute>
-            <attribute name="next-available"> <text/> </attribute>
-          </element>
-        </zeroOrMore>
-      </element>
-    </zeroOrMore>
-  </element>
-  <element name="absolute">
-    <zeroOrMore>
-      <element name="limit">
-        <attribute name="name"> <text/> </attribute>
-        <attribute name="value"> <text/> </attribute>
-      </element>
-    </zeroOrMore>
-  </element>
-</element>

trove/common/utils.py

@@ -185,7 +185,7 @@ class MethodInspector(object):
 
 
 def build_polling_task(retriever, condition=lambda value: value,
-                       sleep_time=1, time_out=0):
+                       sleep_time=1, time_out=0, initial_delay=0):
     """Run a function in a loop with backoff on error.
 
     The condition function runs based on the retriever function result.
@@ -197,7 +197,8 @@ def build_polling_task(retriever, condition=lambda value: value,
             raise loopingcall.LoopingCallDone(retvalue=obj)
 
     call = loopingcall.BackOffLoopingCall(f=poll_and_check)
-    return call.start(initial_delay=0, starting_interval=sleep_time,
+    return call.start(initial_delay=initial_delay,
+                      starting_interval=sleep_time,
                       max_interval=30, timeout=time_out)
 
 
@@ -210,7 +211,7 @@ def wait_for_task(polling_task):
 
 
 def poll_until(retriever, condition=lambda value: value,
-               sleep_time=3, time_out=0):
+               sleep_time=3, time_out=0, initial_delay=0):
     """Retrieves object until it passes condition, then returns it.
 
     If time_out_limit is passed in, PollTimeOut will be raised once that
@@ -218,7 +219,8 @@ def poll_until(retriever, condition=lambda value: value,
 
     """
     task = build_polling_task(retriever, condition=condition,
-                              sleep_time=sleep_time, time_out=time_out)
+                              sleep_time=sleep_time, time_out=time_out,
+                              initial_delay=initial_delay)
     return wait_for_task(task)
 
 

trove/configuration/service.py

@@ -218,8 +218,6 @@ class ConfigurationsController(wsgi.Controller):
     def _refresh_on_all_instances(self, context, configuration_id):
         """Refresh a configuration group on all single instances.
         """
-        LOG.debug("Re-applying configuration group '%s' to all instances.",
-                  configuration_id)
         single_instances = instances_models.DBInstance.find_all(
             tenant_id=context.project_id,
             configuration_id=configuration_id,
@@ -228,8 +226,8 @@ class ConfigurationsController(wsgi.Controller):
 
         config = models.Configuration(context, configuration_id)
         for dbinstance in single_instances:
-            LOG.debug("Re-applying configuration to instance: %s",
+            LOG.info("Re-applying configuration %s to instance: %s",
-                      dbinstance.id)
+                     configuration_id, dbinstance.id)
             instance = instances_models.Instance.load(context, dbinstance.id)
             instance.update_configuration(config)
 

trove/guestagent/api.py

@@ -314,7 +314,7 @@ class API(object):
                 device_path='/dev/vdb', mount_point='/mnt/volume',
                 backup_info=None, config_contents=None, root_password=None,
                 overrides=None, cluster_config=None, snapshot=None,
-                modules=None):
+                modules=None, ds_version=None):
         """Make an asynchronous call to prepare the guest
            as a database container optionally includes a backup id for restores
         """
@@ -335,7 +335,8 @@ class API(object):
             device_path=device_path, mount_point=mount_point,
             backup_info=backup_info, config_contents=config_contents,
             root_password=root_password, overrides=overrides,
-            cluster_config=cluster_config, snapshot=snapshot, modules=modules)
+            cluster_config=cluster_config, snapshot=snapshot, modules=modules,
+            ds_version=ds_version)
 
     def _create_guest_queue(self):
         """Call to construct, start and immediately stop rpc server in order
@@ -409,15 +410,14 @@ class API(object):
         self._call("reset_configuration", self.agent_high_timeout,
                    version=version, configuration=configuration)
 
-    def stop_db(self, do_not_start_on_reboot=False):
+    def stop_db(self):
         """Stop the database server."""
         LOG.debug("Sending the call to stop the database process "
                   "on the Guest.")
         version = self.API_BASE_VERSION
 
-        self._call("stop_db", self.agent_high_timeout,
+        self._call("stop_db", self.agent_low_timeout,
-                   version=version,
+                   version=version)
-                   do_not_start_on_reboot=do_not_start_on_reboot)
 
     def upgrade(self, instance_version, location, metadata=None):
         """Make an asynchronous call to self upgrade the guest agent."""