Add datastore version to backups

This fix adds datastore information to the backup APIs.
These API includes backup list, backup list by instance and
backup show.

Also this fix enhances backup-list to optionally receive a
datastore to filter the backups by.

The integration tests will fail until the update for the client
gets merged: https://review.openstack.org/#/c/90462/

Change-Id: Icfec975b92cd9523e639ad6a2d6787ee4d4cb39d
Implements: blueprint backup-metadata

trove/backup/models.py

@@ -20,6 +20,7 @@ from swiftclient.client import ClientException
 from trove.common import cfg
 from trove.common import exception
 from trove.db.models import DatabaseModelBase
+from trove.datastore import models as datastore_models
 from trove.openstack.common import log as logging
 from trove.taskmanager import api
 from trove.common.remote import create_swift_client
@@ -78,6 +79,8 @@ class Backup(object):
                 instance_model, 'backup_create')
             cls.verify_swift_auth_token(context)
 
+            ds = instance_model.datastore
+            ds_version = instance_model.datastore_version
             parent = None
             if parent_id:
                 # Look up the parent info or fail early if not found or if
@@ -94,6 +97,7 @@ class Backup(object):
                                           state=BackupState.NEW,
                                           instance_id=instance_id,
                                           parent_id=parent_id,
+                                          datastore_version_id=ds_version.id,
                                           deleted=False)
             except exception.InvalidModelError as ex:
                 LOG.exception("Unable to create Backup record:")
@@ -106,6 +110,8 @@ class Backup(object):
                            'backup_type': db_info.backup_type,
                            'checksum': db_info.checksum,
                            'parent': parent,
+                           'datastore': ds.name,
+                           'datastore_version': ds_version.name,
                            }
             api.API(context).create_backup(backup_info, instance_id)
             return db_info
@@ -168,16 +174,23 @@ class Backup(object):
         return query.all(), marker
 
     @classmethod
-    def list(cls, context):
+    def list(cls, context, datastore=None):
         """
         list all live Backups belong to given tenant
         :param cls:
         :param context: tenant_id included
+        :param datastore: datastore to filter by
         :return:
         """
         query = DBBackup.query()
-        query = query.filter_by(tenant_id=context.tenant,
+        filters = [DBBackup.tenant_id == context.tenant,
-                                deleted=False)
+                   DBBackup.deleted == 0]
+        if datastore:
+            ds = datastore_models.Datastore.load(datastore)
+            filters.append(datastore_models.DBDatastoreVersion.
+                           datastore_id == ds.id)
+            query = query.join(datastore_models.DBDatastoreVersion)
+        query = query.filter(*filters)
         return cls._paginate(context, query)
 
     @classmethod
@@ -250,7 +263,8 @@ class DBBackup(DatabaseModelBase):
     _data_fields = ['id', 'name', 'description', 'location', 'backup_type',
                     'size', 'tenant_id', 'state', 'instance_id',
                     'checksum', 'backup_timestamp', 'deleted', 'created',
-                    'updated', 'deleted_at', 'parent_id']
+                    'updated', 'deleted_at', 'parent_id',
+                    'datastore_version_id']
     preserve_on_delete = True
 
     @property
@@ -271,6 +285,18 @@ class DBBackup(DatabaseModelBase):
         else:
             return None
 
+    @property
+    def datastore(self):
+        if self.datastore_version_id:
+            return datastore_models.Datastore.load(
+                self.datastore_version.datastore_id)
+
+    @property
+    def datastore_version(self):
+        if self.datastore_version_id:
+            return datastore_models.DatastoreVersion.load_by_uuid(
+                self.datastore_version_id)
+
     def check_swift_object_exist(self, context, verify_checksum=False):
         try:
             parts = self.location.split('/')

trove/backup/service.py

@@ -37,8 +37,9 @@ class BackupController(wsgi.Controller):
         Return all backups information for a tenant ID.
         """
         LOG.debug("Listing Backups for tenant '%s'" % tenant_id)
+        datastore = req.GET.get('datastore')
         context = req.environ[wsgi.CONTEXT_KEY]
-        backups, marker = Backup.list(context)
+        backups, marker = Backup.list(context, datastore)
         view = views.BackupViews(backups)
         paged = pagination.SimplePaginatedDataView(req.url, 'backups', view,
                                                    marker)

trove/backup/views.py

@@ -20,19 +20,27 @@ class BackupView(object):
         self.backup = backup
 
     def data(self):
-        return {"backup": {
+        result = {
-            "id": self.backup.id,
+            "backup": {
-            "name": self.backup.name,
+                "id": self.backup.id,
-            "description": self.backup.description,
+                "name": self.backup.name,
-            "locationRef": self.backup.location,
+                "description": self.backup.description,
-            "instance_id": self.backup.instance_id,
+                "locationRef": self.backup.location,
-            "created": self.backup.created,
+                "instance_id": self.backup.instance_id,
-            "updated": self.backup.updated,
+                "created": self.backup.created,
-            "size": self.backup.size,
+                "updated": self.backup.updated,
-            "status": self.backup.state,
+                "size": self.backup.size,
-            "parent_id": self.backup.parent_id,
+                "status": self.backup.state,
-        }
+                "parent_id": self.backup.parent_id,
+            }
         }
+        if self.backup.datastore_version_id:
+            result['backup']['datastore'] = {
+                "type": self.backup.datastore.name,
+                "version": self.backup.datastore_version.name,
+                "version_id": self.backup.datastore_version.id
+            }
+        return result
 
 
 class BackupViews(object):

trove/common/exception.py

@@ -359,10 +359,10 @@ class BackupFileNotFound(NotFound):
                 "storage.")
 
 
-class BackupDatastoreVersionMismatchError(TroveError):
+class BackupDatastoreMismatchError(TroveError):
-    message = _("The datastore-version from which the backup was"
+    message = _("The datastore from which the backup was taken, "
-                " taken, %(version1)s, does not match the destination"
+                "%(datastore1)s, does not match the destination"
-                " datastore-version of %(version2)s")
+                " datastore of %(datastore2)s")
 
 
 class SwiftAuthError(TroveError):

trove/db/sqlalchemy/migrate_repo/versions/029_add_backup_datastore.py

@@ -0,0 +1,37 @@
+# Copyright 2013 OpenStack Foundation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from sqlalchemy import ForeignKey
+from sqlalchemy.schema import Column
+from sqlalchemy.schema import MetaData
+
+from trove.db.sqlalchemy.migrate_repo.schema import String
+from trove.db.sqlalchemy.migrate_repo.schema import Table
+
+
+def upgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+    backups = Table('backups', meta, autoload=True)
+    Table('datastore_versions', meta, autoload=True)
+    datastore_version_id = Column('datastore_version_id', String(36),
+                                  ForeignKey('datastore_versions.id'))
+    backups.create_column(datastore_version_id)
+
+
+def downgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+    backups = Table('backups', meta, autoload=True)
+    backups.drop_column('datastore_version_id')

trove/guestagent/backup/backupagent.py

@@ -125,7 +125,11 @@ class BackupAgent(object):
                     if not success:
                         raise BackupError(note)
 
-                    storage.save_metadata(location, bkup.metadata())
+                    meta = bkup.metadata()
+                    meta['datastore'] = backup_info['datastore']
+                    meta['datastore_version'] = backup_info[
+                        'datastore_version']
+                    storage.save_metadata(location, meta)
 
                 except Exception:
                     LOG.exception(_("Error saving %(backup_id)s Backup") %

trove/instance/models.py

@@ -633,16 +633,11 @@ class Instance(BuiltInstance):
                 raise exception.BackupFileNotFound(
                     location=backup_info.location)
 
-            backup_db_info = DBInstance.find_by(
+            if (backup_info.datastore_version_id
-                context=context, id=backup_info.instance_id)
+                    and backup_info.datastore.name != datastore.name):
-            if (backup_db_info.datastore_version_id
+                raise exception.BackupDatastoreMismatchError(
-                    != datastore_version.id):
+                    datastore1=backup_info.datastore.name,
-                ds_version = (datastore_models.DatastoreVersion.
+                    datastore2=datastore.name)
-                              load_by_uuid(backup_db_info.datastore_version_id)
-                              )
-                raise exception.BackupDatastoreVersionMismatchError(
-                    version1=ds_version.name,
-                    version2=datastore_version.name)
 
         if not nics and CONF.default_neutron_networks:
             nics = []

trove/tests/api/backups.py

@@ -97,7 +97,17 @@ class CreateBackups(object):
         assert_equal(instance_info.id, result.instance_id)
         assert_equal('NEW', result.status)
         instance = instance_info.dbaas.instances.get(instance_info.id)
+
+        datastore_version = instance_info.dbaas.datastore_versions.get(
+            instance_info.dbaas_datastore,
+            instance_info.dbaas_datastore_version)
+
         assert_equal('BACKUP', instance.status)
+        assert_equal(instance_info.dbaas_datastore,
+                     result.datastore['type'])
+        assert_equal(instance_info.dbaas_datastore_version,
+                     result.datastore['version'])
+        assert_equal(datastore_version.id, result.datastore['version_id'])
 
 
 @test(runs_after=[CreateBackups],
@@ -163,6 +173,33 @@ class ListBackups(object):
         assert_equal(instance_info.id, backup.instance_id)
         assert_equal('COMPLETED', backup.status)
 
+    @test
+    def test_backup_list_filter_datastore(self):
+        """test list backups and filter by datastore."""
+        result = instance_info.dbaas.backups.list(
+            datastore=instance_info.dbaas_datastore)
+        assert_equal(backup_count_prior_to_create + 1, len(result))
+        backup = result[0]
+        assert_equal(BACKUP_NAME, backup.name)
+        assert_equal(BACKUP_DESC, backup.description)
+        assert_not_equal(0.0, backup.size)
+        assert_equal(instance_info.id, backup.instance_id)
+        assert_equal('COMPLETED', backup.status)
+
+    @test
+    def test_backup_list_filter_different_datastore(self):
+        """test list backups and filter by datastore."""
+        result = instance_info.dbaas.backups.list(
+            datastore='Test_Datastore_1')
+        # There should not be any backups for this datastore
+        assert_equal(0, len(result))
+
+    @test
+    def test_backup_list_filter_datastore_not_found(self):
+        """test list backups and filter by datastore."""
+        assert_raises(exceptions.BadRequest, instance_info.dbaas.backups.list,
+                      datastore='NOT_FOUND')
+
     @test
     def test_backup_list_for_instance(self):
         """Test backup list for instance."""
@@ -186,6 +223,15 @@ class ListBackups(object):
         assert_equal(instance_info.id, backup.instance_id)
         assert_not_equal(0.0, backup.size)
         assert_equal('COMPLETED', backup.status)
+        assert_equal(instance_info.dbaas_datastore,
+                     backup.datastore['type'])
+        assert_equal(instance_info.dbaas_datastore_version,
+                     backup.datastore['version'])
+
+        datastore_version = instance_info.dbaas.datastore_versions.get(
+            instance_info.dbaas_datastore,
+            instance_info.dbaas_datastore_version)
+        assert_equal(datastore_version.id, backup.datastore['version_id'])
 
         # Test to make sure that user in other tenant is not able
         # to GET this backup

trove/tests/unittests/backup/test_backup_models.py

@@ -58,6 +58,8 @@ class BackupCreateTest(testtools.TestCase):
                           return_value=instance):
             instance.validate_can_perform_action = MagicMock(
                 return_value=None)
+            instance.datastore_version = MagicMock()
+            instance.datastore_version.id = 'datastore-id-999'
             with patch.object(models.Backup, 'validate_can_perform_action',
                               return_value=None):
                 with patch.object(models.Backup, 'verify_swift_auth_token',
@@ -80,6 +82,8 @@ class BackupCreateTest(testtools.TestCase):
                                      db_record['instance_id'])
                     self.assertEqual(models.BackupState.NEW,
                                      db_record['state'])
+                    self.assertEqual(instance.datastore_version.id,
+                                     db_record['datastore_version_id'])
 
     def test_create_incremental(self):
         instance = MagicMock()
@@ -88,6 +92,10 @@ class BackupCreateTest(testtools.TestCase):
                           return_value=instance):
             instance.validate_can_perform_action = MagicMock(
                 return_value=None)
+            instance.validate_can_perform_action = MagicMock(
+                return_value=None)
+            instance.datastore_version = MagicMock()
+            instance.datastore_version.id = 'datastore-id-999'
             with patch.object(models.Backup, 'validate_can_perform_action',
                               return_value=None):
                 with patch.object(models.Backup, 'verify_swift_auth_token',
@@ -118,6 +126,8 @@ class BackupCreateTest(testtools.TestCase):
                                          db_record['state'])
                         self.assertEqual('parent_uuid',
                                          db_record['parent_id'])
+                        self.assertEqual(instance.datastore_version.id,
+                                         db_record['datastore_version_id'])
 
     def test_create_instance_not_found(self):
         self.assertRaises(exception.NotFound, models.Backup.create,

trove/tests/unittests/backup/test_backupagent.py

@@ -234,6 +234,8 @@ class BackupAgentTest(testtools.TestCase):
                        'location': 'fake-location',
                        'type': 'InnoBackupEx',
                        'checksum': 'fake-checksum',
+                       'datastore': 'mysql',
+                       'datastore_version': '5.5'
                        }
         agent.execute_backup(context=None, backup_info=backup_info,
                              runner=MockBackup)
@@ -267,6 +269,8 @@ class BackupAgentTest(testtools.TestCase):
                        'location': 'fake-location',
                        'type': 'InnoBackupEx',
                        'checksum': 'fake-checksum',
+                       'datastore': 'mysql',
+                       'datastore_version': '5.5'
                        }
 
         self.assertRaises(backupagent.BackupError, agent.execute_backup,