From 1543db553e0ad992bca156e9da91c8d86beadc94 Mon Sep 17 00:00:00 2001 From: Steve Leon Date: Mon, 21 Apr 2014 22:07:20 -0700 Subject: [PATCH] Add datastore version to backups This fix adds datastore information to the backup APIs. These API includes backup list, backup list by instance and backup show. Also this fix enhances backup-list to optionally receive a datastore to filter the backups by. The integration tests will fail until the update for the client gets merged: https://review.openstack.org/#/c/90462/ Change-Id: Icfec975b92cd9523e639ad6a2d6787ee4d4cb39d Implements: blueprint backup-metadata --- trove/backup/models.py | 34 ++++++++++++-- trove/backup/service.py | 3 +- trove/backup/views.py | 32 ++++++++----- trove/common/exception.py | 8 ++-- .../versions/029_add_backup_datastore.py | 37 +++++++++++++++ trove/guestagent/backup/backupagent.py | 6 ++- trove/instance/models.py | 15 ++---- trove/tests/api/backups.py | 46 +++++++++++++++++++ .../unittests/backup/test_backup_models.py | 10 ++++ .../unittests/backup/test_backupagent.py | 4 ++ 10 files changed, 163 insertions(+), 32 deletions(-) create mode 100644 trove/db/sqlalchemy/migrate_repo/versions/029_add_backup_datastore.py diff --git a/trove/backup/models.py b/trove/backup/models.py index 92720d55b9..bbb5bba9b7 100644 --- a/trove/backup/models.py +++ b/trove/backup/models.py @@ -20,6 +20,7 @@ from swiftclient.client import ClientException from trove.common import cfg from trove.common import exception from trove.db.models import DatabaseModelBase +from trove.datastore import models as datastore_models from trove.openstack.common import log as logging from trove.taskmanager import api from trove.common.remote import create_swift_client @@ -78,6 +79,8 @@ class Backup(object): instance_model, 'backup_create') cls.verify_swift_auth_token(context) + ds = instance_model.datastore + ds_version = instance_model.datastore_version parent = None if parent_id: # Look up the parent info or fail early if not found or if @@ -94,6 +97,7 @@ class Backup(object): state=BackupState.NEW, instance_id=instance_id, parent_id=parent_id, + datastore_version_id=ds_version.id, deleted=False) except exception.InvalidModelError as ex: LOG.exception("Unable to create Backup record:") @@ -106,6 +110,8 @@ class Backup(object): 'backup_type': db_info.backup_type, 'checksum': db_info.checksum, 'parent': parent, + 'datastore': ds.name, + 'datastore_version': ds_version.name, } api.API(context).create_backup(backup_info, instance_id) return db_info @@ -168,16 +174,23 @@ class Backup(object): return query.all(), marker @classmethod - def list(cls, context): + def list(cls, context, datastore=None): """ list all live Backups belong to given tenant :param cls: :param context: tenant_id included + :param datastore: datastore to filter by :return: """ query = DBBackup.query() - query = query.filter_by(tenant_id=context.tenant, - deleted=False) + filters = [DBBackup.tenant_id == context.tenant, + DBBackup.deleted == 0] + if datastore: + ds = datastore_models.Datastore.load(datastore) + filters.append(datastore_models.DBDatastoreVersion. + datastore_id == ds.id) + query = query.join(datastore_models.DBDatastoreVersion) + query = query.filter(*filters) return cls._paginate(context, query) @classmethod @@ -250,7 +263,8 @@ class DBBackup(DatabaseModelBase): _data_fields = ['id', 'name', 'description', 'location', 'backup_type', 'size', 'tenant_id', 'state', 'instance_id', 'checksum', 'backup_timestamp', 'deleted', 'created', - 'updated', 'deleted_at', 'parent_id'] + 'updated', 'deleted_at', 'parent_id', + 'datastore_version_id'] preserve_on_delete = True @property @@ -271,6 +285,18 @@ class DBBackup(DatabaseModelBase): else: return None + @property + def datastore(self): + if self.datastore_version_id: + return datastore_models.Datastore.load( + self.datastore_version.datastore_id) + + @property + def datastore_version(self): + if self.datastore_version_id: + return datastore_models.DatastoreVersion.load_by_uuid( + self.datastore_version_id) + def check_swift_object_exist(self, context, verify_checksum=False): try: parts = self.location.split('/') diff --git a/trove/backup/service.py b/trove/backup/service.py index 6fb52f667d..08baa38994 100644 --- a/trove/backup/service.py +++ b/trove/backup/service.py @@ -37,8 +37,9 @@ class BackupController(wsgi.Controller): Return all backups information for a tenant ID. """ LOG.debug("Listing Backups for tenant '%s'" % tenant_id) + datastore = req.GET.get('datastore') context = req.environ[wsgi.CONTEXT_KEY] - backups, marker = Backup.list(context) + backups, marker = Backup.list(context, datastore) view = views.BackupViews(backups) paged = pagination.SimplePaginatedDataView(req.url, 'backups', view, marker) diff --git a/trove/backup/views.py b/trove/backup/views.py index f35322e705..9e1cfd2bd6 100644 --- a/trove/backup/views.py +++ b/trove/backup/views.py @@ -20,19 +20,27 @@ class BackupView(object): self.backup = backup def data(self): - return {"backup": { - "id": self.backup.id, - "name": self.backup.name, - "description": self.backup.description, - "locationRef": self.backup.location, - "instance_id": self.backup.instance_id, - "created": self.backup.created, - "updated": self.backup.updated, - "size": self.backup.size, - "status": self.backup.state, - "parent_id": self.backup.parent_id, - } + result = { + "backup": { + "id": self.backup.id, + "name": self.backup.name, + "description": self.backup.description, + "locationRef": self.backup.location, + "instance_id": self.backup.instance_id, + "created": self.backup.created, + "updated": self.backup.updated, + "size": self.backup.size, + "status": self.backup.state, + "parent_id": self.backup.parent_id, + } } + if self.backup.datastore_version_id: + result['backup']['datastore'] = { + "type": self.backup.datastore.name, + "version": self.backup.datastore_version.name, + "version_id": self.backup.datastore_version.id + } + return result class BackupViews(object): diff --git a/trove/common/exception.py b/trove/common/exception.py index 6529989548..44d44aaaf9 100644 --- a/trove/common/exception.py +++ b/trove/common/exception.py @@ -359,10 +359,10 @@ class BackupFileNotFound(NotFound): "storage.") -class BackupDatastoreVersionMismatchError(TroveError): - message = _("The datastore-version from which the backup was" - " taken, %(version1)s, does not match the destination" - " datastore-version of %(version2)s") +class BackupDatastoreMismatchError(TroveError): + message = _("The datastore from which the backup was taken, " + "%(datastore1)s, does not match the destination" + " datastore of %(datastore2)s") class SwiftAuthError(TroveError): diff --git a/trove/db/sqlalchemy/migrate_repo/versions/029_add_backup_datastore.py b/trove/db/sqlalchemy/migrate_repo/versions/029_add_backup_datastore.py new file mode 100644 index 0000000000..dc1fcdd486 --- /dev/null +++ b/trove/db/sqlalchemy/migrate_repo/versions/029_add_backup_datastore.py @@ -0,0 +1,37 @@ +# Copyright 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import ForeignKey +from sqlalchemy.schema import Column +from sqlalchemy.schema import MetaData + +from trove.db.sqlalchemy.migrate_repo.schema import String +from trove.db.sqlalchemy.migrate_repo.schema import Table + + +def upgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + backups = Table('backups', meta, autoload=True) + Table('datastore_versions', meta, autoload=True) + datastore_version_id = Column('datastore_version_id', String(36), + ForeignKey('datastore_versions.id')) + backups.create_column(datastore_version_id) + + +def downgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + backups = Table('backups', meta, autoload=True) + backups.drop_column('datastore_version_id') diff --git a/trove/guestagent/backup/backupagent.py b/trove/guestagent/backup/backupagent.py index c9d77a96e8..9c9f66b39b 100644 --- a/trove/guestagent/backup/backupagent.py +++ b/trove/guestagent/backup/backupagent.py @@ -125,7 +125,11 @@ class BackupAgent(object): if not success: raise BackupError(note) - storage.save_metadata(location, bkup.metadata()) + meta = bkup.metadata() + meta['datastore'] = backup_info['datastore'] + meta['datastore_version'] = backup_info[ + 'datastore_version'] + storage.save_metadata(location, meta) except Exception: LOG.exception(_("Error saving %(backup_id)s Backup") % diff --git a/trove/instance/models.py b/trove/instance/models.py index 431dc65cd5..136af57747 100644 --- a/trove/instance/models.py +++ b/trove/instance/models.py @@ -633,16 +633,11 @@ class Instance(BuiltInstance): raise exception.BackupFileNotFound( location=backup_info.location) - backup_db_info = DBInstance.find_by( - context=context, id=backup_info.instance_id) - if (backup_db_info.datastore_version_id - != datastore_version.id): - ds_version = (datastore_models.DatastoreVersion. - load_by_uuid(backup_db_info.datastore_version_id) - ) - raise exception.BackupDatastoreVersionMismatchError( - version1=ds_version.name, - version2=datastore_version.name) + if (backup_info.datastore_version_id + and backup_info.datastore.name != datastore.name): + raise exception.BackupDatastoreMismatchError( + datastore1=backup_info.datastore.name, + datastore2=datastore.name) if not nics and CONF.default_neutron_networks: nics = [] diff --git a/trove/tests/api/backups.py b/trove/tests/api/backups.py index f7ae6c0f31..12065634b0 100644 --- a/trove/tests/api/backups.py +++ b/trove/tests/api/backups.py @@ -97,7 +97,17 @@ class CreateBackups(object): assert_equal(instance_info.id, result.instance_id) assert_equal('NEW', result.status) instance = instance_info.dbaas.instances.get(instance_info.id) + + datastore_version = instance_info.dbaas.datastore_versions.get( + instance_info.dbaas_datastore, + instance_info.dbaas_datastore_version) + assert_equal('BACKUP', instance.status) + assert_equal(instance_info.dbaas_datastore, + result.datastore['type']) + assert_equal(instance_info.dbaas_datastore_version, + result.datastore['version']) + assert_equal(datastore_version.id, result.datastore['version_id']) @test(runs_after=[CreateBackups], @@ -163,6 +173,33 @@ class ListBackups(object): assert_equal(instance_info.id, backup.instance_id) assert_equal('COMPLETED', backup.status) + @test + def test_backup_list_filter_datastore(self): + """test list backups and filter by datastore.""" + result = instance_info.dbaas.backups.list( + datastore=instance_info.dbaas_datastore) + assert_equal(backup_count_prior_to_create + 1, len(result)) + backup = result[0] + assert_equal(BACKUP_NAME, backup.name) + assert_equal(BACKUP_DESC, backup.description) + assert_not_equal(0.0, backup.size) + assert_equal(instance_info.id, backup.instance_id) + assert_equal('COMPLETED', backup.status) + + @test + def test_backup_list_filter_different_datastore(self): + """test list backups and filter by datastore.""" + result = instance_info.dbaas.backups.list( + datastore='Test_Datastore_1') + # There should not be any backups for this datastore + assert_equal(0, len(result)) + + @test + def test_backup_list_filter_datastore_not_found(self): + """test list backups and filter by datastore.""" + assert_raises(exceptions.BadRequest, instance_info.dbaas.backups.list, + datastore='NOT_FOUND') + @test def test_backup_list_for_instance(self): """Test backup list for instance.""" @@ -186,6 +223,15 @@ class ListBackups(object): assert_equal(instance_info.id, backup.instance_id) assert_not_equal(0.0, backup.size) assert_equal('COMPLETED', backup.status) + assert_equal(instance_info.dbaas_datastore, + backup.datastore['type']) + assert_equal(instance_info.dbaas_datastore_version, + backup.datastore['version']) + + datastore_version = instance_info.dbaas.datastore_versions.get( + instance_info.dbaas_datastore, + instance_info.dbaas_datastore_version) + assert_equal(datastore_version.id, backup.datastore['version_id']) # Test to make sure that user in other tenant is not able # to GET this backup diff --git a/trove/tests/unittests/backup/test_backup_models.py b/trove/tests/unittests/backup/test_backup_models.py index 24ef77839e..67983defba 100644 --- a/trove/tests/unittests/backup/test_backup_models.py +++ b/trove/tests/unittests/backup/test_backup_models.py @@ -58,6 +58,8 @@ class BackupCreateTest(testtools.TestCase): return_value=instance): instance.validate_can_perform_action = MagicMock( return_value=None) + instance.datastore_version = MagicMock() + instance.datastore_version.id = 'datastore-id-999' with patch.object(models.Backup, 'validate_can_perform_action', return_value=None): with patch.object(models.Backup, 'verify_swift_auth_token', @@ -80,6 +82,8 @@ class BackupCreateTest(testtools.TestCase): db_record['instance_id']) self.assertEqual(models.BackupState.NEW, db_record['state']) + self.assertEqual(instance.datastore_version.id, + db_record['datastore_version_id']) def test_create_incremental(self): instance = MagicMock() @@ -88,6 +92,10 @@ class BackupCreateTest(testtools.TestCase): return_value=instance): instance.validate_can_perform_action = MagicMock( return_value=None) + instance.validate_can_perform_action = MagicMock( + return_value=None) + instance.datastore_version = MagicMock() + instance.datastore_version.id = 'datastore-id-999' with patch.object(models.Backup, 'validate_can_perform_action', return_value=None): with patch.object(models.Backup, 'verify_swift_auth_token', @@ -118,6 +126,8 @@ class BackupCreateTest(testtools.TestCase): db_record['state']) self.assertEqual('parent_uuid', db_record['parent_id']) + self.assertEqual(instance.datastore_version.id, + db_record['datastore_version_id']) def test_create_instance_not_found(self): self.assertRaises(exception.NotFound, models.Backup.create, diff --git a/trove/tests/unittests/backup/test_backupagent.py b/trove/tests/unittests/backup/test_backupagent.py index 941f8ef968..6d7dff6cd3 100644 --- a/trove/tests/unittests/backup/test_backupagent.py +++ b/trove/tests/unittests/backup/test_backupagent.py @@ -234,6 +234,8 @@ class BackupAgentTest(testtools.TestCase): 'location': 'fake-location', 'type': 'InnoBackupEx', 'checksum': 'fake-checksum', + 'datastore': 'mysql', + 'datastore_version': '5.5' } agent.execute_backup(context=None, backup_info=backup_info, runner=MockBackup) @@ -267,6 +269,8 @@ class BackupAgentTest(testtools.TestCase): 'location': 'fake-location', 'type': 'InnoBackupEx', 'checksum': 'fake-checksum', + 'datastore': 'mysql', + 'datastore_version': '5.5' } self.assertRaises(backupagent.BackupError, agent.execute_backup,