diff --git a/trove/guestagent/datastore/mysql_common/service.py b/trove/guestagent/datastore/mysql_common/service.py index da04d278d3..96c9330d5a 100644 --- a/trove/guestagent/datastore/mysql_common/service.py +++ b/trove/guestagent/datastore/mysql_common/service.py @@ -25,6 +25,7 @@ import uuid from oslo_log import log as logging from oslo_utils import encodeutils +from pymysql import err as pymysql_err from six.moves import urllib import sqlalchemy from sqlalchemy import exc @@ -568,6 +569,14 @@ class BaseKeepAliveConnection(interfaces.PoolListener): raise exc.DisconnectionError() else: raise + # MariaDB seems to timeout the client in a different + # way than MySQL and PXC, which manifests itself as + # an invalid packet sequence. Handle it as well. + except pymysql_err.InternalError as ex: + if "Packet sequence number wrong" in ex.message: + raise exc.DisconnectionError() + else: + raise @six.add_metaclass(abc.ABCMeta) diff --git a/trove/guestagent/strategies/replication/experimental/mariadb_gtid.py b/trove/guestagent/strategies/replication/experimental/mariadb_gtid.py index 89446ed529..c73d9315fa 100644 --- a/trove/guestagent/strategies/replication/experimental/mariadb_gtid.py +++ b/trove/guestagent/strategies/replication/experimental/mariadb_gtid.py @@ -55,6 +55,7 @@ class MariaDBGTIDReplication(mysql_base.MysqlReplicationBase): "MASTER_PORT=%(port)s, " "MASTER_USER='%(user)s', " "MASTER_PASSWORD='%(password)s', " + "MASTER_CONNECT_RETRY=15, " "MASTER_USE_GTID=slave_pos" % { 'host': snapshot['master']['host'], diff --git a/trove/taskmanager/manager.py b/trove/taskmanager/manager.py index d375b19a7c..a70872b723 100644 --- a/trove/taskmanager/manager.py +++ b/trove/taskmanager/manager.py @@ -120,41 +120,43 @@ class Manager(periodic_task.PeriodicTasks): # and possibly some number of "orphaned" slaves exception_replicas = [] + error_messages = "" for replica in replica_models: try: if replica.id != master_candidate.id: replica.detach_replica(old_master, for_failover=True) replica.attach_replica(master_candidate) - except exception.TroveError: - msg = _("promote-to-replica-source: Unable to migrate " - "replica %(slave)s from old replica source " - "%(old_master)s to new source %(new_master)s.") - msg_values = { - "slave": replica.id, - "old_master": old_master.id, - "new_master": master_candidate.id - } - LOG.exception(msg % msg_values) + except exception.TroveError as ex: + msg = (_("Unable to migrate replica %(slave)s from " + "old replica source %(old_master)s to " + "new source %(new_master)s on promote.") % + {"slave": replica.id, + "old_master": old_master.id, + "new_master": master_candidate.id}) + LOG.exception(msg) exception_replicas.append(replica) + error_messages += "%s (%s)\n" % (msg, ex) try: old_master.demote_replication_master() - except Exception: - LOG.exception(_("Exception demoting old replica source")) + except Exception as ex: + msg = (_("Exception demoting old replica source %s.") % + old_master.id) + LOG.exception(msg) exception_replicas.append(old_master) + error_messages += "%s (%s)\n" % (msg, ex) self._set_task_status([old_master] + replica_models, InstanceTasks.NONE) if exception_replicas: self._set_task_status(exception_replicas, InstanceTasks.PROMOTION_ERROR) - msg = _("promote-to-replica-source %(id)s: The following " - "replicas may not have been switched: %(replicas)s") - msg_values = { - "id": master_candidate.id, - "replicas": exception_replicas - } - raise ReplicationSlaveAttachError(msg % msg_values) + msg = (_("promote-to-replica-source %(id)s: The following " + "replicas may not have been switched: %(replicas)s") % + {"id": master_candidate.id, + "replicas": [repl.id for repl in exception_replicas]}) + raise ReplicationSlaveAttachError("%s:\n%s" % + (msg, error_messages)) with EndNotification(context): master_candidate = BuiltInstanceTasks.load(context, instance_id) @@ -207,35 +209,34 @@ class Manager(periodic_task.PeriodicTasks): old_master.attach_public_ips(slave_ips) exception_replicas = [] + error_messages = "" for replica in replica_models: try: if replica.id != master_candidate.id: replica.detach_replica(old_master, for_failover=True) replica.attach_replica(master_candidate) - except exception.TroveError: - msg = _("eject-replica-source: Unable to migrate " - "replica %(slave)s from old replica source " - "%(old_master)s to new source %(new_master)s.") - msg_values = { - "slave": replica.id, - "old_master": old_master.id, - "new_master": master_candidate.id - } - LOG.exception(msg % msg_values) - exception_replicas.append(replica.id) + except exception.TroveError as ex: + msg = (_("Unable to migrate replica %(slave)s from " + "old replica source %(old_master)s to " + "new source %(new_master)s on eject.") % + {"slave": replica.id, + "old_master": old_master.id, + "new_master": master_candidate.id}) + LOG.exception(msg) + exception_replicas.append(replica) + error_messages += "%s (%s)\n" % (msg, ex) self._set_task_status([old_master] + replica_models, InstanceTasks.NONE) if exception_replicas: self._set_task_status(exception_replicas, InstanceTasks.EJECTION_ERROR) - msg = _("eject-replica-source %(id)s: The following " - "replicas may not have been switched: %(replicas)s") - msg_values = { - "id": master_candidate.id, - "replicas": exception_replicas - } - raise ReplicationSlaveAttachError(msg % msg_values) + msg = (_("eject-replica-source %(id)s: The following " + "replicas may not have been switched: %(replicas)s") % + {"id": master_candidate.id, + "replicas": [repl.id for repl in exception_replicas]}) + raise ReplicationSlaveAttachError("%s:\n%s" % + (msg, error_messages)) with EndNotification(context): master = BuiltInstanceTasks.load(context, instance_id) diff --git a/trove/tests/int_tests.py b/trove/tests/int_tests.py index 18bfe5c96b..519db3ec46 100644 --- a/trove/tests/int_tests.py +++ b/trove/tests/int_tests.py @@ -177,6 +177,10 @@ guest_log_groups.extend([guest_log_group.GROUP]) instance_actions_groups = list(instance_create_groups) instance_actions_groups.extend([instance_actions_group.GROUP]) +instance_groups = list(instance_actions_groups) +instance_groups.extend([instance_error_create_group.GROUP, + instance_force_delete_group.GROUP]) + module_groups = list(instance_create_groups) module_groups.extend([module_group.GROUP]) @@ -197,9 +201,8 @@ user_actions_groups = list(instance_create_groups) user_actions_groups.extend([user_actions_group.GROUP]) # groups common to all datastores -common_groups = list(instance_actions_groups) -common_groups.extend([guest_log_groups, instance_error_create_groups, - instance_force_delete_groups, module_groups]) +common_groups = list(instance_groups) +common_groups.extend([guest_log_groups, module_groups]) # Register: Component based groups register(["backup"], backup_groups) @@ -210,11 +213,12 @@ register(["configuration"], configuration_groups) register(["configuration_create"], configuration_create_groups) register(["database"], database_actions_groups) register(["guest_log"], guest_log_groups) -register(["instance", "instance_actions"], instance_actions_groups) +register(["instance"], instance_groups) +register(["instance_actions"], instance_actions_groups) register(["instance_create"], instance_create_groups) -register(["instance_error_create"], instance_error_create_groups) -register(["instance_upgrade"], instance_upgrade_groups) +register(["instance_error"], instance_error_create_groups) register(["instance_force_delete"], instance_force_delete_groups) +register(["instance_upgrade"], instance_upgrade_groups) register(["module"], module_groups) register(["module_create"], module_create_groups) register(["replication"], replication_groups) @@ -223,35 +227,100 @@ register(["root"], root_actions_groups) register(["user"], user_actions_groups) # Register: Datastore based groups -# These should contain all functionality currently supported by the datastore -register(["db2_supported"], common_groups, - database_actions_groups, user_actions_groups, configuration_groups) -register(["cassandra_supported"], common_groups, - user_actions_groups, database_actions_groups, - backup_groups, configuration_groups, cluster_actions_groups) -register(["couchbase_supported"], common_groups, backup_groups, - root_actions_groups) -register(["couchdb_supported"], common_groups, backup_groups, - user_actions_groups, database_actions_groups, root_actions_groups) -register(["postgresql_supported"], common_groups, - backup_groups, database_actions_groups, configuration_groups, - root_actions_groups, user_actions_groups, - backup_incremental_groups, replication_groups) -register(["mysql_supported", "percona_supported"], common_groups, - backup_groups, configuration_groups, database_actions_groups, - replication_promote_groups, instance_upgrade_groups, - root_actions_groups, user_actions_groups, backup_incremental_groups) -register(["mariadb_supported"], common_groups, - backup_groups, cluster_actions_groups, configuration_groups, - database_actions_groups, replication_promote_groups, - root_actions_groups, user_actions_groups) -register(["mongodb_supported"], common_groups, - backup_groups, cluster_actions_groups, configuration_groups, - database_actions_groups, root_actions_groups, user_actions_groups) -register(["pxc_supported"], common_groups, - backup_groups, configuration_groups, database_actions_groups, - cluster_actions_groups, root_actions_groups, user_actions_groups) -register(["redis_supported"], common_groups, - backup_groups, replication_promote_groups, cluster_actions_groups) -register(["vertica_supported"], common_groups, - cluster_actions_groups, root_actions_groups, configuration_groups) +# These should contain all functionality currently supported by the datastore. +# Keeping them in alphabetical order may reduce the number of merge conflicts. +register( + ["db2_supported"], common_groups, + configuration_groups, + database_actions_groups, + user_actions_groups, +) + +register( + ["cassandra_supported"], common_groups, + backup_groups, + database_actions_groups, + cluster_actions_groups, + configuration_groups, + user_actions_groups, +) + +register( + ["couchbase_supported"], common_groups, + backup_groups, + root_actions_groups, +) + +register( + ["couchdb_supported"], common_groups, + backup_groups, + database_actions_groups, + root_actions_groups, + user_actions_groups, +) + +register( + ["postgresql_supported"], common_groups, + backup_incremental_groups, + database_actions_groups, + configuration_groups, + replication_groups, + root_actions_groups, + user_actions_groups, +) + +register( + ["mysql_supported", "percona_supported"], common_groups, + backup_incremental_groups, + configuration_groups, + database_actions_groups, + instance_upgrade_groups, + replication_promote_groups, + root_actions_groups, + user_actions_groups, +) + +register( + ["mariadb_supported"], common_groups, + backup_incremental_groups, + cluster_actions_groups, + configuration_groups, + database_actions_groups, + replication_promote_groups, + root_actions_groups, + user_actions_groups, +) + +register( + ["mongodb_supported"], common_groups, + backup_groups, + cluster_actions_groups, + configuration_groups, + database_actions_groups, + root_actions_groups, + user_actions_groups, +) + +register( + ["pxc_supported"], common_groups, + backup_incremental_groups, + cluster_actions_groups, + configuration_groups, + database_actions_groups, + root_actions_groups, + user_actions_groups, +) + +register( + ["redis_supported"], common_groups, + backup_groups, + cluster_actions_groups, + replication_promote_groups, +) + +register( + ["vertica_supported"], common_groups, + cluster_actions_groups, + configuration_groups, + root_actions_groups, +) diff --git a/trove/tests/scenario/groups/instance_create_group.py b/trove/tests/scenario/groups/instance_create_group.py index 46a0356b57..228be246b7 100644 --- a/trove/tests/scenario/groups/instance_create_group.py +++ b/trove/tests/scenario/groups/instance_create_group.py @@ -58,7 +58,8 @@ class InstanceCreateGroup(TestGroup): @test(depends_on_groups=[groups.INST_CREATE], groups=[GROUP, groups.INST_CREATE_WAIT], - runs_after_groups=[groups.MODULE_CREATE, groups.CFGGRP_CREATE]) + runs_after_groups=[groups.MODULE_CREATE, groups.CFGGRP_CREATE, + groups.INST_ERROR_CREATE_WAIT]) class InstanceCreateWaitGroup(TestGroup): """Test that Instance Create Completes.""" diff --git a/trove/tests/scenario/runners/backup_runners.py b/trove/tests/scenario/runners/backup_runners.py index 3833806c7d..1c2f26fd38 100644 --- a/trove/tests/scenario/runners/backup_runners.py +++ b/trove/tests/scenario/runners/backup_runners.py @@ -260,8 +260,7 @@ class BackupRunner(TestRunner): self.unauth_client.backups.get, self.backup_info.id) # we're using a different client, so we'll check the return code # on it explicitly, instead of depending on 'assert_raises' - self.assert_client_code(expected_http_code=expected_http_code, - client=self.unauth_client) + self.assert_client_code(expected_http_code, client=self.unauth_client) def run_add_data_for_inc_backup_1(self): self.backup_host = self.get_instance_host() @@ -304,7 +303,7 @@ class BackupRunner(TestRunner): def assert_restore_from_backup(self, backup_ref, suffix='', expected_http_code=200): result = self._restore_from_backup(backup_ref, suffix=suffix) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) self.assert_equal('BUILD', result.status, 'Unexpected instance status') return result.id @@ -374,7 +373,7 @@ class BackupRunner(TestRunner): def assert_delete_restored_instance( self, instance_id, expected_http_code): self.auth_client.instances.delete(instance_id) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) def run_delete_restored_inc_1_instance(self, expected_http_code=202): self.assert_delete_restored_instance( @@ -412,8 +411,7 @@ class BackupRunner(TestRunner): self.unauth_client.backups.delete, self.backup_info.id) # we're using a different client, so we'll check the return code # on it explicitly, instead of depending on 'assert_raises' - self.assert_client_code(expected_http_code=expected_http_code, - client=self.unauth_client) + self.assert_client_code(expected_http_code, client=self.unauth_client) def run_delete_inc_2_backup(self, expected_http_code=202): self.assert_delete_backup( @@ -423,7 +421,7 @@ class BackupRunner(TestRunner): def assert_delete_backup( self, backup_id, expected_http_code): self.auth_client.backups.delete(backup_id) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) self._wait_until_backup_is_gone(backup_id) def _wait_until_backup_is_gone(self, backup_id): diff --git a/trove/tests/scenario/runners/cluster_actions_runners.py b/trove/tests/scenario/runners/cluster_actions_runners.py index 5c08d8462b..61b2b24a58 100644 --- a/trove/tests/scenario/runners/cluster_actions_runners.py +++ b/trove/tests/scenario/runners/cluster_actions_runners.py @@ -139,7 +139,7 @@ class ClusterActionsRunner(TestRunner): def assert_cluster_list(self, expected_count, expected_http_code): count = len(self.auth_client.clusters.list()) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) self.assert_equal(expected_count, count, "Unexpected cluster count") def run_cluster_show(self, expected_http_code=200, @@ -328,7 +328,8 @@ class ClusterActionsRunner(TestRunner): self, cluster_id, expected_task_name, expected_http_code, check_locality=True): if expected_http_code is not None: - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, + client=self.auth_client) if expected_task_name: self._assert_cluster_response(cluster_id, expected_task_name, check_locality=check_locality) @@ -365,7 +366,7 @@ class ClusterActionsRunner(TestRunner): def _assert_cluster_response(self, cluster_id, expected_task_name, expected_http_code=200, check_locality=True): cluster = self.auth_client.clusters.get(cluster_id) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) self._assert_cluster_values(cluster, expected_task_name, check_locality=check_locality) @@ -401,7 +402,7 @@ class ClusterActionsRunner(TestRunner): "Cluster '%s' still existed after %s seconds." % (cluster_id, self._time_since(t0))) except exceptions.NotFound: - self.assert_client_code(404) + self.assert_client_code(404, client=self.auth_client) class CassandraClusterActionsRunner(ClusterActionsRunner): diff --git a/trove/tests/scenario/runners/configuration_runners.py b/trove/tests/scenario/runners/configuration_runners.py index 6d0a81019a..53584e3c53 100644 --- a/trove/tests/scenario/runners/configuration_runners.py +++ b/trove/tests/scenario/runners/configuration_runners.py @@ -134,7 +134,7 @@ class ConfigurationRunner(TestRunner): description, datastore=self.instance_info.dbaas_datastore, datastore_version=self.instance_info.dbaas_datastore_version) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) with TypeCheck('Configuration', result) as configuration: configuration.has_field('name', basestring) @@ -473,7 +473,7 @@ class ConfigurationRunner(TestRunner): def assert_group_delete(self, group_id, expected_http_code): self.auth_client.configurations.delete(group_id) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) def run_delete_non_dynamic_group(self, expected_http_code=202): if self.non_dynamic_group_id: @@ -514,15 +514,14 @@ class ConfigurationRunner(TestRunner): datastore_version=self.instance_info.dbaas_datastore_version, availability_zone="nova", configuration=config_id) - self.assert_client_code(200) + self.assert_client_code(200, client=self.auth_client) self.assert_equal("BUILD", result.status, 'Unexpected inst status') return result.id def run_wait_for_conf_instance( - self, expected_states=['BUILD', 'ACTIVE'], expected_http_code=200): + self, expected_states=['BUILD', 'ACTIVE']): if self.config_inst_id: - self.assert_instance_action(self.config_inst_id, expected_states, - expected_http_code) + self.assert_instance_action(self.config_inst_id, expected_states) self.create_test_helper_on_instance(self.config_inst_id) inst = self.auth_client.instances.get(self.config_inst_id) self.assert_equal(self.config_id_for_inst, @@ -546,7 +545,7 @@ class ConfigurationRunner(TestRunner): def assert_delete_conf_instance(self, instance_id, expected_http_code): self.auth_client.instances.delete(instance_id) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) def run_wait_for_delete_conf_instance( self, expected_last_state=['SHUTDOWN']): diff --git a/trove/tests/scenario/runners/database_actions_runners.py b/trove/tests/scenario/runners/database_actions_runners.py index b2f890a342..52829ee930 100644 --- a/trove/tests/scenario/runners/database_actions_runners.py +++ b/trove/tests/scenario/runners/database_actions_runners.py @@ -53,7 +53,7 @@ class DatabaseActionsRunner(TestRunner): def assert_databases_create(self, instance_id, serial_databases_def, expected_http_code): self.auth_client.databases.create(instance_id, serial_databases_def) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) self.wait_for_database_create(instance_id, serial_databases_def) return serial_databases_def @@ -64,7 +64,7 @@ class DatabaseActionsRunner(TestRunner): def assert_databases_list(self, instance_id, expected_database_defs, expected_http_code, limit=2): full_list = self.auth_client.databases.list(instance_id) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) listed_databases = {database.name: database for database in full_list} self.assert_is_none(full_list.next, "Unexpected pagination in the list.") @@ -86,7 +86,7 @@ class DatabaseActionsRunner(TestRunner): # Test list pagination. list_page = self.auth_client.databases.list(instance_id, limit=limit) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) self.assert_true(len(list_page) <= limit) if len(full_list) > limit: @@ -104,7 +104,8 @@ class DatabaseActionsRunner(TestRunner): "in the page.") list_page = self.auth_client.databases.list( instance_id, marker=marker) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, + client=self.auth_client) self.assert_pagination_match( list_page, full_list, limit, len(full_list)) @@ -163,7 +164,7 @@ class DatabaseActionsRunner(TestRunner): database_name, expected_http_code): self.auth_client.databases.delete(instance_id, database_name) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) self._wait_for_database_delete(instance_id, database_name) def _wait_for_database_delete(self, instance_id, deleted_database_name): diff --git a/trove/tests/scenario/runners/guest_log_runners.py b/trove/tests/scenario/runners/guest_log_runners.py index 87a9a35ee6..3c7d299edc 100644 --- a/trove/tests/scenario/runners/guest_log_runners.py +++ b/trove/tests/scenario/runners/guest_log_runners.py @@ -100,7 +100,7 @@ class GuestLogRunner(TestRunner): self.report.log("Executing log_show for log '%s'" % log_name) log_details = client.instances.log_show( self.instance_info.id, log_name) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=client) self.assert_log_details( log_details, log_name, expected_type=expected_type, @@ -183,7 +183,7 @@ class GuestLogRunner(TestRunner): self.report.log("Executing log_enable for log '%s'" % log_name) log_details = client.instances.log_enable( self.instance_info.id, log_name) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=client) self.assert_log_details( log_details, log_name, expected_type=expected_type, @@ -200,7 +200,7 @@ class GuestLogRunner(TestRunner): (log_name, discard)) log_details = client.instances.log_disable( self.instance_info.id, log_name, discard=discard) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=client) self.assert_log_details( log_details, log_name, expected_type=expected_type, @@ -218,7 +218,7 @@ class GuestLogRunner(TestRunner): (log_name, disable, discard)) log_details = client.instances.log_publish( self.instance_info.id, log_name, disable=disable, discard=discard) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=client) self.assert_log_details( log_details, log_name, expected_type=expected_type, @@ -234,7 +234,7 @@ class GuestLogRunner(TestRunner): self.report.log("Executing log_discard for log '%s'" % log_name) log_details = client.instances.log_discard( self.instance_info.id, log_name) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=client) self.assert_log_details( log_details, log_name, expected_type=expected_type, @@ -302,7 +302,7 @@ class GuestLogRunner(TestRunner): client.instances.log_enable, self.instance_info.id, log_name) # we may not be using the main client, so check explicitly here - self.assert_client_code(expected_http_code, client) + self.assert_client_code(expected_http_code, client=client) def run_test_log_disable_sys(self, expected_exception=exceptions.BadRequest, @@ -320,7 +320,7 @@ class GuestLogRunner(TestRunner): self.instance_info.id, log_name, discard=discard) # we may not be using the main client, so check explicitly here - self.assert_client_code(expected_http_code, client) + self.assert_client_code(expected_http_code, client=client) def run_test_log_show_unauth_user(self, expected_exception=exceptions.NotFound, @@ -337,7 +337,7 @@ class GuestLogRunner(TestRunner): client.instances.log_show, self.instance_info.id, log_name) # we may not be using the main client, so check explicitly here - self.assert_client_code(expected_http_code, client) + self.assert_client_code(expected_http_code, client=client) def run_test_log_list_unauth_user(self, expected_exception=exceptions.NotFound, @@ -346,7 +346,7 @@ class GuestLogRunner(TestRunner): self.unauth_client.instances.log_list, self.instance_info.id) # we're not using the main client, so check explicitly here - self.assert_client_code(expected_http_code, self.unauth_client) + self.assert_client_code(expected_http_code, client=self.unauth_client) def run_test_log_generator_unauth_user(self): self.assert_log_generator_unauth_user( @@ -406,7 +406,7 @@ class GuestLogRunner(TestRunner): self.instance_info.id, log_name, disable=disable, discard=discard) # we may not be using the main client, so check explicitly here - self.assert_client_code(expected_http_code, client) + self.assert_client_code(expected_http_code, client=client) def run_test_log_discard_unexposed_user( self, expected_exception=exceptions.BadRequest, @@ -423,7 +423,7 @@ class GuestLogRunner(TestRunner): client.instances.log_discard, self.instance_info.id, log_name) # we may not be using the main client, so check explicitly here - self.assert_client_code(expected_http_code, client) + self.assert_client_code(expected_http_code, client=client) def run_test_log_enable_user(self): expected_status = guest_log.LogStatus.Ready.name @@ -468,7 +468,8 @@ class GuestLogRunner(TestRunner): expected_states = ['RESTART_REQUIRED', 'ACTIVE'] self.assert_instance_action(instance_id, expected_states, None) self.auth_client.instances.restart(instance_id) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, + client=self.auth_client) def run_test_wait_for_restart(self, expected_states=['REBOOT', 'ACTIVE']): if self.test_helper.log_enable_requires_restart(): diff --git a/trove/tests/scenario/runners/instance_actions_runners.py b/trove/tests/scenario/runners/instance_actions_runners.py index 17f738a01d..6b18204650 100644 --- a/trove/tests/scenario/runners/instance_actions_runners.py +++ b/trove/tests/scenario/runners/instance_actions_runners.py @@ -88,7 +88,7 @@ class InstanceActionsRunner(TestRunner): (resize_flavor_id, instance_id)) self.auth_client.instances.resize_instance( instance_id, resize_flavor_id) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) def run_wait_for_instance_resize_flavor( self, expected_states=['RESIZE', 'ACTIVE']): diff --git a/trove/tests/scenario/runners/instance_create_runners.py b/trove/tests/scenario/runners/instance_create_runners.py index bfb3ad8e42..5eb6a8377c 100644 --- a/trove/tests/scenario/runners/instance_create_runners.py +++ b/trove/tests/scenario/runners/instance_create_runners.py @@ -73,7 +73,8 @@ class InstanceCreateRunner(TestRunner): "Configuration group used by instance create tests.", datastore=self.instance_info.dbaas_datastore, datastore_version=self.instance_info.dbaas_datastore_version) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, + client=self.auth_client) self.config_group_id = result.id else: @@ -309,7 +310,8 @@ class InstanceCreateRunner(TestRunner): def run_initialized_instance_delete(self, expected_http_code=202): if self.init_inst_id: self.auth_client.instances.delete(self.init_inst_id) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, + client=self.auth_client) else: raise SkipTest("Cleanup is not required.") @@ -331,7 +333,8 @@ class InstanceCreateRunner(TestRunner): def run_initial_configuration_delete(self, expected_http_code=202): if self.config_group_id: self.auth_client.configurations.delete(self.config_group_id) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, + client=self.auth_client) else: raise SkipTest("Cleanup is not required.") self.config_group_id = None diff --git a/trove/tests/scenario/runners/instance_delete_runners.py b/trove/tests/scenario/runners/instance_delete_runners.py index 09003db501..f5d1bb3e57 100644 --- a/trove/tests/scenario/runners/instance_delete_runners.py +++ b/trove/tests/scenario/runners/instance_delete_runners.py @@ -36,7 +36,7 @@ class InstanceDeleteRunner(TestRunner): self.report.log("Testing delete on instance: %s" % instance_id) self.auth_client.instances.delete(instance_id) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) def run_instance_delete_wait(self, expected_states=['SHUTDOWN']): if self.has_do_not_delete_instance: diff --git a/trove/tests/scenario/runners/instance_error_create_runners.py b/trove/tests/scenario/runners/instance_error_create_runners.py index 279763a0c1..c0f3c4df4e 100644 --- a/trove/tests/scenario/runners/instance_error_create_runners.py +++ b/trove/tests/scenario/runners/instance_error_create_runners.py @@ -40,7 +40,7 @@ class InstanceErrorCreateRunner(TestRunner): nics=self.instance_info.nics, datastore=self.instance_info.dbaas_datastore, datastore_version=self.instance_info.dbaas_datastore_version) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) self.error_inst_id = inst.id def run_create_error2_instance(self, expected_http_code=200): @@ -57,7 +57,7 @@ class InstanceErrorCreateRunner(TestRunner): nics=self.instance_info.nics, datastore=self.instance_info.dbaas_datastore, datastore_version=self.instance_info.dbaas_datastore_version) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) self.error2_inst_id = inst.id def run_wait_for_error_instances(self, expected_states=['ERROR']): @@ -75,7 +75,8 @@ class InstanceErrorCreateRunner(TestRunner): if not self.error_inst_id: raise SkipTest("No error instance created.") - instance = self.get_instance(self.error_inst_id) + instance = self.get_instance( + self.error_inst_id, self.auth_client) with CheckInstance(instance._info) as check: check.fault() @@ -101,10 +102,12 @@ class InstanceErrorCreateRunner(TestRunner): def run_delete_error_instances(self, expected_http_code=202): if self.error_inst_id: self.auth_client.instances.delete(self.error_inst_id) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, + client=self.auth_client) if self.error2_inst_id: self.auth_client.instances.delete(self.error2_inst_id) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, + client=self.auth_client) def run_wait_for_error_delete(self, expected_states=['SHUTDOWN']): delete_ids = [] diff --git a/trove/tests/scenario/runners/module_runners.py b/trove/tests/scenario/runners/module_runners.py index df9cb22f68..a7c7675638 100644 --- a/trove/tests/scenario/runners/module_runners.py +++ b/trove/tests/scenario/runners/module_runners.py @@ -386,8 +386,7 @@ class ModuleRunner(TestRunner): self.unauth_client.modules.get, self.main_test_module.id) # we're using a different client, so we'll check the return code # on it explicitly, instead of depending on 'assert_raises' - self.assert_client_code(expected_http_code=expected_http_code, - client=self.unauth_client) + self.assert_client_code(expected_http_code, client=self.unauth_client) def run_module_list(self): self.assert_module_list( @@ -686,7 +685,7 @@ class ModuleRunner(TestRunner): def assert_module_list_instance(self, client, instance_id, expected_count, expected_http_code=200): module_list = client.instances.modules(instance_id) - self.assert_client_code(expected_http_code, client) + self.assert_client_code(expected_http_code, client=client) count = len(module_list) self.assert_equal(expected_count, count, "Wrong number of modules from list instance") @@ -701,7 +700,7 @@ class ModuleRunner(TestRunner): def assert_module_instances(self, client, module_id, expected_count, expected_http_code=200): instance_list = client.modules.instances(module_id) - self.assert_client_code(expected_http_code, client) + self.assert_client_code(expected_http_code, client=client) count = len(instance_list) self.assert_equal(expected_count, count, "Wrong number of instances applied from module") @@ -713,7 +712,7 @@ class ModuleRunner(TestRunner): def assert_module_query(self, client, instance_id, expected_count, expected_http_code=200, expected_results=None): modquery_list = client.instances.module_query(instance_id) - self.assert_client_code(expected_http_code, client) + self.assert_client_code(expected_http_code, client=client) count = len(modquery_list) self.assert_equal(expected_count, count, "Wrong number of modules from query") @@ -736,7 +735,7 @@ class ModuleRunner(TestRunner): expected_http_code=200): module_apply_list = client.instances.module_apply( instance_id, [module.id]) - self.assert_client_code(expected_http_code, client) + self.assert_client_code(expected_http_code, client=client) admin_only = (not module.visible or module.auto_apply or not module.tenant_id) expected_status = expected_status or 'OK' @@ -856,7 +855,7 @@ class ModuleRunner(TestRunner): nics=self.instance_info.nics, modules=[module_id], ) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) return inst.id def run_module_delete_applied( @@ -873,7 +872,7 @@ class ModuleRunner(TestRunner): def assert_module_remove(self, client, instance_id, module_id, expected_http_code=200): client.instances.module_remove(instance_id, module_id) - self.assert_client_code(expected_http_code, client) + self.assert_client_code(expected_http_code, client=client) def run_wait_for_inst_with_mods(self, expected_states=['BUILD', 'ACTIVE']): self.assert_instance_action(self.mod_inst_id, expected_states, None) @@ -903,7 +902,7 @@ class ModuleRunner(TestRunner): prefix = 'contents' modretrieve_list = client.instances.module_retrieve( instance_id, directory=temp_dir, prefix=prefix) - self.assert_client_code(expected_http_code, client) + self.assert_client_code(expected_http_code, client=client) count = len(modretrieve_list) self.assert_equal(expected_count, count, "Wrong number of modules from retrieve") @@ -964,7 +963,7 @@ class ModuleRunner(TestRunner): def assert_delete_instance(self, instance_id, expected_http_code): self.auth_client.instances.delete(instance_id) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) def run_wait_for_delete_inst_with_mods( self, expected_last_state=['SHUTDOWN']): diff --git a/trove/tests/scenario/runners/replication_runners.py b/trove/tests/scenario/runners/replication_runners.py index 99e2143723..2d40232116 100644 --- a/trove/tests/scenario/runners/replication_runners.py +++ b/trove/tests/scenario/runners/replication_runners.py @@ -69,7 +69,7 @@ class ReplicationRunner(TestRunner): datastore=self.instance_info.dbaas_datastore, datastore_version=self.instance_info.dbaas_datastore_version, locality='anti-affinity').id - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) def run_create_single_replica(self, expected_http_code=200): self.master_backup_count = len( @@ -87,7 +87,7 @@ class ReplicationRunner(TestRunner): datastore_version=self.instance_info.dbaas_datastore_version, nics=self.instance_info.nics, replica_count=replica_count) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) return replica.id def run_wait_for_single_replica(self, expected_states=['BUILD', 'ACTIVE']): @@ -98,8 +98,8 @@ class ReplicationRunner(TestRunner): self.replica_1_host = self.get_instance_host(self.replica_1_id) def _assert_is_master(self, instance_id, replica_ids): - instance = self.get_instance(instance_id) - self.assert_client_code(200) + instance = self.get_instance(instance_id, client=self.admin_client) + self.assert_client_code(200, client=self.admin_client) CheckInstance(instance._info).slaves() self.assert_true( set(replica_ids).issubset(self._get_replica_set(instance_id))) @@ -110,8 +110,8 @@ class ReplicationRunner(TestRunner): return set([replica['id'] for replica in instance._info['replicas']]) def _assert_is_replica(self, instance_id, master_id): - instance = self.get_instance(instance_id) - self.assert_client_code(200) + instance = self.get_instance(instance_id, client=self.admin_client) + self.assert_client_code(200, client=self.admin_client) CheckInstance(instance._info).replica_of() self.assert_equal(master_id, instance._info['replica_of']['id'], 'Unexpected replication master ID') @@ -145,7 +145,7 @@ class ReplicationRunner(TestRunner): datastore_version=self.instance_info.dbaas_datastore_version, replica_of=self.non_affinity_master_id, replica_count=1).id - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) def run_create_multiple_replicas(self, expected_http_code=200): self.replica_2_id = self.assert_replica_create( @@ -176,7 +176,8 @@ class ReplicationRunner(TestRunner): else [instance_ids]) for instance_id in instance_ids: self.auth_client.instances.delete(instance_id) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, + client=self.auth_client) def run_wait_for_delete_non_affinity_repl( self, expected_last_status=['SHUTDOWN']): @@ -344,8 +345,8 @@ class ReplicationRunner(TestRunner): replica_id, expected_states, expected_http_code) def _assert_is_not_replica(self, instance_id): - instance = self.get_instance(instance_id) - self.assert_client_code(200) + instance = self.get_instance(instance_id, client=self.admin_client) + self.assert_client_code(200, client=self.admin_client) if 'replica_of' not in instance._info: try: diff --git a/trove/tests/scenario/runners/test_runners.py b/trove/tests/scenario/runners/test_runners.py index 1ef0f5aa69..ebf1ecb469 100644 --- a/trove/tests/scenario/runners/test_runners.py +++ b/trove/tests/scenario/runners/test_runners.py @@ -572,7 +572,7 @@ class TestRunner(object): def _has_status(self, instance_id, status, fast_fail_status=None): fast_fail_status = fast_fail_status or [] - instance = self.get_instance(instance_id) + instance = self.get_instance(instance_id, self.admin_client) self.report.log("Polling instance '%s' for state '%s', was '%s'." % (instance_id, status, instance.status)) if instance.status in fast_fail_status: diff --git a/trove/tests/scenario/runners/user_actions_runners.py b/trove/tests/scenario/runners/user_actions_runners.py index 30bd19b6cc..d822b1bb14 100644 --- a/trove/tests/scenario/runners/user_actions_runners.py +++ b/trove/tests/scenario/runners/user_actions_runners.py @@ -66,7 +66,7 @@ class UserActionsRunner(TestRunner): def assert_users_create(self, instance_id, serial_users_def, expected_http_code): self.auth_client.users.create(instance_id, serial_users_def) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) self.wait_for_user_create(instance_id, serial_users_def) return serial_users_def @@ -82,7 +82,7 @@ class UserActionsRunner(TestRunner): queried_user = self.auth_client.users.get( instance_id, user_name, user_host) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) self._assert_user_matches(queried_user, expected_user_def) def _assert_user_matches(self, user, expected_user_def): @@ -100,7 +100,7 @@ class UserActionsRunner(TestRunner): def assert_users_list(self, instance_id, expected_user_defs, expected_http_code, limit=2): full_list = self.auth_client.users.list(instance_id) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) listed_users = {user.name: user for user in full_list} self.assert_is_none(full_list.next, "Unexpected pagination in the list.") @@ -121,7 +121,7 @@ class UserActionsRunner(TestRunner): # Test list pagination. list_page = self.auth_client.users.list(instance_id, limit=limit) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) self.assert_true(len(list_page) <= limit) if len(full_list) > limit: @@ -138,7 +138,8 @@ class UserActionsRunner(TestRunner): "Pagination marker should be the last element " "in the page.") list_page = self.auth_client.users.list(instance_id, marker=marker) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, + client=self.auth_client) self.assert_pagination_match( list_page, full_list, limit, len(full_list)) @@ -155,7 +156,7 @@ class UserActionsRunner(TestRunner): user_name, user_host = self._get_user_name_host_pair(user_def) user_dbs = self.auth_client.users.list_access(instance_id, user_name, hostname=user_host) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) expected_dbs = {db_def['name'] for db_def in user_def['databases']} listed_dbs = [db.name for db in user_dbs] @@ -190,7 +191,7 @@ class UserActionsRunner(TestRunner): database, expected_http_code): self.auth_client.users.revoke( instance_id, user_name, database, hostname=user_host) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) user_dbs = self.auth_client.users.list_access( instance_id, user_name, hostname=user_host) self.assert_false(any(db.name == database for db in user_dbs), @@ -206,7 +207,7 @@ class UserActionsRunner(TestRunner): database, expected_http_code): self.auth_client.users.grant( instance_id, user_name, [database], hostname=user_host) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) user_dbs = self.auth_client.users.list_access( instance_id, user_name, hostname=user_host) self.assert_true(any(db.name == database for db in user_dbs), @@ -339,7 +340,7 @@ class UserActionsRunner(TestRunner): self.auth_client.users.update_attributes( instance_id, user_name, update_attribites, user_host) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) # Update the stored definitions with the new value. expected_def = None @@ -388,7 +389,7 @@ class UserActionsRunner(TestRunner): user_name, user_host = self._get_user_name_host_pair(user_def) self.auth_client.users.delete(instance_id, user_name, user_host) - self.assert_client_code(expected_http_code) + self.assert_client_code(expected_http_code, client=self.auth_client) self._wait_for_user_delete(instance_id, user_name) def _wait_for_user_delete(self, instance_id, deleted_user_name):