trove/reddwarf/tests/api/instances_actions.py

# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import time

from proboscis import after_class
from proboscis import before_class
from proboscis import test
from proboscis.asserts import assert_equal
from proboscis.asserts import assert_false
from proboscis.asserts import assert_not_equal
from proboscis.asserts import assert_raises
from proboscis.asserts import assert_true
from proboscis.asserts import fail
from proboscis.decorators import time_out
from proboscis import SkipTest

from reddwarf import tests
from reddwarf.tests.util.check import Checker
from reddwarfclient.exceptions import BadRequest
from reddwarfclient.exceptions import UnprocessableEntity
from reddwarf.tests.api.instances import GROUP as INSTANCE_GROUP
from reddwarf.tests.api.instances import GROUP_START
from reddwarf.tests.api.instances import instance_info
from reddwarf.tests.api.instances import assert_unprocessable
from reddwarf.tests import util
from reddwarf.tests.util import poll_until
from reddwarf.tests.config import CONFIG
from reddwarf.tests.util import LocalSqlClient
from sqlalchemy import create_engine
from sqlalchemy import exc as sqlalchemy_exc
from reddwarf.tests.util.check import TypeCheck
from sqlalchemy.sql.expression import text

GROUP = "dbaas.api.instances.actions"
GROUP_REBOOT = "dbaas.api.instances.actions.reboot"
GROUP_RESTART = "dbaas.api.instances.actions.restart"
GROUP_STOP_MYSQL = "dbaas.api.instances.actions.stop"
MYSQL_USERNAME = "test_user"
MYSQL_PASSWORD = "abcde"
FAKE_MODE = CONFIG.values['fake_mode']
# If true, then we will actually log into the database.
USE_IP = not CONFIG.values['fake_mode']
# If true, then we will actually search for the process
USE_LOCAL_OVZ = CONFIG.values['use_local_ovz']


class MySqlConnection(object):

    def __init__(self, host):
        self.host = host

    def connect(self):
        """Connect to MySQL database."""
        print("Connecting to MySQL, mysql --host %s -u %s -p%s"
              % (self.host, MYSQL_USERNAME, MYSQL_PASSWORD))
        sql_engine = LocalSqlClient.init_engine(MYSQL_USERNAME, MYSQL_PASSWORD,
                                                self.host)
        self.client = LocalSqlClient(sql_engine, use_flush=False)

    def is_connected(self):
        try:
            with self.client:
                self.client.execute(text("""SELECT "Hello.";"""))
            return True
        except (sqlalchemy_exc.OperationalError,
                sqlalchemy_exc.DisconnectionError,
                sqlalchemy_exc.TimeoutError):
            return False
        except Exception as ex:
            print("EX WAS:")
            print(type(ex))
            print(ex)
            raise ex


TIME_OUT_TIME = 10 * 60
USER_WAS_DELETED = False


class ActionTestBase(object):
    """Has some helpful functions for testing actions.

    The test user must be created for some of these functions to work.

    """

    def set_up(self):
        """If you're using this as a base class, call this method first."""
        if USE_IP:
            address = instance_info.get_address()
            self.connection = MySqlConnection(address)
        self.dbaas = instance_info.dbaas

    @property
    def instance(self):
        return self.dbaas.instances.get(self.instance_id)

    @property
    def instance_address(self):
        return instance_info.get_address()

    @property
    def instance_id(self):
        return instance_info.id

    def create_user(self):
        """Create a MySQL user we can use for this test."""

        users = [{"name": MYSQL_USERNAME, "password": MYSQL_PASSWORD,
                  "database": MYSQL_USERNAME}]
        self.dbaas.users.create(instance_info.id, users)

        def has_user():
            users = self.dbaas.users.list(instance_info.id)
            return any([user.name == MYSQL_USERNAME for user in users])

        poll_until(has_user, time_out=30)
        if not FAKE_MODE:
            time.sleep(5)

    def ensure_mysql_is_running(self):
        """Make sure MySQL is accessible before restarting."""
        with Checker() as check:
            if USE_IP:
                self.connection.connect()
                check.true(self.connection.is_connected(),
                           "Able to connect to MySQL.")
            if USE_LOCAL_OVZ:
                self.proc_id = self.find_mysql_proc_on_instance()
                check.true(self.proc_id is not None,
                           "MySQL process can be found.")
            instance = self.instance
            check.false(instance is None)
            check.equal(instance.status, "ACTIVE")

    def find_mysql_proc_on_instance(self):
        return util.find_mysql_procid_on_instance(self.instance_address)

    def log_current_users(self):
        users = self.dbaas.users.list(self.instance_id)
        CONFIG.get_report().log("Current user count = %d" % len(users))
        for user in users:
            CONFIG.get_report().log("\t" + str(user))


@test(depends_on_groups=[GROUP_START])
def create_user():
    """Create a test user so that subsequent tests can log in."""
    helper = ActionTestBase()
    helper.set_up()
    if USE_IP:
        try:
            helper.create_user()
        except BadRequest:
            pass  # Ignore this if the user already exists.
        helper.connection.connect()
        assert_true(helper.connection.is_connected(),
                    "Test user must be able to connect to MySQL.")


class RebootTestBase(ActionTestBase):
    """Tests restarting MySQL."""

    def call_reboot(self):
        raise NotImplementedError()

    def wait_for_broken_connection(self):
        """Wait until our connection breaks."""
        if not USE_IP:
            return
        poll_until(self.connection.is_connected,
                   lambda connected: not connected,
                   time_out=TIME_OUT_TIME)

    def wait_for_successful_restart(self):
        """Wait until status becomes running."""
        def is_finished_rebooting():
            instance = self.instance
            instance.get()
            print(instance.status)
            if instance.status == "REBOOT":
                return False
            assert_equal("ACTIVE", instance.status)
            return True

        poll_until(is_finished_rebooting, time_out=TIME_OUT_TIME)

    def assert_mysql_proc_is_different(self):
        if not USE_LOCAL_OVZ:
            return
        new_proc_id = self.find_mysql_proc_on_instance()
        assert_not_equal(new_proc_id, self.proc_id,
                         "MySQL process ID should be different!")

    def successful_restart(self):
        """Restart MySQL via the REST API successfully."""
        self.fix_mysql()
        self.call_reboot()
        self.wait_for_broken_connection()
        self.wait_for_successful_restart()
        self.assert_mysql_proc_is_different()

    def mess_up_mysql(self):
        """Ruin MySQL's ability to restart."""
        self.fix_mysql()  # kill files
        cmd = """%s %s 'sudo cp /dev/null /var/lib/mysql/ib_logfile%d'"""
        for index in range(2):
            full_cmd = cmd % (tests.SSH_CMD, self.instance_address, index)
            print("RUNNING COMMAND: %s" % full_cmd)
            util.process(full_cmd)

    def fix_mysql(self):
        """Fix MySQL's ability to restart."""
        if not FAKE_MODE:
            cmd = "%s %s 'sudo rm /var/lib/mysql/ib_logfile%d'"
            for index in range(2):
                util.process(cmd % (tests.SSH_CMD,
                                    self.instance_address,
                                    index))

    def wait_for_failure_status(self):
        """Wait until status becomes running."""
        def is_finished_rebooting():
            instance = self.instance
            if instance.status == "REBOOT" or instance.status == "ACTIVE":
                return False
            assert_equal("SHUTDOWN", instance.status)
            return True

        poll_until(is_finished_rebooting, time_out=TIME_OUT_TIME)

    def unsuccessful_restart(self):
        """Restart MySQL via the REST when it should fail, assert it does."""
        assert not FAKE_MODE
        self.mess_up_mysql()
        self.call_reboot()
        self.wait_for_broken_connection()
        self.wait_for_failure_status()

    def restart_normally(self):
        """Fix iblogs and reboot normally."""
        self.fix_mysql()
        self.test_successful_restart()


@test(groups=[tests.INSTANCES, INSTANCE_GROUP, GROUP, GROUP_RESTART],
      depends_on_groups=[GROUP_START], depends_on=[create_user])
class RestartTests(RebootTestBase):
    """Tests restarting MySQL."""

    def call_reboot(self):
        self.instance.restart()
        assert_equal(202, self.dbaas.last_http_code)

    @before_class
    def test_set_up(self):
        self.set_up()

    @test
    def test_ensure_mysql_is_running(self):
        """Make sure MySQL is accessible before restarting."""
        self.ensure_mysql_is_running()

    @test(depends_on=[test_ensure_mysql_is_running], enabled=not FAKE_MODE)
    def test_unsuccessful_restart(self):
        """Restart MySQL via the REST when it should fail, assert it does."""
        self.unsuccessful_restart()

    @test(depends_on=[test_set_up],
          runs_after=[test_ensure_mysql_is_running, test_unsuccessful_restart])
    def test_successful_restart(self):
        """Restart MySQL via the REST API successfully."""
        self.successful_restart()


@test(groups=[tests.INSTANCES, INSTANCE_GROUP, GROUP, GROUP_STOP_MYSQL],
      depends_on_groups=[GROUP_START], depends_on=[create_user])
class StopTests(RebootTestBase):
    """Tests which involve stopping MySQL."""

    def call_reboot(self):
        self.instance.restart()

    @before_class
    def test_set_up(self):
        self.set_up()

    @test
    def test_ensure_mysql_is_running(self):
        """Make sure MySQL is accessible before restarting."""
        self.ensure_mysql_is_running()

    @test(depends_on=[test_ensure_mysql_is_running])
    def test_stop_mysql(self):
        """Stops MySQL."""
        instance_info.dbaas_admin.management.stop(self.instance_id)
        self.wait_for_broken_connection()
        self.wait_for_failure_status()

    @test(depends_on=[test_stop_mysql])
    def test_instance_get_shows_volume_info_while_mysql_is_down(self):
        """
        Confirms the get call behaves appropriately while an instance is
        down.
        """
        if not CONFIG.reddwarf_main_instance_has_volume:
            raise SkipTest("Not testing volumes.")
        instance = self.dbaas.instances.get(self.instance_id)
        with TypeCheck("instance", instance) as check:
            check.has_field("volume", dict)
            check.true('size' in instance.volume)
            check.true('used' in instance.volume)
            check.true(isinstance(instance.volume.get('size', None), int))
            check.true(isinstance(instance.volume.get('used', None), float))

    @test(depends_on=[test_set_up],
          runs_after=[test_instance_get_shows_volume_info_while_mysql_is_down])
    def test_successful_restart_when_in_shutdown_state(self):
        """Restart MySQL via the REST API successfully when MySQL is down."""
        self.successful_restart()


@test(groups=[tests.INSTANCES, INSTANCE_GROUP, GROUP, GROUP_REBOOT],
      depends_on_groups=[GROUP_START], depends_on=[RestartTests, create_user])
class RebootTests(RebootTestBase):
    """Tests restarting instance."""

    def call_reboot(self):
        instance_info.dbaas_admin.management.reboot(self.instance_id)

    @before_class
    def test_set_up(self):
        self.set_up()

    @test
    def test_ensure_mysql_is_running(self):
        """Make sure MySQL is accessible before restarting."""
        self.ensure_mysql_is_running()

    @test(depends_on=[test_ensure_mysql_is_running], enabled=not FAKE_MODE)
    def test_unsuccessful_restart(self):
        """Restart MySQL via the REST when it should fail, assert it does."""
        self.unsuccessful_restart()

    @after_class(always_run=True)
    def test_successful_restart(self):
        """Restart MySQL via the REST API successfully."""
        self.successful_restart()


@test(groups=[tests.INSTANCES, INSTANCE_GROUP, GROUP,
              GROUP + ".resize.instance"],
      depends_on_groups=[GROUP_START], depends_on=[create_user],
      runs_after=[RebootTests])
class ResizeInstanceTest(ActionTestBase):

    """
    Integration Test cases for resize instance
    """
    @property
    def flavor_id(self):
        return instance_info.dbaas_flavor_href

    def get_flavor_href(self, flavor_id=2):
        res = instance_info.dbaas.find_flavor_and_self_href(flavor_id)
        dbaas_flavor, dbaas_flavor_href = res
        return dbaas_flavor_href

    def wait_for_resize(self):
        def is_finished_resizing():
            instance = self.instance
            if instance.status == "RESIZE":
                return False
            assert_equal("ACTIVE", instance.status)
            return True
        poll_until(is_finished_resizing, time_out=TIME_OUT_TIME)

    @before_class
    def setup(self):
        self.set_up()
        if USE_IP:
            self.connection.connect()
            assert_true(self.connection.is_connected(),
                        "Should be able to connect before resize.")
        self.user_was_deleted = False

    @test
    def test_instance_resize_same_size_should_fail(self):
        assert_raises(BadRequest, self.dbaas.instances.resize_instance,
                      self.instance_id, self.flavor_id)

    def obtain_flavor_ids(self):
        old_id = self.instance.flavor['id']
        self.expected_old_flavor_id = old_id
        res = instance_info.dbaas.find_flavor_and_self_href(old_id)
        self.expected_dbaas_flavor, _dontcare_ = res

        flavor_name = CONFIG.values.get('instance_bigger_flavor_name',
                                        'm1.small')
        flavors = self.dbaas.find_flavors_by_name(flavor_name)
        assert_equal(len(flavors), 1, "Number of flavors with name '%s' "
                     "found was '%d'." % (flavor_name, len(flavors)))
        flavor = flavors[0]
        assert_true(flavor is not None, "Flavor '%s' not found!" % flavor_name)
        flavor_href = self.dbaas.find_flavor_self_href(flavor)
        assert_true(flavor_href is not None,
                    "Flavor href '%s' not found!" % flavor_name)
        self.expected_new_flavor_id = flavor.id

    @test(depends_on=[test_instance_resize_same_size_should_fail])
    def test_status_changed_to_resize(self):
        self.log_current_users()
        self.obtain_flavor_ids()
        self.dbaas.instances.resize_instance(
            self.instance_id,
            self.get_flavor_href(flavor_id=self.expected_new_flavor_id))
        assert_equal(202, self.dbaas.last_http_code)

        #(WARNING) IF THE RESIZE IS WAY TOO FAST THIS WILL FAIL
        assert_unprocessable(
            self.dbaas.instances.resize_instance,
            self.instance_id,
            self.get_flavor_href(flavor_id=self.expected_new_flavor_id))

    @test(depends_on=[test_status_changed_to_resize])
    @time_out(TIME_OUT_TIME)
    def test_instance_returns_to_active_after_resize(self):
        self.wait_for_resize()

    @test(depends_on=[test_instance_returns_to_active_after_resize])
    def resize_should_not_delete_users(self):
        """Resize should not delete users."""
        # Resize has an incredibly weird bug where users are deleted after
        # a resize. The code below is an attempt to catch this while proceeding
        # with the rest of the test (note the use of runs_after).
        if USE_IP:
            self.connection.connect()
            if not self.connection.is_connected():
                # Ok, this is def. a failure, but before we toss up an error
                # lets recreate to see how far we can get.
                CONFIG.get_report().log(
                    "Having to recreate the test_user! Resizing killed it!")
                self.log_current_users()
                self.create_user()
                fail("Somehow, the resize made the test user disappear.")

    @test(depends_on=[test_instance_returns_to_active_after_resize],
          runs_after=[resize_should_not_delete_users])
    def test_make_sure_mysql_is_running_after_resize(self):
        self.ensure_mysql_is_running()
        actual = self.get_flavor_href(self.instance.flavor['id'])
        expected = self.get_flavor_href(flavor_id=self.expected_new_flavor_id)
        assert_equal(actual, expected)

    @test(depends_on=[test_make_sure_mysql_is_running_after_resize])
    @time_out(TIME_OUT_TIME)
    def test_resize_down(self):
        expected_dbaas_flavor = self.expected_dbaas_flavor
        self.dbaas.instances.resize_instance(
            self.instance_id,
            self.get_flavor_href(flavor_id=self.expected_old_flavor_id))
        assert_equal(202, self.dbaas.last_http_code)
        self.old_dbaas_flavor = instance_info.dbaas_flavor
        instance_info.dbaas_flavor = expected_dbaas_flavor
        self.wait_for_resize()
        assert_equal(str(self.instance.flavor['id']),
                     str(self.expected_old_flavor_id))


@test(groups=[tests.INSTANCES, INSTANCE_GROUP, GROUP,
              GROUP + ".resize.instance"],
      depends_on_groups=[GROUP_START], depends_on=[create_user],
      runs_after=[RebootTests, ResizeInstanceTest])
def resize_should_not_delete_users():
    if USER_WAS_DELETED:
        fail("Somehow, the resize made the test user disappear.")


@test(depends_on_classes=[ResizeInstanceTest], depends_on=[create_user],
      groups=[GROUP, tests.INSTANCES],
      enabled=CONFIG.reddwarf_main_instance_has_volume)
class ResizeInstanceVolume(object):
    """ Resize the volume of the instance """

    @before_class
    def setUp(self):
        self.old_volume_size = int(instance_info.volume['size'])
        self.new_volume_size = self.old_volume_size + 1

        # Create some databases to check they still exist after the resize
        self.expected_dbs = ['salmon', 'halibut']
        databases = []
        for name in self.expected_dbs:
            databases.append({"name": name})
        instance_info.dbaas.databases.create(instance_info.id, databases)

    @test
    @time_out(60)
    def test_volume_resize(self):
        instance_info.dbaas.instances.resize_volume(instance_info.id,
                                                    self.new_volume_size)

    @test
    @time_out(300)
    def test_volume_resize_success(self):

        def check_resize_status():
            instance = instance_info.dbaas.instances.get(instance_info.id)
            if instance.status == "ACTIVE":
                return True
            elif instance.status == "RESIZE":
                return False
            else:
                fail("Status should not be %s" % instance.status)

        poll_until(check_resize_status, sleep_time=2, time_out=300)
        instance = instance_info.dbaas.instances.get(instance_info.id)
        assert_equal(instance.volume['size'], self.new_volume_size)

    @test
    @time_out(300)
    def test_volume_resize_success_databases(self):
        databases = instance_info.dbaas.databases.list(instance_info.id)
        db_list = []
        for database in databases:
            db_list.append(database.name)
        for name in self.expected_dbs:
            if not name in db_list:
                fail("Database %s was not found after the volume resize. "
                     "Returned list: %s" % (name, databases))


# This tests the ability of the guest to upgrade itself.
# It is necessarily tricky because we need to be able to upload a new copy of
# the guest into an apt-repo in the middle of the test.
# "guest-update-test" is where the knowledge of how to do this is set in the
# test conf. If it is not specified this test never runs.
UPDATE_GUEST_CONF = CONFIG.values.get("guest-update-test", None)


@test(groups=[tests.INSTANCES, INSTANCE_GROUP, GROUP, GROUP + ".update_guest"],
      depends_on=[create_user],
      depends_on_groups=[GROUP_START])
class UpdateGuest(object):

    def get_version(self):
        info = instance_info.dbaas_admin.diagnostics.get(instance_info.id)
        return info.version

    @before_class(enabled=UPDATE_GUEST_CONF is not None)
    def check_version_is_old(self):
        """Make sure we have the old version before proceeding."""
        self.old_version = self.get_version()
        self.next_version = UPDATE_GUEST_CONF["next-version"]
        assert_not_equal(self.old_version, self.next_version)

    @test(enabled=UPDATE_GUEST_CONF is not None)
    def upload_update_to_repo(self):
        cmds = UPDATE_GUEST_CONF["install-repo-cmd"]
        utils.execute(*cmds, run_as_root=True)

    @test(enabled=UPDATE_GUEST_CONF is not None,
          depends_on=[upload_update_to_repo])
    def update_and_wait_to_finish(self):
        instance_info.dbaas_admin.management.update(instance_info.id)

        def finished():
            current_version = self.get_version()
            if current_version == self.next_version:
                return True
            # The only valid thing for it to be aside from next_version is
            # old version.
            assert_equal(current_version, self.old_version)
        poll_until(finished, sleep_time=1, time_out=3 * 60)

    @test(enabled=UPDATE_GUEST_CONF is not None,
          depends_on=[upload_update_to_repo])
    @time_out(30)
    def update_again(self):
        """Test the wait time of a pointless update."""
        instance_info.dbaas_admin.management.update(instance_info.id)
        # Make sure this isn't taking too long.
        instance_info.dbaas_admin.diagnostics.get(instance_info.id)