Merge "Fix mysqldump backup and restore"

trove/guestagent/backup/backupagent.py

@@ -18,7 +18,6 @@ import logging
 from trove.backup.models import BackupState
 from trove.common import cfg
 from trove.common import context as trove_context
-from trove.common import utils
 from trove.conductor import api as conductor_api
 from trove.guestagent.dbaas import get_filesystem_volume_stats
 from trove.guestagent.datastore.mysql.service import ADMIN_USER_NAME
@@ -111,11 +110,6 @@ class BackupAgent(object):
     def execute_restore(self, context, backup_info, restore_location):
 
         try:
-            LOG.debug("Cleaning out restore location: %s", restore_location)
-            utils.execute_with_timeout("sudo", "chmod", "-R",
-                                       "0777", restore_location)
-            utils.clean_out(restore_location)
-
             LOG.debug("Getting Restore Runner of type %s", backup_info['type'])
             restore_runner = self._get_restore_runner(backup_info['type'])
 
@@ -139,9 +133,6 @@ class BackupAgent(object):
                          backup_info['id'], restore_location)
                 LOG.info("Restore size: %s", content_size)
 
-                utils.execute_with_timeout("sudo", "chown", "-R",
-                                           "mysql", restore_location)
-
         except Exception as e:
             LOG.error(e)
             LOG.error("Error restoring backup %s", backup_info['id'])

trove/guestagent/strategies/backup/mysql_impl.py

@@ -28,12 +28,13 @@ class MySQLDump(base.BackupRunner):
 
     @property
     def cmd(self):
-        cmd = ('/usr/bin/mysqldump'
+        cmd = ('mysqldump'
                ' --all-databases'
                ' %(extra_opts)s'
                ' --opt'
                ' --password=%(password)s'
-               ' -u %(user)s')
+               ' -u %(user)s'
+               ' 2>/tmp/mysqldump.log')
         return cmd + self.zip_cmd + self.encrypt_cmd
 
     @property

trove/guestagent/strategies/restore/base.py

@@ -19,10 +19,6 @@ from trove.common import exception
 from trove.common import utils
 from trove.openstack.common import log as logging
 from eventlet.green import subprocess
-import tempfile
-import pexpect
-import os
-import glob
 
 LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
@@ -30,11 +26,6 @@ CHUNK_SIZE = CONF.backup_chunk_size
 BACKUP_USE_GZIP = CONF.backup_use_gzip_compression
 BACKUP_USE_OPENSSL = CONF.backup_use_openssl_encryption
 BACKUP_DECRYPT_KEY = CONF.backup_aes_cbc_key
-RESET_ROOT_RETRY_TIMEOUT = 100
-RESET_ROOT_SLEEP_INTERVAL = 10
-RESET_ROOT_MYSQL_COMMAND = """
-SET PASSWORD FOR 'root'@'localhost'=PASSWORD('');
-"""
 
 
 def exec_with_root_helper(*cmd):
@@ -46,33 +37,6 @@ def exec_with_root_helper(*cmd):
         return False
 
 
-def mysql_is_running():
-    if exec_with_root_helper("/usr/bin/mysqladmin", "ping"):
-        LOG.info("The mysqld daemon is up and running.")
-        return True
-    else:
-        LOG.info("The mysqld daemon is not running.")
-        return False
-
-
-def mysql_is_not_running():
-    if exec_with_root_helper("/usr/bin/pgrep", "mysqld"):
-        LOG.info("The mysqld daemon is still running.")
-        return False
-    else:
-        LOG.info("The mysqld daemon is not running.")
-        return True
-
-
-def poll_until_then_raise(event, exception):
-    try:
-        utils.poll_until(event,
-                         sleep_time=RESET_ROOT_SLEEP_INTERVAL,
-                         time_out=RESET_ROOT_RETRY_TIMEOUT)
-    except exception.PollTimeOut:
-        raise exception
-
-
 class RestoreError(Exception):
     """Error running the Backup Command."""
 
@@ -86,7 +50,6 @@ class RestoreRunner(Strategy):
 
     # The actual system calls to run the restore and prepare
     restore_cmd = None
-    prepare_cmd = None
 
     # The backup format type
     restore_type = None
@@ -103,8 +66,6 @@ class RestoreRunner(Strategy):
         self.restore_cmd = (self.decrypt_cmd +
                             self.unzip_cmd +
                             (self.base_restore_cmd % kwargs))
-        self.prepare_cmd = self.base_prepare_cmd % kwargs \
-            if hasattr(self, 'base_prepare_cmd') else None
         super(RestoreRunner, self).__init__()
 
     def __enter__(self):
@@ -126,11 +87,18 @@ class RestoreRunner(Strategy):
 
         return True
 
+    def pre_restore(self):
+        """Hook that is called before the restore command"""
+        pass
+
+    def post_restore(self):
+        """Hook that is called after the restore command"""
+        pass
+
     def restore(self):
-        self._pre_restore()
+        self.pre_restore()
         content_length = self._run_restore()
-        self._run_prepare()
+        self.post_restore()
-        self._post_restore()
         return content_length
 
     def _run_restore(self):
@@ -151,51 +119,6 @@ class RestoreRunner(Strategy):
 
         return content_length
 
-    def _run_prepare(self):
-        if hasattr(self, 'prepare_cmd'):
-            LOG.info("Running innobackupex prepare...")
-            self.prep_retcode = utils.execute(self.prepare_cmd,
-                                              shell=True)
-            LOG.info("Innobackupex prepare finished successfully")
-
-    def _spawn_with_init_file(self, temp_file):
-        child = pexpect.spawn("sudo mysqld_safe --init-file=%s" %
-                              temp_file.name)
-        try:
-            i = child.expect(['Starting mysqld daemon'])
-            if i == 0:
-                LOG.info("Starting mysqld daemon")
-        except pexpect.TIMEOUT as e:
-            LOG.error("wait_and_close_proc failed: %s" % e)
-        finally:
-            # There is a race condition here where we kill mysqld before
-            # the init file been executed. We need to ensure mysqld is up.
-            poll_until_then_raise(mysql_is_running,
-                                  RestoreError("Reset root password failed: "
-                                               "mysqld did not start!"))
-            LOG.info("Root password reset successfully!")
-            LOG.info("Cleaning up the temp mysqld process...")
-            child.delayafterclose = 1
-            child.delayafterterminate = 1
-            child.close(force=True)
-            utils.execute_with_timeout("sudo", "killall", "mysqld")
-            poll_until_then_raise(mysql_is_not_running,
-                                  RestoreError("Reset root password failed: "
-                                               "mysqld did not stop!"))
-
-    def _reset_root_password(self):
-        #Create temp file with reset root password
-        with tempfile.NamedTemporaryFile() as fp:
-            fp.write(RESET_ROOT_MYSQL_COMMAND)
-            fp.flush()
-            utils.execute_with_timeout("sudo", "chmod", "a+r", fp.name)
-            self._spawn_with_init_file(fp)
-
-    def _delete_old_binlogs(self):
-        filelist = glob.glob(self.restore_location + "/ib_logfile*")
-        for f in filelist:
-            os.unlink(f)
-
     @property
     def decrypt_cmd(self):
         if self.is_encrypted:

trove/guestagent/strategies/restore/mysql_impl.py

@@ -14,44 +14,129 @@
 #    under the License.
 #
 
+import glob
+import os
+import pexpect
+import tempfile
+
 from trove.guestagent.strategies.restore import base
 from trove.openstack.common import log as logging
+from trove.common import exception
 from trove.common import utils
 import trove.guestagent.datastore.mysql.service as dbaas
 
 LOG = logging.getLogger(__name__)
 
 
-class MySQLDump(base.RestoreRunner):
+class MySQLRestoreMixin(object):
-    """Implementation of Restore Strategy for MySQLDump """
+    """Common utils for restoring MySQL databases"""
+    RESET_ROOT_RETRY_TIMEOUT = 100
+    RESET_ROOT_SLEEP_INTERVAL = 10
+    RESET_ROOT_MYSQL_COMMAND = ("SET PASSWORD FOR"
+                                "'root'@'localhost'=PASSWORD('');")
+
+    def mysql_is_running(self):
+        if base.exec_with_root_helper("/usr/bin/mysqladmin", "ping"):
+            LOG.info("The mysqld daemon is up and running.")
+            return True
+        else:
+            LOG.info("The mysqld daemon is not running.")
+            return False
+
+    def mysql_is_not_running(self):
+        if base.exec_with_root_helper("/usr/bin/pgrep", "mysqld"):
+            LOG.info("The mysqld daemon is still running.")
+            return False
+        else:
+            LOG.info("The mysqld daemon is not running.")
+            return True
+
+    def poll_until_then_raise(self, event, exc):
+        try:
+            utils.poll_until(event,
+                             sleep_time=self.RESET_ROOT_SLEEP_INTERVAL,
+                             time_out=self.RESET_ROOT_RETRY_TIMEOUT)
+        except exception.PollTimeOut:
+            raise exc
+
+    def _spawn_with_init_file(self, temp_file):
+        child = pexpect.spawn("sudo mysqld_safe --init-file=%s" %
+                              temp_file.name)
+        try:
+            i = child.expect(['Starting mysqld daemon'])
+            if i == 0:
+                LOG.info("Starting mysqld daemon")
+        except pexpect.TIMEOUT as e:
+            LOG.error("wait_and_close_proc failed: %s" % e)
+        finally:
+            # There is a race condition here where we kill mysqld before
+            # the init file been executed. We need to ensure mysqld is up.
+            self.poll_until_then_raise(
+                self.mysql_is_running,
+                base.RestoreError("Reset root password failed: "
+                                  "mysqld did not start!"))
+            LOG.info("Root password reset successfully!")
+            LOG.info("Cleaning up the temp mysqld process...")
+            child.delayafterclose = 1
+            child.delayafterterminate = 1
+            child.close(force=True)
+            utils.execute_with_timeout("sudo", "killall", "mysqld")
+            self.poll_until_then_raise(
+                self.mysql_is_not_running,
+                base.RestoreError("Reset root password failed: "
+                                  "mysqld did not stop!"))
+
+    def reset_root_password(self):
+        #Create temp file with reset root password
+        with tempfile.NamedTemporaryFile() as fp:
+            fp.write(self.RESET_ROOT_MYSQL_COMMAND)
+            fp.flush()
+            utils.execute_with_timeout("sudo", "chmod", "a+r", fp.name)
+            self._spawn_with_init_file(fp)
+
+
+class MySQLDump(base.RestoreRunner, MySQLRestoreMixin):
+    """Implementation of Restore Strategy for MySQLDump"""
     __strategy_name__ = 'mysqldump'
-    base_restore_cmd = ('mysql '
+    base_restore_cmd = 'sudo mysql'
-                        '--password=%(password)s '
-                        '-u %(user)s')
-
-    def _pre_restore(self):
-        pass
-
-    def _post_restore(self):
-        pass
 
 
-class InnoBackupEx(base.RestoreRunner):
+class InnoBackupEx(base.RestoreRunner, MySQLRestoreMixin):
-    """Implementation of Restore Strategy for InnoBackupEx """
+    """Implementation of Restore Strategy for InnoBackupEx"""
     __strategy_name__ = 'innobackupex'
     base_restore_cmd = 'sudo xbstream -x -C %(restore_location)s'
     base_prepare_cmd = ('sudo innobackupex --apply-log %(restore_location)s'
                         ' --defaults-file=%(restore_location)s/backup-my.cnf'
                         ' --ibbackup xtrabackup 2>/tmp/innoprepare.log')
 
-    def _pre_restore(self):
+    def __init__(self, *args, **kwargs):
+        super(InnoBackupEx, self).__init__(*args, **kwargs)
+        self.prepare_cmd = self.base_prepare_cmd % kwargs
+        self.prep_retcode = None
+
+    def pre_restore(self):
         app = dbaas.MySqlApp(dbaas.MySqlAppStatus.get())
         app.stop_db()
+        LOG.info("Cleaning out restore location: %s", self.restore_location)
+        utils.execute_with_timeout("sudo", "chmod", "-R",
+                                   "0777", self.restore_location)
+        utils.clean_out(self.restore_location)
 
-    def _post_restore(self):
+    def _run_prepare(self):
+        LOG.info("Running innobackupex prepare: %s", self.prepare_cmd)
+        self.prep_retcode = utils.execute(self.prepare_cmd, shell=True)
+        LOG.info("Innobackupex prepare finished successfully")
+
+    def post_restore(self):
+        self._run_prepare()
         utils.execute_with_timeout("sudo", "chown", "-R", "-f",
                                    "mysql", self.restore_location)
         self._delete_old_binlogs()
-        self._reset_root_password()
+        self.reset_root_password()
         app = dbaas.MySqlApp(dbaas.MySqlAppStatus.get())
         app.start_mysql()
+
+    def _delete_old_binlogs(self):
+        files = glob.glob(os.path.join(self.restore_location, "ib_logfile*"))
+        for f in files:
+            os.unlink(f)

trove/tests/unittests/backup/test_backupagent.py

@@ -161,12 +161,13 @@ class BackupAgentTest(testtools.TestCase):
                                           user='123',
                                           extra_opts='')
         self.assertIsNotNone(mysql_dump.cmd)
-        str_mysql_dump_cmd = ('/usr/bin/mysqldump'
+        str_mysql_dump_cmd = ('mysqldump'
                               ' --all-databases'
                               ' %(extra_opts)s'
                               ' --opt'
                               ' --password=%(password)s'
                               ' -u %(user)s'
+                              ' 2>/tmp/mysqldump.log'
                               ' | gzip |'
                               ' openssl enc -aes-256-cbc -salt '
                               '-pass pass:default_aes_cbc_key')

trove/tests/unittests/guestagent/test_backups.py

@@ -34,13 +34,14 @@ XTRA_BACKUP_RAW = ("sudo innobackupex --stream=xbstream %(extra_opts)s"
                    " /var/lib/mysql 2>/tmp/innobackupex.log")
 XTRA_BACKUP = XTRA_BACKUP_RAW % {'extra_opts': ''}
 XTRA_BACKUP_EXTRA_OPTS = XTRA_BACKUP_RAW % {'extra_opts': '--no-lock'}
-SQLDUMP_BACKUP_RAW = ("/usr/bin/mysqldump --all-databases %(extra_opts)s "
+SQLDUMP_BACKUP_RAW = ("mysqldump --all-databases %(extra_opts)s "
-                      "--opt --password=password -u user")
+                      "--opt --password=password -u user"
+                      " 2>/tmp/mysqldump.log")
 SQLDUMP_BACKUP = SQLDUMP_BACKUP_RAW % {'extra_opts': ''}
 SQLDUMP_BACKUP_EXTRA_OPTS = (SQLDUMP_BACKUP_RAW %
                              {'extra_opts': '--events --routines --triggers'})
 XTRA_RESTORE = "sudo xbstream -x -C /var/lib/mysql"
-SQLDUMP_RESTORE = "mysql --password=password -u user"
+SQLDUMP_RESTORE = "sudo mysql"
 PREPARE = ("sudo innobackupex --apply-log /var/lib/mysql "
            "--defaults-file=/var/lib/mysql/backup-my.cnf "
            "--ibbackup xtrabackup 2>/tmp/innoprepare.log")
@@ -132,7 +133,6 @@ class GuestAgentBackupTest(testtools.TestCase):
         restr = RunnerClass(None, restore_location="/var/lib/mysql",
                             user="user", password="password")
         self.assertEqual(restr.restore_cmd, UNZIP + PIPE + SQLDUMP_RESTORE)
-        self.assertIsNone(restr.prepare_cmd)
 
     def test_restore_encrypted_mysqldump_command(self):
         restoreBase.RestoreRunner.is_zipped = True
@@ -143,4 +143,3 @@ class GuestAgentBackupTest(testtools.TestCase):
                             user="user", password="password")
         self.assertEqual(restr.restore_cmd,
                          DECRYPT + PIPE + UNZIP + PIPE + SQLDUMP_RESTORE)
-        self.assertIsNone(restr.prepare_cmd)