diff --git a/trove/guestagent/backup/backupagent.py b/trove/guestagent/backup/backupagent.py index e749ed63cc..2813c31bc7 100644 --- a/trove/guestagent/backup/backupagent.py +++ b/trove/guestagent/backup/backupagent.py @@ -18,7 +18,6 @@ import logging from trove.backup.models import BackupState from trove.common import cfg from trove.common import context as trove_context -from trove.common import utils from trove.conductor import api as conductor_api from trove.guestagent.dbaas import get_filesystem_volume_stats from trove.guestagent.datastore.mysql.service import ADMIN_USER_NAME @@ -111,11 +110,6 @@ class BackupAgent(object): def execute_restore(self, context, backup_info, restore_location): try: - LOG.debug("Cleaning out restore location: %s", restore_location) - utils.execute_with_timeout("sudo", "chmod", "-R", - "0777", restore_location) - utils.clean_out(restore_location) - LOG.debug("Getting Restore Runner of type %s", backup_info['type']) restore_runner = self._get_restore_runner(backup_info['type']) @@ -139,9 +133,6 @@ class BackupAgent(object): backup_info['id'], restore_location) LOG.info("Restore size: %s", content_size) - utils.execute_with_timeout("sudo", "chown", "-R", - "mysql", restore_location) - except Exception as e: LOG.error(e) LOG.error("Error restoring backup %s", backup_info['id']) diff --git a/trove/guestagent/strategies/backup/mysql_impl.py b/trove/guestagent/strategies/backup/mysql_impl.py index 21a5d39073..37d1616a93 100644 --- a/trove/guestagent/strategies/backup/mysql_impl.py +++ b/trove/guestagent/strategies/backup/mysql_impl.py @@ -28,12 +28,13 @@ class MySQLDump(base.BackupRunner): @property def cmd(self): - cmd = ('/usr/bin/mysqldump' + cmd = ('mysqldump' ' --all-databases' ' %(extra_opts)s' ' --opt' ' --password=%(password)s' - ' -u %(user)s') + ' -u %(user)s' + ' 2>/tmp/mysqldump.log') return cmd + self.zip_cmd + self.encrypt_cmd @property diff --git a/trove/guestagent/strategies/restore/base.py b/trove/guestagent/strategies/restore/base.py index e3997824bc..782517dc4e 100644 --- a/trove/guestagent/strategies/restore/base.py +++ b/trove/guestagent/strategies/restore/base.py @@ -19,10 +19,6 @@ from trove.common import exception from trove.common import utils from trove.openstack.common import log as logging from eventlet.green import subprocess -import tempfile -import pexpect -import os -import glob LOG = logging.getLogger(__name__) CONF = cfg.CONF @@ -30,11 +26,6 @@ CHUNK_SIZE = CONF.backup_chunk_size BACKUP_USE_GZIP = CONF.backup_use_gzip_compression BACKUP_USE_OPENSSL = CONF.backup_use_openssl_encryption BACKUP_DECRYPT_KEY = CONF.backup_aes_cbc_key -RESET_ROOT_RETRY_TIMEOUT = 100 -RESET_ROOT_SLEEP_INTERVAL = 10 -RESET_ROOT_MYSQL_COMMAND = """ -SET PASSWORD FOR 'root'@'localhost'=PASSWORD(''); -""" def exec_with_root_helper(*cmd): @@ -46,33 +37,6 @@ def exec_with_root_helper(*cmd): return False -def mysql_is_running(): - if exec_with_root_helper("/usr/bin/mysqladmin", "ping"): - LOG.info("The mysqld daemon is up and running.") - return True - else: - LOG.info("The mysqld daemon is not running.") - return False - - -def mysql_is_not_running(): - if exec_with_root_helper("/usr/bin/pgrep", "mysqld"): - LOG.info("The mysqld daemon is still running.") - return False - else: - LOG.info("The mysqld daemon is not running.") - return True - - -def poll_until_then_raise(event, exception): - try: - utils.poll_until(event, - sleep_time=RESET_ROOT_SLEEP_INTERVAL, - time_out=RESET_ROOT_RETRY_TIMEOUT) - except exception.PollTimeOut: - raise exception - - class RestoreError(Exception): """Error running the Backup Command.""" @@ -86,7 +50,6 @@ class RestoreRunner(Strategy): # The actual system calls to run the restore and prepare restore_cmd = None - prepare_cmd = None # The backup format type restore_type = None @@ -103,8 +66,6 @@ class RestoreRunner(Strategy): self.restore_cmd = (self.decrypt_cmd + self.unzip_cmd + (self.base_restore_cmd % kwargs)) - self.prepare_cmd = self.base_prepare_cmd % kwargs \ - if hasattr(self, 'base_prepare_cmd') else None super(RestoreRunner, self).__init__() def __enter__(self): @@ -126,11 +87,18 @@ class RestoreRunner(Strategy): return True + def pre_restore(self): + """Hook that is called before the restore command""" + pass + + def post_restore(self): + """Hook that is called after the restore command""" + pass + def restore(self): - self._pre_restore() + self.pre_restore() content_length = self._run_restore() - self._run_prepare() - self._post_restore() + self.post_restore() return content_length def _run_restore(self): @@ -151,51 +119,6 @@ class RestoreRunner(Strategy): return content_length - def _run_prepare(self): - if hasattr(self, 'prepare_cmd'): - LOG.info("Running innobackupex prepare...") - self.prep_retcode = utils.execute(self.prepare_cmd, - shell=True) - LOG.info("Innobackupex prepare finished successfully") - - def _spawn_with_init_file(self, temp_file): - child = pexpect.spawn("sudo mysqld_safe --init-file=%s" % - temp_file.name) - try: - i = child.expect(['Starting mysqld daemon']) - if i == 0: - LOG.info("Starting mysqld daemon") - except pexpect.TIMEOUT as e: - LOG.error("wait_and_close_proc failed: %s" % e) - finally: - # There is a race condition here where we kill mysqld before - # the init file been executed. We need to ensure mysqld is up. - poll_until_then_raise(mysql_is_running, - RestoreError("Reset root password failed: " - "mysqld did not start!")) - LOG.info("Root password reset successfully!") - LOG.info("Cleaning up the temp mysqld process...") - child.delayafterclose = 1 - child.delayafterterminate = 1 - child.close(force=True) - utils.execute_with_timeout("sudo", "killall", "mysqld") - poll_until_then_raise(mysql_is_not_running, - RestoreError("Reset root password failed: " - "mysqld did not stop!")) - - def _reset_root_password(self): - #Create temp file with reset root password - with tempfile.NamedTemporaryFile() as fp: - fp.write(RESET_ROOT_MYSQL_COMMAND) - fp.flush() - utils.execute_with_timeout("sudo", "chmod", "a+r", fp.name) - self._spawn_with_init_file(fp) - - def _delete_old_binlogs(self): - filelist = glob.glob(self.restore_location + "/ib_logfile*") - for f in filelist: - os.unlink(f) - @property def decrypt_cmd(self): if self.is_encrypted: diff --git a/trove/guestagent/strategies/restore/mysql_impl.py b/trove/guestagent/strategies/restore/mysql_impl.py index 872227fc3c..b6d7d66c7b 100644 --- a/trove/guestagent/strategies/restore/mysql_impl.py +++ b/trove/guestagent/strategies/restore/mysql_impl.py @@ -14,44 +14,129 @@ # under the License. # +import glob +import os +import pexpect +import tempfile + from trove.guestagent.strategies.restore import base from trove.openstack.common import log as logging +from trove.common import exception from trove.common import utils import trove.guestagent.datastore.mysql.service as dbaas LOG = logging.getLogger(__name__) -class MySQLDump(base.RestoreRunner): - """Implementation of Restore Strategy for MySQLDump """ +class MySQLRestoreMixin(object): + """Common utils for restoring MySQL databases""" + RESET_ROOT_RETRY_TIMEOUT = 100 + RESET_ROOT_SLEEP_INTERVAL = 10 + RESET_ROOT_MYSQL_COMMAND = ("SET PASSWORD FOR" + "'root'@'localhost'=PASSWORD('');") + + def mysql_is_running(self): + if base.exec_with_root_helper("/usr/bin/mysqladmin", "ping"): + LOG.info("The mysqld daemon is up and running.") + return True + else: + LOG.info("The mysqld daemon is not running.") + return False + + def mysql_is_not_running(self): + if base.exec_with_root_helper("/usr/bin/pgrep", "mysqld"): + LOG.info("The mysqld daemon is still running.") + return False + else: + LOG.info("The mysqld daemon is not running.") + return True + + def poll_until_then_raise(self, event, exc): + try: + utils.poll_until(event, + sleep_time=self.RESET_ROOT_SLEEP_INTERVAL, + time_out=self.RESET_ROOT_RETRY_TIMEOUT) + except exception.PollTimeOut: + raise exc + + def _spawn_with_init_file(self, temp_file): + child = pexpect.spawn("sudo mysqld_safe --init-file=%s" % + temp_file.name) + try: + i = child.expect(['Starting mysqld daemon']) + if i == 0: + LOG.info("Starting mysqld daemon") + except pexpect.TIMEOUT as e: + LOG.error("wait_and_close_proc failed: %s" % e) + finally: + # There is a race condition here where we kill mysqld before + # the init file been executed. We need to ensure mysqld is up. + self.poll_until_then_raise( + self.mysql_is_running, + base.RestoreError("Reset root password failed: " + "mysqld did not start!")) + LOG.info("Root password reset successfully!") + LOG.info("Cleaning up the temp mysqld process...") + child.delayafterclose = 1 + child.delayafterterminate = 1 + child.close(force=True) + utils.execute_with_timeout("sudo", "killall", "mysqld") + self.poll_until_then_raise( + self.mysql_is_not_running, + base.RestoreError("Reset root password failed: " + "mysqld did not stop!")) + + def reset_root_password(self): + #Create temp file with reset root password + with tempfile.NamedTemporaryFile() as fp: + fp.write(self.RESET_ROOT_MYSQL_COMMAND) + fp.flush() + utils.execute_with_timeout("sudo", "chmod", "a+r", fp.name) + self._spawn_with_init_file(fp) + + +class MySQLDump(base.RestoreRunner, MySQLRestoreMixin): + """Implementation of Restore Strategy for MySQLDump""" __strategy_name__ = 'mysqldump' - base_restore_cmd = ('mysql ' - '--password=%(password)s ' - '-u %(user)s') - - def _pre_restore(self): - pass - - def _post_restore(self): - pass + base_restore_cmd = 'sudo mysql' -class InnoBackupEx(base.RestoreRunner): - """Implementation of Restore Strategy for InnoBackupEx """ +class InnoBackupEx(base.RestoreRunner, MySQLRestoreMixin): + """Implementation of Restore Strategy for InnoBackupEx""" __strategy_name__ = 'innobackupex' base_restore_cmd = 'sudo xbstream -x -C %(restore_location)s' base_prepare_cmd = ('sudo innobackupex --apply-log %(restore_location)s' ' --defaults-file=%(restore_location)s/backup-my.cnf' ' --ibbackup xtrabackup 2>/tmp/innoprepare.log') - def _pre_restore(self): + def __init__(self, *args, **kwargs): + super(InnoBackupEx, self).__init__(*args, **kwargs) + self.prepare_cmd = self.base_prepare_cmd % kwargs + self.prep_retcode = None + + def pre_restore(self): app = dbaas.MySqlApp(dbaas.MySqlAppStatus.get()) app.stop_db() + LOG.info("Cleaning out restore location: %s", self.restore_location) + utils.execute_with_timeout("sudo", "chmod", "-R", + "0777", self.restore_location) + utils.clean_out(self.restore_location) - def _post_restore(self): + def _run_prepare(self): + LOG.info("Running innobackupex prepare: %s", self.prepare_cmd) + self.prep_retcode = utils.execute(self.prepare_cmd, shell=True) + LOG.info("Innobackupex prepare finished successfully") + + def post_restore(self): + self._run_prepare() utils.execute_with_timeout("sudo", "chown", "-R", "-f", "mysql", self.restore_location) self._delete_old_binlogs() - self._reset_root_password() + self.reset_root_password() app = dbaas.MySqlApp(dbaas.MySqlAppStatus.get()) app.start_mysql() + + def _delete_old_binlogs(self): + files = glob.glob(os.path.join(self.restore_location, "ib_logfile*")) + for f in files: + os.unlink(f) diff --git a/trove/tests/unittests/backup/test_backupagent.py b/trove/tests/unittests/backup/test_backupagent.py index dd6d14e2cb..6040c7c22c 100644 --- a/trove/tests/unittests/backup/test_backupagent.py +++ b/trove/tests/unittests/backup/test_backupagent.py @@ -161,12 +161,13 @@ class BackupAgentTest(testtools.TestCase): user='123', extra_opts='') self.assertIsNotNone(mysql_dump.cmd) - str_mysql_dump_cmd = ('/usr/bin/mysqldump' + str_mysql_dump_cmd = ('mysqldump' ' --all-databases' ' %(extra_opts)s' ' --opt' ' --password=%(password)s' ' -u %(user)s' + ' 2>/tmp/mysqldump.log' ' | gzip |' ' openssl enc -aes-256-cbc -salt ' '-pass pass:default_aes_cbc_key') diff --git a/trove/tests/unittests/guestagent/test_backups.py b/trove/tests/unittests/guestagent/test_backups.py index 0978422fcc..0b0a8afeb0 100644 --- a/trove/tests/unittests/guestagent/test_backups.py +++ b/trove/tests/unittests/guestagent/test_backups.py @@ -34,13 +34,14 @@ XTRA_BACKUP_RAW = ("sudo innobackupex --stream=xbstream %(extra_opts)s" " /var/lib/mysql 2>/tmp/innobackupex.log") XTRA_BACKUP = XTRA_BACKUP_RAW % {'extra_opts': ''} XTRA_BACKUP_EXTRA_OPTS = XTRA_BACKUP_RAW % {'extra_opts': '--no-lock'} -SQLDUMP_BACKUP_RAW = ("/usr/bin/mysqldump --all-databases %(extra_opts)s " - "--opt --password=password -u user") +SQLDUMP_BACKUP_RAW = ("mysqldump --all-databases %(extra_opts)s " + "--opt --password=password -u user" + " 2>/tmp/mysqldump.log") SQLDUMP_BACKUP = SQLDUMP_BACKUP_RAW % {'extra_opts': ''} SQLDUMP_BACKUP_EXTRA_OPTS = (SQLDUMP_BACKUP_RAW % {'extra_opts': '--events --routines --triggers'}) XTRA_RESTORE = "sudo xbstream -x -C /var/lib/mysql" -SQLDUMP_RESTORE = "mysql --password=password -u user" +SQLDUMP_RESTORE = "sudo mysql" PREPARE = ("sudo innobackupex --apply-log /var/lib/mysql " "--defaults-file=/var/lib/mysql/backup-my.cnf " "--ibbackup xtrabackup 2>/tmp/innoprepare.log") @@ -132,7 +133,6 @@ class GuestAgentBackupTest(testtools.TestCase): restr = RunnerClass(None, restore_location="/var/lib/mysql", user="user", password="password") self.assertEqual(restr.restore_cmd, UNZIP + PIPE + SQLDUMP_RESTORE) - self.assertIsNone(restr.prepare_cmd) def test_restore_encrypted_mysqldump_command(self): restoreBase.RestoreRunner.is_zipped = True @@ -143,4 +143,3 @@ class GuestAgentBackupTest(testtools.TestCase): user="user", password="password") self.assertEqual(restr.restore_cmd, DECRYPT + PIPE + UNZIP + PIPE + SQLDUMP_RESTORE) - self.assertIsNone(restr.prepare_cmd)