From 68c4246998eb13511d48f151b643be3911d321cf Mon Sep 17 00:00:00 2001 From: Sushil Kumar Date: Fri, 20 Mar 2015 20:54:51 +0000 Subject: [PATCH] Fixes the resize APIs for Vertica-guest Identified that there was a service vertica-agent still running after Vertica database was stopped, because of which Volume mount was not complete and hence resize-volume was failing. start_db_with_conf_changes implemented to start Vertica to complete the workflow of resize-instance. Also, updated the raise exception for bad-cluster-config to raise RuntimeError, and removed not needed raise from prepare exception handler. Change-Id: I8ccef3aef7725b0c7d46fd8851d69d3d80cfa995 Closes-Bug: #1433852 --- .../datastore/experimental/vertica/manager.py | 6 +- .../datastore/experimental/vertica/service.py | 99 ++++++++++++------- .../datastore/experimental/vertica/system.py | 3 +- .../tests/unittests/guestagent/test_dbaas.py | 89 +++++++++-------- 4 files changed, 117 insertions(+), 80 deletions(-) diff --git a/trove/guestagent/datastore/experimental/vertica/manager.py b/trove/guestagent/datastore/experimental/vertica/manager.py index 6ca4aa51e2..5aaa1eee94 100644 --- a/trove/guestagent/datastore/experimental/vertica/manager.py +++ b/trove/guestagent/datastore/experimental/vertica/manager.py @@ -71,12 +71,11 @@ class Manager(periodic_task.PeriodicTasks): else: LOG.error(_("Bad cluster configuration; instance type " "given as %s.") % cluster_config['instance_type']) - raise + raise RuntimeError("Bad cluster configuration.") LOG.info(_('Completed setup of Vertica database instance.')) except Exception: LOG.exception(_('Cannot prepare Vertica database instance.')) self.appStatus.set_status(rd_ins.ServiceStatuses.FAILED) - raise def restart(self, context): LOG.debug("Restarting the database.") @@ -196,8 +195,7 @@ class Manager(periodic_task.PeriodicTasks): def start_db_with_conf_changes(self, context, config_contents): LOG.debug("Starting with configuration changes.") - raise exception.DatastoreOperationNotSupported( - operation='start_db_with_conf_changes', datastore=MANAGER) + self.app.start_db_with_conf_changes(config_contents) def get_public_keys(self, context, user): LOG.debug("Retrieving public keys for %s." % user) diff --git a/trove/guestagent/datastore/experimental/vertica/service.py b/trove/guestagent/datastore/experimental/vertica/service.py index dfeff05558..2352df0267 100644 --- a/trove/guestagent/datastore/experimental/vertica/service.py +++ b/trove/guestagent/datastore/experimental/vertica/service.py @@ -13,6 +13,7 @@ import ConfigParser import os +import subprocess import tempfile from trove.common import cfg @@ -45,16 +46,9 @@ class VerticaAppStatus(service.BaseDbStatus): #UP status is confirmed LOG.info(_("Service Status is RUNNING.")) return rd_instance.ServiceStatuses.RUNNING - elif out.strip() == "": - #nothing returned, means no db running lets verify - out, err = system.shell_execute(system.STATUS_DB_DOWN, - "dbadmin") - if out.strip() == DB_NAME: - #DOWN status is confirmed - LOG.info(_("Service Status is SHUTDOWN.")) - return rd_instance.ServiceStatuses.SHUTDOWN - else: - return rd_instance.ServiceStatuses.UNKNOWN + else: + LOG.info(_("Service Status is SHUTDOWN.")) + return rd_instance.ServiceStatuses.SHUTDOWN except exception.ProcessExecutionError: LOG.exception(_("Failed to get database status.")) return rd_instance.ServiceStatuses.CRASHED @@ -68,52 +62,85 @@ class VerticaApp(object): self.status = status def _enable_db_on_boot(self): - command = (system.SET_RESTART_POLICY % (DB_NAME, "always")) try: - system.shell_execute(command, "dbadmin") - except exception.ProcessExecutionError: + command = ["sudo", "su", "-", "dbadmin", "-c", + (system.SET_RESTART_POLICY % (DB_NAME, "always"))] + subprocess.Popen(command) + command = ["sudo", "su", "-", "root", "-c", + (system.VERTICA_AGENT_SERVICE_COMMAND % "enable")] + subprocess.Popen(command) + except Exception: LOG.exception(_("Failed to enable db on boot.")) - raise + raise RuntimeError("Could not enable db on boot.") def _disable_db_on_boot(self): - command = (system.SET_RESTART_POLICY % (DB_NAME, "never")) try: + command = (system.SET_RESTART_POLICY % (DB_NAME, "never")) system.shell_execute(command, "dbadmin") + command = (system.VERTICA_AGENT_SERVICE_COMMAND % "disable") + system.shell_execute(command) except exception.ProcessExecutionError: LOG.exception(_("Failed to disable db on boot.")) - raise + raise RuntimeError("Could not disable db on boot.") def stop_db(self, update_db=False, do_not_start_on_reboot=False): """Stop the database.""" LOG.info(_("Stopping Vertica.")) if do_not_start_on_reboot: self._disable_db_on_boot() - # Using Vertica adminTools to stop db. - db_password = self._get_database_password() - stop_db_command = (system.STOP_DB % (DB_NAME, db_password)) - system.shell_execute(stop_db_command, "dbadmin") - if not self.status.wait_for_real_status_to_change_to( - rd_instance.ServiceStatuses.SHUTDOWN, - self.state_change_wait_time, update_db): - LOG.error(_("Could not stop Vertica.")) - self.status.end_install_or_restart() - raise RuntimeError("Could not stop Vertica!") + + try: + # Stop vertica-agent service + command = (system.VERTICA_AGENT_SERVICE_COMMAND % "stop") + system.shell_execute(command) + # Using Vertica adminTools to stop db. + db_password = self._get_database_password() + stop_db_command = (system.STOP_DB % (DB_NAME, db_password)) + out, err = system.shell_execute(system.STATUS_ACTIVE_DB, "dbadmin") + if out.strip() == DB_NAME: + system.shell_execute(stop_db_command, "dbadmin") + if not self.status._is_restarting: + if not self.status.wait_for_real_status_to_change_to( + rd_instance.ServiceStatuses.SHUTDOWN, + self.state_change_wait_time, update_db): + LOG.error(_("Could not stop Vertica.")) + self.status.end_install_or_restart() + raise RuntimeError("Could not stop Vertica!") + LOG.debug("Database stopped.") + else: + LOG.debug("Database is not running.") + except exception.ProcessExecutionError: + LOG.exception(_("Failed to stop database.")) + raise RuntimeError("Could not stop database.") def start_db(self, update_db=False): """Start the database.""" LOG.info(_("Starting Vertica.")) - self._enable_db_on_boot() - # Using Vertica adminTools to start db. - db_password = self._get_database_password() - start_db_command = (system.START_DB % (DB_NAME, db_password)) - system.shell_execute(start_db_command, "dbadmin") - if not self.status.wait_for_real_status_to_change_to( - rd_instance.ServiceStatuses.RUNNING, - self.state_change_wait_time, update_db): - LOG.error(_("Start up of Vertica failed.")) - self.status.end_install_or_restart() + try: + self._enable_db_on_boot() + # Start vertica-agent service + command = ["sudo", "su", "-", "root", "-c", + (system.VERTICA_AGENT_SERVICE_COMMAND % "start")] + subprocess.Popen(command) + # Using Vertica adminTools to start db. + db_password = self._get_database_password() + start_db_command = ["sudo", "su", "-", "dbadmin", "-c", + (system.START_DB % (DB_NAME, db_password))] + subprocess.Popen(start_db_command) + if not self.status._is_restarting: + self.status.end_install_or_restart() + LOG.debug("Database started.") + except Exception: raise RuntimeError("Could not start Vertica!") + def start_db_with_conf_changes(self, config_contents): + """ + Currently all that this method does is to start Vertica. This method + needs to be implemented to enable volume resize on guestagent side. + """ + LOG.info(_("Starting Vertica with configuration changes.")) + self.start_db(True) + def restart(self): """Restart the database.""" try: diff --git a/trove/guestagent/datastore/experimental/vertica/system.py b/trove/guestagent/datastore/experimental/vertica/system.py index 31e98850cd..4de914c3c9 100644 --- a/trove/guestagent/datastore/experimental/vertica/system.py +++ b/trove/guestagent/datastore/experimental/vertica/system.py @@ -29,6 +29,7 @@ SEND_CONF_TO_SERVER = ("rsync -v -e 'ssh -o " "StrictHostKeyChecking=no' --perms --owner --group " "%s %s:%s") SSH_KEY_GEN = "ssh-keygen -f %s/.ssh/id_rsa -t rsa -N ''" +VERTICA_AGENT_SERVICE_COMMAND = "service vertica_agent %s" VERTICA_CONF = "/etc/vertica.cnf" INSTALL_TIMEOUT = 1000 @@ -40,5 +41,5 @@ def shell_execute(command, command_executor="root"): #Note: This method uses su because using sudo -i -u #does not works with vertica installer - #and it has problems while executing remote commmands. + #and it has problems while executing remote commands. return utils.execute("sudo", "su", "-", command_executor, "-c", command) diff --git a/trove/tests/unittests/guestagent/test_dbaas.py b/trove/tests/unittests/guestagent/test_dbaas.py index fa1e76e9c3..557d69418d 100644 --- a/trove/tests/unittests/guestagent/test_dbaas.py +++ b/trove/tests/unittests/guestagent/test_dbaas.py @@ -14,11 +14,13 @@ import ConfigParser import os +import subprocess import tempfile from uuid import uuid4 import time from mock import Mock from mock import MagicMock +from mock import PropertyMock from mock import patch from mock import ANY import sqlalchemy @@ -2098,13 +2100,6 @@ class VerticaAppStatusTest(testtools.TestCase): status = self.verticaAppStatus._get_actual_db_status() self.assertEqual(rd_instance.ServiceStatuses.CRASHED, status) - def test_get_actual_db_status_error_unknown(self): - self.verticaAppStatus = VerticaAppStatus() - with patch.object(vertica_system, 'shell_execute', - MagicMock(return_value=['', None])): - status = self.verticaAppStatus._get_actual_db_status() - self.assertEqual(rd_instance.ServiceStatuses.UNKNOWN, status) - class VerticaAppTest(testtools.TestCase): @@ -2115,9 +2110,11 @@ class VerticaAppTest(testtools.TestCase): rd_instance.ServiceStatuses.NEW) self.app = VerticaApp(self.appStatus) self.setread = VolumeDevice.set_readahead_size + self.Popen = subprocess.Popen vertica_system.shell_execute = MagicMock(return_value=('', '')) VolumeDevice.set_readahead_size = Mock() + subprocess.Popen = Mock() self.test_config = ConfigParser.ConfigParser() self.test_config.add_section('credentials') self.test_config.set('credentials', @@ -2127,6 +2124,7 @@ class VerticaAppTest(testtools.TestCase): super(VerticaAppTest, self).tearDown() self.app = None VolumeDevice.set_readahead_size = self.setread + subprocess.Popen = self.Popen def test_install_if_needed_installed(self): with patch.object(pkg.Package, 'pkg_is_installed', return_value=True): @@ -2227,71 +2225,84 @@ class VerticaAppTest(testtools.TestCase): mock_status.begin_restart.assert_any_call() VerticaApp.stop_db.assert_any_call() VerticaApp.start_db.assert_any_call() - mock_status.end_install_or_restart.assert_any_call() def test_start_db(self): mock_status = MagicMock() + type(mock_status)._is_restarting = PropertyMock(return_value=False) app = VerticaApp(mock_status) with patch.object(app, '_enable_db_on_boot', return_value=None): with patch.object(app, 'read_config', return_value=self.test_config): - mock_status.wait_for_real_status_to_change_to = MagicMock( - return_value=True) mock_status.end_install_or_restart = MagicMock( return_value=None) - app.start_db() - - arguments = vertica_system.shell_execute.call_args_list[0] - expected_cmd = (vertica_system.START_DB % ('db_srvr', - 'some_password')) - self.assertTrue( - mock_status.wait_for_real_status_to_change_to.called) - arguments.assert_called_with(expected_cmd, 'dbadmin') + agent_start, db_start = subprocess.Popen.call_args_list + agent_expected_command = [ + 'sudo', 'su', '-', 'root', '-c', + (vertica_system.VERTICA_AGENT_SERVICE_COMMAND % 'start')] + db_expected_cmd = [ + 'sudo', 'su', '-', 'dbadmin', '-c', + (vertica_system.START_DB % ('db_srvr', 'some_password'))] + self.assertTrue(mock_status.end_install_or_restart.called) + agent_start.assert_called_with(agent_expected_command) + db_start.assert_called_with(db_expected_cmd) def test_start_db_failure(self): mock_status = MagicMock() app = VerticaApp(mock_status) - with patch.object(app, '_enable_db_on_boot', return_value=None): + with patch.object(app, '_enable_db_on_boot', + side_effect=RuntimeError()): with patch.object(app, 'read_config', return_value=self.test_config): - mock_status.wait_for_real_status_to_change_to = MagicMock( - return_value=None) - mock_status.end_install_or_restart = MagicMock( - return_value=None) self.assertRaises(RuntimeError, app.start_db) def test_stop_db(self): mock_status = MagicMock() + type(mock_status)._is_restarting = PropertyMock(return_value=False) app = VerticaApp(mock_status) with patch.object(app, '_disable_db_on_boot', return_value=None): with patch.object(app, 'read_config', return_value=self.test_config): - mock_status.wait_for_real_status_to_change_to = MagicMock( - return_value=True) - mock_status.end_install_or_restart = MagicMock( - return_value=None) + with patch.object(vertica_system, 'shell_execute', + MagicMock(side_effect=[['', ''], + ['db_srvr', None], + ['', '']])): + mock_status.wait_for_real_status_to_change_to = MagicMock( + return_value=True) + mock_status.end_install_or_restart = MagicMock( + return_value=None) + app.stop_db() - app.stop_db() - - arguments = vertica_system.shell_execute.call_args_list[0] - expected_command = (vertica_system.STOP_DB % ('db_srvr', + self.assertEqual(vertica_system.shell_execute.call_count, + 3) + # There are 3 shell-executions: + # a) stop vertica-agent service + # b) check daatabase status + # c) stop_db + # We are matcing that 3rd command called was stop_db + arguments = vertica_system.shell_execute.call_args_list[2] + expected_cmd = (vertica_system.STOP_DB % ('db_srvr', 'some_password')) - self.assertTrue( - mock_status.wait_for_real_status_to_change_to.called) - arguments.assert_called_with(expected_command, 'dbadmin') + self.assertTrue( + mock_status.wait_for_real_status_to_change_to.called) + arguments.assert_called_with(expected_cmd, 'dbadmin') def test_stop_db_failure(self): mock_status = MagicMock() + type(mock_status)._is_restarting = PropertyMock(return_value=False) app = VerticaApp(mock_status) with patch.object(app, '_disable_db_on_boot', return_value=None): with patch.object(app, 'read_config', return_value=self.test_config): - mock_status.wait_for_real_status_to_change_to = MagicMock( - return_value=None) - mock_status.end_install_or_restart = MagicMock( - return_value=None) - self.assertRaises(RuntimeError, app.stop_db) + with patch.object(vertica_system, 'shell_execute', + MagicMock(side_effect=[['', ''], + ['db_srvr', None], + ['', '']])): + mock_status.wait_for_real_status_to_change_to = MagicMock( + return_value=None) + mock_status.end_install_or_restart = MagicMock( + return_value=None) + self.assertRaises(RuntimeError, app.stop_db) def test_export_conf_to_members(self): self.app._export_conf_to_members(members=['member1', 'member2'])