MongoDB prepare needs to wait for Mongo to start

MongoDB's prepare method assumes that the mongod service has finished
starting. This may not be the case. It needs to wait.
The fix is to use the polling code in the start_db function, so factor
this out.

Change-Id: I2f1e6ecec3f9c0b438007f01334168ea4fbe3884
Closes-bug: #1474522

trove/guestagent/datastore/experimental/mongodb/manager.py

@@ -62,6 +62,7 @@ class Manager(periodic_task.PeriodicTasks):
 
         self.status.begin_install()
         self.app.install_if_needed(packages)
+        self.app.wait_for_start()
         self.app.stop_db()
         self.app.clear_storage()
         mount_point = system.MONGODB_MOUNT_POINT

trove/guestagent/datastore/experimental/mongodb/service.py

@@ -126,7 +126,10 @@ class MongoDBApp(object):
             pass
         except KeyError:
             raise RuntimeError("MongoDB service is not discovered.")
+        self.wait_for_start(update_db=update_db)
 
+    def wait_for_start(self, update_db=False):
+        LOG.debug('Waiting for MongoDB to start.')
         if not self.status.wait_for_real_status_to_change_to(
                 ds_instance.ServiceStatuses.RUNNING,
                 self.state_change_wait_time, update_db):
@@ -144,6 +147,7 @@ class MongoDBApp(object):
                 # There's nothing more we can do...
             self.status.end_install_or_restart()
             raise RuntimeError("Could not start MongoDB.")
+        LOG.debug('MongoDB started successfully.')
 
     def start_db_with_conf_changes(self, config_contents):
         LOG.info(_("Starting MongoDB with configuration changes."))
@@ -418,16 +422,11 @@ class MongoDBAppStatus(service.BaseDbStatus):
 
     def _get_actual_db_status(self):
         try:
-            if self._is_config_server() is True:
-                status_check = (system.CMD_STATUS %
-                                (netutils.get_my_ipv4() +
-                                 ' --port %s' % CONFIGSVR_PORT))
-            else:
-                status_check = (system.CMD_STATUS %
-                                netutils.get_my_ipv4())
-
-            out, err = utils.execute_with_timeout(status_check, shell=True,
-                                                  check_exit_code=[0, 1])
+            port = CONFIGSVR_PORT if self._is_config_server() else MONGODB_PORT
+            out, err = utils.execute_with_timeout(
+                'mongostat', '--host', str(netutils.get_my_ipv4()),
+                '--port', str(port), '-n', str(1), check_exit_code=[0, 1]
+            )
             if not err:
                 return ds_instance.ServiceStatuses.RUNNING
             else:

trove/guestagent/datastore/experimental/mongodb/system.py

@@ -21,9 +21,6 @@ from trove.guestagent import pkg
 OS_NAME = operating_system.get_os()
 
 MONGODB_MOUNT_POINT = "/var/lib/mongodb"
-# After changing bind address mongodb accepts connection
-# on real IP, not on the localhost
-CMD_STATUS = "mongostat --host %s -n 1"
 
 TMP_CONFIG = "/tmp/mongodb.conf.tmp"
 CONFIG_CANDIDATES = ["/etc/mongodb.conf", "/etc/mongod.conf"]

trove/tests/unittests/guestagent/test_mongodb_cluster_manager.py

@@ -193,6 +193,7 @@ class GuestAgentMongoDBClusterManagerTest(trove_testtools.TestCase):
     @mock.patch.object(service.MongoDBApp, 'clear_storage')
     @mock.patch.object(service.MongoDBApp, 'start_db')
     @mock.patch.object(service.MongoDBApp, 'stop_db')
+    @mock.patch.object(service.MongoDBApp, 'wait_for_start')
     @mock.patch.object(service.MongoDBApp, 'install_if_needed')
     @mock.patch.object(service.MongoDBAppStatus, 'begin_install')
     def _prepare_method(self, instance_id, instance_type, key, *args):