From ced867e7feabe830ca193add5437f12276b9dc05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ionu=C8=9B=20Ar=C8=9B=C4=83ri=C8=99i?= <iartarisi@suse.cz>
Date: Thu, 10 Apr 2014 16:25:52 +0200
Subject: [PATCH] reconnect to mongodb on connection failure

Try to reconnect to mongodb when the client library raised a
ConnectionFailure error. This is especially useful when mongodb is in
the process of initializing a replica set and it does not respond to
connections even though the mongodb service is up.

Change-Id: I1818d90c135b248a28b35dc0b9f6106cf89496ab
Closes-Bug: #1305920
---
 ceilometer/storage/pymongo_base.py            | 35 ++++++++++++++--
 ceilometer/tests/storage/test_pymongo_base.py | 42 +++++++++++++++++++
 2 files changed, 74 insertions(+), 3 deletions(-)

diff --git a/ceilometer/storage/pymongo_base.py b/ceilometer/storage/pymongo_base.py
index cea51d97e..b145e25c2 100644
--- a/ceilometer/storage/pymongo_base.py
+++ b/ceilometer/storage/pymongo_base.py
@@ -19,6 +19,9 @@
 """Common functions for MongoDB and DB2 backends
 """
 
+import time
+
+from oslo.config import cfg
 import pymongo
 import weakref
 
@@ -31,6 +34,11 @@ from ceilometer import utils
 
 LOG = log.getLogger(__name__)
 
+cfg.CONF.import_opt('max_retries', 'ceilometer.openstack.common.db.options',
+                    group="database")
+cfg.CONF.import_opt('retry_interval', 'ceilometer.openstack.common.db.options',
+                    group="database")
+
 
 def make_timestamp_range(start, end,
                          start_timestamp_op=None, end_timestamp_op=None):
@@ -120,12 +128,33 @@ class ConnectionPool(object):
         log_data = {'db': splitted_url.scheme,
                     'nodelist': connection_options['nodelist']}
         LOG.info(_('Connecting to %(db)s on %(nodelist)s') % log_data)
-        client = pymongo.MongoClient(
-            url,
-            safe=True)
+        client = self._mongo_connect(url)
         self._pool[pool_key] = weakref.ref(client)
         return client
 
+    @staticmethod
+    def _mongo_connect(url):
+        max_retries = cfg.CONF.database.max_retries
+        retry_interval = cfg.CONF.database.retry_interval
+        attempts = 0
+        while True:
+            try:
+                client = pymongo.MongoClient(url, safe=True)
+            except pymongo.errors.ConnectionFailure as e:
+                if max_retries >= 0 and attempts >= max_retries:
+                    LOG.error(_('Unable to connect to the database after '
+                                '%(retries)d retries. Giving up.') %
+                              {'retries': max_retries})
+                    raise
+                LOG.warn(_('Unable to connect to the database server: '
+                           '%(errmsg)s. Trying again in %(retry_interval)d '
+                           'seconds.') %
+                         {'errmsg': e, 'retry_interval': retry_interval})
+                attempts += 1
+                time.sleep(retry_interval)
+            else:
+                return client
+
 
 COMMON_AVAILABLE_CAPABILITIES = {
     'meters': {'query': {'simple': True,
diff --git a/ceilometer/tests/storage/test_pymongo_base.py b/ceilometer/tests/storage/test_pymongo_base.py
index 4b5574ad9..0a152416e 100644
--- a/ceilometer/tests/storage/test_pymongo_base.py
+++ b/ceilometer/tests/storage/test_pymongo_base.py
@@ -13,14 +13,19 @@
 """Tests the mongodb and db2 common functionality
 """
 
+import contextlib
 import copy
 import datetime
 
+from mock import call
 from mock import patch
+import pymongo
 import testscenarios
 
+from ceilometer.openstack.common.gettextutils import _
 from ceilometer.publisher import utils
 from ceilometer import sample
+from ceilometer.storage import pymongo_base
 from ceilometer.tests import db as tests_db
 from ceilometer.tests.storage import test_storage_scenarios
 
@@ -172,3 +177,40 @@ class CompatibilityTest(test_storage_scenarios.DBTestBase,
     def test_counter_unit(self):
         meters = list(self.conn.get_meters())
         self.assertEqual(1, len(meters))
+
+    def test_mongodb_connect_raises_after_custom_number_of_attempts(self):
+        retry_interval = 13
+        max_retries = 37
+        self.CONF.set_override(
+            'retry_interval', retry_interval, group='database')
+        self.CONF.set_override(
+            'max_retries', max_retries, group='database')
+        # PyMongo is being used to connect even to DB2, but it only
+        # accepts URLs with the 'mongodb' scheme. This replacement is
+        # usually done in the DB2 connection implementation, but since
+        # we don't call that, we have to do it here.
+        self.CONF.set_override(
+            'connection', self.db_manager.url.replace('db2:', 'mongodb:', 1),
+            group='database')
+
+        pool = pymongo_base.ConnectionPool()
+        with contextlib.nested(
+                patch('pymongo.MongoClient',
+                      side_effect=pymongo.errors.ConnectionFailure('foo')),
+                patch.object(pymongo_base.LOG, 'error'),
+                patch.object(pymongo_base.LOG, 'warn'),
+                patch.object(pymongo_base.time, 'sleep')
+        ) as (MockMongo, MockLOGerror, MockLOGwarn, Mocksleep):
+            self.assertRaises(pymongo.errors.ConnectionFailure,
+                              pool.connect, self.CONF.database.connection)
+            Mocksleep.assert_has_calls([call(retry_interval)
+                                        for i in range(max_retries)])
+            MockLOGwarn.assert_any_call(
+                _('Unable to connect to the database server: %(errmsg)s.'
+                  ' Trying again in %(retry_interval)d seconds.') %
+                {'errmsg': 'foo',
+                 'retry_interval': retry_interval})
+            MockLOGerror.assert_called_with(
+                _('Unable to connect to the database after '
+                  '%(retries)d retries. Giving up.') %
+                {'retries': max_retries})