openstack/aodh
Code Issues Proposed changes

Improved MongoClient pooling to avoid out of connections error

Browse Source 
Use 1 MongoClient instance per database server determined by host:port +
connection options

Fixes bug #1218488

Change-Id: If06844f6bf09674216b029310c1a5f445c4476fe
This commit is contained in:
Alexei Kornienko 2013-09-05 12:36:40 +03:00
parent 2152627f1a
commit d17dde508b
3 changed files with 34 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
ceilometer/storage/impl_db2.py
View File

@ -22,7 +22,6 @@
""" """
import copy import copy
import urlparse
import weakref import weakref
import bson.code import bson.code
@ -138,21 +137,22 @@ class ConnectionPool(object):
self._pool = {} self._pool = {}
def connect(self, url): def connect(self, url):
if url in self._pool: connection_options = pymongo.uri_parser.parse_uri(url)
client = self._pool.get(url)() del connection_options['database']
del connection_options['username']
del connection_options['password']
del connection_options['collection']
pool_key = tuple(connection_options)
if pool_key in self._pool:
client = self._pool.get(pool_key)()
if client: if client:
return client return client
LOG.info('connecting to DB2 on %s', url) LOG.info('connecting to MongoDB on %s', url)
url_parsed = urlparse.urlparse(url) client = pymongo.MongoClient(
if url_parsed.path.startswith('/ceilometer_for_tox_testing_'): url,
#note(sileht): this is a workaround for running tests without reach safe=True)
#the maximum allowed connection of mongod in gate self._pool[pool_key] = weakref.ref(client)
#this only work with pymongo >= 2.6, this is not in the
#requirements file because is not needed for normal use of mongo
client = pymongo.MongoClient(url, safe=True, max_pool_size=None)
else:
client = pymongo.MongoClient(url, safe=True)
self._pool[url] = weakref.ref(client)
return client return client
@ -205,6 +205,9 @@ class Connection(base.Connection):
connection_options = pymongo.uri_parser.parse_uri(url) connection_options = pymongo.uri_parser.parse_uri(url)
self.db = getattr(self.conn, connection_options['database']) self.db = getattr(self.conn, connection_options['database'])
if connection_options.get('username'):
self.db.authenticate(connection_options['username'],
connection_options['password'])
self.upgrade() self.upgrade()

29
ceilometer/storage/impl_mongodb.py
View File

@ -23,7 +23,6 @@
import calendar import calendar
import copy import copy
import operator import operator
import urlparse
import weakref import weakref
import bson.code import bson.code
@ -145,21 +144,22 @@ class ConnectionPool(object):
self._pool = {} self._pool = {}
def connect(self, url): def connect(self, url):
if url in self._pool: connection_options = pymongo.uri_parser.parse_uri(url)
client = self._pool.get(url)() del connection_options['database']
del connection_options['username']
del connection_options['password']
del connection_options['collection']
pool_key = tuple(connection_options)
if pool_key in self._pool:
client = self._pool.get(pool_key)()
if client: if client:
return client return client
LOG.info('connecting to MongoDB on %s', url) LOG.info('connecting to MongoDB on %s', url)
url_parsed = urlparse.urlparse(url) client = pymongo.MongoClient(
if url_parsed.path.startswith('/ceilometer_for_tox_testing_'): url,
#note(sileht): this is a workaround for running tests without reach safe=True)
#the maximum allowed connection of mongod in gate self._pool[pool_key] = weakref.ref(client)
#this only work with pymongo >= 2.6, this is not in the
#requirements file because is not needed for normal use of mongo
client = pymongo.MongoClient(url, safe=True, max_pool_size=None)
else:
client = pymongo.MongoClient(url, safe=True)
self._pool[url] = weakref.ref(client)
return client return client
@ -313,6 +313,9 @@ class Connection(base.Connection):
connection_options = pymongo.uri_parser.parse_uri(url) connection_options = pymongo.uri_parser.parse_uri(url)
self.db = getattr(self.conn, connection_options['database']) self.db = getattr(self.conn, connection_options['database'])
if connection_options.get('username'):
self.db.authenticate(connection_options['username'],
connection_options['password'])
# NOTE(jd) Upgrading is just about creating index, so let's do this # NOTE(jd) Upgrading is just about creating index, so let's do this
# on connection to be sure at least the TTL is correcly updated if # on connection to be sure at least the TTL is correcly updated if

2
run-tests.sh
View File

@ -36,5 +36,5 @@ done < ${MONGO_DATA}/out
# Read the fifo for ever otherwise mongod would block # Read the fifo for ever otherwise mongod would block
# + that gives us the log on screen # + that gives us the log on screen
cat ${MONGO_DATA}/out > /dev/null & cat ${MONGO_DATA}/out > /dev/null &
export CEILOMETER_TEST_MONGODB_URL="mongodb://localhost:29000/ceilometer_for_tox_testing" export CEILOMETER_TEST_MONGODB_URL="mongodb://localhost:29000/ceilometer"
python setup.py testr --slowest --testr-args="$*" $COVERAGE_ARG python setup.py testr --slowest --testr-args="$*" $COVERAGE_ARG