From 3b251f2041dad042def705a9fa0029a112cdec13 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 2 Jul 2013 18:50:47 -0400 Subject: [PATCH] Update swift pollster to use cache Use the pollster cache to store the list of tenants and the information swift has about them. blueprint one-meter-per-plugin Change-Id: I2c1fbbb99b2ae4796720ec6b7ac94ac27ab9ba5c Signed-off-by: Doug Hellmann --- ceilometer/objectstore/swift.py | 24 +++++++++++----- tests/objectstore/test_swift.py | 50 +++++++++++++++++++++++++++------ 2 files changed, 59 insertions(+), 15 deletions(-) diff --git a/ceilometer/objectstore/swift.py b/ceilometer/objectstore/swift.py index 79e58647c..4f255c97e 100644 --- a/ceilometer/objectstore/swift.py +++ b/ceilometer/objectstore/swift.py @@ -27,6 +27,7 @@ from swiftclient import client as swift from keystoneclient import exceptions from ceilometer import counter +from ceilometer.openstack.common.gettextutils import _ from ceilometer.openstack.common import log from ceilometer.openstack.common import timeutils from ceilometer import plugin @@ -50,13 +51,12 @@ class _Base(plugin.PollsterBase): __metaclass__ = abc.ABCMeta - @staticmethod @abc.abstractmethod - def iter_accounts(ksclient): + def _iter_accounts(ksclient, cache): """Iterate over all accounts, yielding (tenant_id, stats) tuples.""" def get_counters(self, manager, cache): - for tenant, account in self.iter_accounts(manager.keystone): + for tenant, account in self._iter_accounts(manager.keystone, cache): yield counter.Counter( name='storage.objects', type=counter.TYPE_GAUGE, @@ -102,17 +102,27 @@ class SwiftPollster(_Base): 'storage.objects.size', 'storage.objects.containers'] - @staticmethod - def iter_accounts(ksclient): + CACHE_KEY_TENANT = 'tenants' + CACHE_KEY_HEAD = 'swift.head_account' + + def _iter_accounts(self, ksclient, cache): + if self.CACHE_KEY_TENANT not in cache: + cache[self.CACHE_KEY_TENANT] = ksclient.tenants.list() + if self.CACHE_KEY_HEAD not in cache: + cache[self.CACHE_KEY_HEAD] = list(self._get_account_info(ksclient, + cache)) + return iter(cache['swift.head_account']) + + def _get_account_info(self, ksclient, cache): try: endpoint = ksclient.service_catalog.url_for( service_type='object-store', endpoint_type='adminURL') except exceptions.EndpointNotFound: LOG.debug(_("Swift endpoint not found")) - return + raise StopIteration() - for t in ksclient.tenants.list(): + for t in cache['tenants']: yield (t.id, swift.head_account(SwiftPollster. _neaten_url(endpoint, t.id), ksclient.auth_token)) diff --git a/tests/objectstore/test_swift.py b/tests/objectstore/test_swift.py index ab5b133d5..eadbc5fbf 100644 --- a/tests/objectstore/test_swift.py +++ b/tests/objectstore/test_swift.py @@ -17,6 +17,8 @@ # License for the specific language governing permissions and limitations # under the License. +import collections + import mock from ceilometer.central import manager @@ -24,6 +26,8 @@ from ceilometer.objectstore import swift from ceilometer.tests import base from keystoneclient import exceptions +from swiftclient import client as swift_client + ACCOUNTS = [('tenant-000', {'x-account-object-count': 12, 'x-account-bytes-used': 321321321, @@ -48,8 +52,7 @@ class TestSwiftPollster(base.TestCase): def fake_ks_service_catalog_url_for(*args, **kwargs): raise exceptions.EndpointNotFound("Fake keystone exception") - @staticmethod - def fake_iter_accounts(self, ksclient): + def fake_iter_accounts(self, ksclient, cache): for i in ACCOUNTS: yield i @@ -59,7 +62,38 @@ class TestSwiftPollster(base.TestCase): self.pollster = swift.SwiftPollster() self.manager = TestManager() - def test_objectstore_neaten_url(self): + def test_iter_accounts_no_cache(self): + def empty_account_info(obj, ksclient, cache): + return [] + self.stubs.Set(swift.SwiftPollster, '_get_account_info', + empty_account_info) + cache = {} + data = list(self.pollster._iter_accounts(mock.Mock(), cache)) + self.assertTrue(self.pollster.CACHE_KEY_TENANT in cache) + self.assertTrue(self.pollster.CACHE_KEY_HEAD in cache) + self.assertEqual(data, []) + + def test_iter_accounts_tenants_cached(self): + # Verify that if there are tenants pre-cached then the account + # info loop iterates over those instead of asking for the list + # again. + ksclient = mock.Mock() + ksclient.tenants.list.side_effect = AssertionError( + 'should not be called', + ) + self.stubs.Set(swift_client, 'head_account', + ksclient) + self.stubs.Set(swift.SwiftPollster, '_neaten_url', + mock.Mock()) + Tenant = collections.namedtuple('Tenant', 'id') + cache = { + self.pollster.CACHE_KEY_TENANT: [Tenant(ACCOUNTS[0][0])], + } + data = list(self.pollster._iter_accounts(mock.Mock(), cache)) + self.assertTrue(self.pollster.CACHE_KEY_HEAD in cache) + self.assertEqual(data[0][0], ACCOUNTS[0][0]) + + def test_neaten_url(self): test_endpoint = 'http://127.0.0.1:8080' test_tenant_id = 'a7fd1695fa154486a647e44aa99a1b9b' standard_url = test_endpoint + '/v1/' + 'AUTH_' + test_tenant_id @@ -74,20 +108,20 @@ class TestSwiftPollster(base.TestCase): self.pollster._neaten_url(test_endpoint + '/v1', test_tenant_id)) - def test_objectstore_metering(self): - self.stubs.Set(swift.SwiftPollster, 'iter_accounts', + def test_metering(self): + self.stubs.Set(swift.SwiftPollster, '_iter_accounts', self.fake_iter_accounts) counters = list(self.pollster.get_counters(self.manager, {})) self.assertEqual(len(counters), 6) - def test_objectstore_get_counter_names(self): - self.stubs.Set(swift.SwiftPollster, 'iter_accounts', + def test_get_counter_names(self): + self.stubs.Set(swift.SwiftPollster, '_iter_accounts', self.fake_iter_accounts) counters = list(self.pollster.get_counters(self.manager, {})) self.assertEqual(set([c.name for c in counters]), set(self.pollster.get_counter_names())) - def test_objectstore_endpoint_notfound(self): + def test_endpoint_notfound(self): self.stubs.Set(self.manager.keystone.service_catalog, 'url_for', self.fake_ks_service_catalog_url_for) counters = list(self.pollster.get_counters(self.manager, {}))