From af20bc270761455efc2875dc9f86946d0a8bdddc Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 4 Jul 2013 09:13:22 -0400 Subject: [PATCH] Break up the swift pollsters This change also adds a test dependency on testscenarios, a library for reusing test class definitions for parameterized tests. blueprint one-meter-per-plugin Change-Id: I0e1e0e4e225261598cc28d2e3335be6f163bbc85 Signed-off-by: Doug Hellmann --- ceilometer/objectstore/swift.py | 130 ++++++++++++++++++-------------- setup.cfg | 4 +- test-requirements.txt | 1 + tests/objectstore/test_swift.py | 37 ++++++--- 4 files changed, 101 insertions(+), 71 deletions(-) diff --git a/ceilometer/objectstore/swift.py b/ceilometer/objectstore/swift.py index 4f255c97e..7849ea051 100644 --- a/ceilometer/objectstore/swift.py +++ b/ceilometer/objectstore/swift.py @@ -20,8 +20,6 @@ from __future__ import absolute_import -import abc - from oslo.config import cfg from swiftclient import client as swift from keystoneclient import exceptions @@ -49,59 +47,6 @@ cfg.CONF.register_opts(OPTS) class _Base(plugin.PollsterBase): - __metaclass__ = abc.ABCMeta - - @abc.abstractmethod - def _iter_accounts(ksclient, cache): - """Iterate over all accounts, yielding (tenant_id, stats) tuples.""" - - def get_counters(self, manager, cache): - for tenant, account in self._iter_accounts(manager.keystone, cache): - yield counter.Counter( - name='storage.objects', - type=counter.TYPE_GAUGE, - volume=int(account['x-account-object-count']), - unit='object', - user_id=None, - project_id=tenant, - resource_id=tenant, - timestamp=timeutils.isotime(), - resource_metadata=None, - ) - yield counter.Counter( - name='storage.objects.size', - type=counter.TYPE_GAUGE, - volume=int(account['x-account-bytes-used']), - unit='B', - user_id=None, - project_id=tenant, - resource_id=tenant, - timestamp=timeutils.isotime(), - resource_metadata=None, - ) - yield counter.Counter( - name='storage.objects.containers', - type=counter.TYPE_GAUGE, - volume=int(account['x-account-container-count']), - unit='container', - user_id=None, - project_id=tenant, - resource_id=tenant, - timestamp=timeutils.isotime(), - resource_metadata=None, - ) - - -class SwiftPollster(_Base): - """Iterate over all accounts, using keystone. - """ - - @staticmethod - def get_counter_names(): - return ['storage.objects', - 'storage.objects.size', - 'storage.objects.containers'] - CACHE_KEY_TENANT = 'tenants' CACHE_KEY_HEAD = 'swift.head_account' @@ -123,13 +68,13 @@ class SwiftPollster(_Base): raise StopIteration() for t in cache['tenants']: - yield (t.id, swift.head_account(SwiftPollster. - _neaten_url(endpoint, t.id), + yield (t.id, swift.head_account(self._neaten_url(endpoint, t.id), ksclient.auth_token)) - # Transform the registered url to standard and valid format. @staticmethod def _neaten_url(endpoint, tenant_id): + """Transform the registered url to standard and valid format. + """ path = 'v1/' + cfg.CONF.reseller_prefix + tenant_id @@ -138,3 +83,72 @@ class SwiftPollster(_Base): endpoint = endpoint[:-1] return urljoin(endpoint, path) + + +class ObjectsPollster(_Base): + """Iterate over all accounts, using keystone. + """ + + @staticmethod + def get_counter_names(): + return ['storage.objects'] + + def get_counters(self, manager, cache): + for tenant, account in self._iter_accounts(manager.keystone, cache): + yield counter.Counter( + name='storage.objects', + type=counter.TYPE_GAUGE, + volume=int(account['x-account-object-count']), + unit='object', + user_id=None, + project_id=tenant, + resource_id=tenant, + timestamp=timeutils.isotime(), + resource_metadata=None, + ) + + +class ObjectsSizePollster(_Base): + """Iterate over all accounts, using keystone. + """ + + @staticmethod + def get_counter_names(): + return ['storage.objects.size'] + + def get_counters(self, manager, cache): + for tenant, account in self._iter_accounts(manager.keystone, cache): + yield counter.Counter( + name='storage.objects.size', + type=counter.TYPE_GAUGE, + volume=int(account['x-account-bytes-used']), + unit='B', + user_id=None, + project_id=tenant, + resource_id=tenant, + timestamp=timeutils.isotime(), + resource_metadata=None, + ) + + +class ObjectsContainersPollster(_Base): + """Iterate over all accounts, using keystone. + """ + + @staticmethod + def get_counter_names(): + return ['storage.objects.containers'] + + def get_counters(self, manager, cache): + for tenant, account in self._iter_accounts(manager.keystone, cache): + yield counter.Counter( + name='storage.objects.containers', + type=counter.TYPE_GAUGE, + volume=int(account['x-account-container-count']), + unit='container', + user_id=None, + project_id=tenant, + resource_id=tenant, + timestamp=timeutils.isotime(), + resource_metadata=None, + ) diff --git a/setup.cfg b/setup.cfg index e7c575adf..73d2945e7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -68,7 +68,9 @@ ceilometer.poll.central = network_floatingip = ceilometer.network.floatingip:FloatingIPPollster image = ceilometer.image.glance:ImagePollster image.size = ceilometer.image.glance:ImageSizePollster - objectstore = ceilometer.objectstore.swift:SwiftPollster + storage.objects = ceilometer.objectstore.swift:ObjectsPollster + storage.objects.size = ceilometer.objectstore.swift:ObjectsSizePollster + storage.objects.containers = ceilometer.objectstore.swift:ObjectsContainersPollster energy = ceilometer.energy.kwapi:EnergyPollster power = ceilometer.energy.kwapi:PowerPollster diff --git a/test-requirements.txt b/test-requirements.txt index 5a018c92b..dff1f1951 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,3 +22,4 @@ python-spidermonkey python-subunit testrepository>=0.0.13 testtools>=0.9.29 +testscenarios<0.5 diff --git a/tests/objectstore/test_swift.py b/tests/objectstore/test_swift.py index eadbc5fbf..8b50f875b 100644 --- a/tests/objectstore/test_swift.py +++ b/tests/objectstore/test_swift.py @@ -20,6 +20,7 @@ import collections import mock +import testscenarios from ceilometer.central import manager from ceilometer.objectstore import swift @@ -28,6 +29,7 @@ from ceilometer.tests import base from keystoneclient import exceptions from swiftclient import client as swift_client +load_tests = testscenarios.load_tests_apply_scenarios ACCOUNTS = [('tenant-000', {'x-account-object-count': 12, 'x-account-bytes-used': 321321321, @@ -48,6 +50,17 @@ class TestManager(manager.AgentManager): class TestSwiftPollster(base.TestCase): + # Define scenarios to run all of the tests against all of the + # pollsters. + scenarios = [ + ('storage.objects', + {'factory': swift.ObjectsPollster}), + ('storage.objects.size', + {'factory': swift.ObjectsSizePollster}), + ('storage.objects.containers', + {'factory': swift.ObjectsContainersPollster}), + ] + @staticmethod def fake_ks_service_catalog_url_for(*args, **kwargs): raise exceptions.EndpointNotFound("Fake keystone exception") @@ -59,13 +72,13 @@ class TestSwiftPollster(base.TestCase): @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock()) def setUp(self): super(TestSwiftPollster, self).setUp() - self.pollster = swift.SwiftPollster() + self.pollster = self.factory() self.manager = TestManager() def test_iter_accounts_no_cache(self): def empty_account_info(obj, ksclient, cache): return [] - self.stubs.Set(swift.SwiftPollster, '_get_account_info', + self.stubs.Set(self.factory, '_get_account_info', empty_account_info) cache = {} data = list(self.pollster._iter_accounts(mock.Mock(), cache)) @@ -83,7 +96,7 @@ class TestSwiftPollster(base.TestCase): ) self.stubs.Set(swift_client, 'head_account', ksclient) - self.stubs.Set(swift.SwiftPollster, '_neaten_url', + self.stubs.Set(self.factory, '_neaten_url', mock.Mock()) Tenant = collections.namedtuple('Tenant', 'id') cache = { @@ -99,23 +112,23 @@ class TestSwiftPollster(base.TestCase): standard_url = test_endpoint + '/v1/' + 'AUTH_' + test_tenant_id self.assertEqual(standard_url, - self.pollster._neaten_url(test_endpoint, - test_tenant_id)) + swift._Base._neaten_url(test_endpoint, + test_tenant_id)) self.assertEqual(standard_url, - self.pollster._neaten_url(test_endpoint + '/', - test_tenant_id)) + swift._Base._neaten_url(test_endpoint + '/', + test_tenant_id)) self.assertEqual(standard_url, - self.pollster._neaten_url(test_endpoint + '/v1', - test_tenant_id)) + swift._Base._neaten_url(test_endpoint + '/v1', + test_tenant_id)) def test_metering(self): - self.stubs.Set(swift.SwiftPollster, '_iter_accounts', + self.stubs.Set(self.factory, '_iter_accounts', self.fake_iter_accounts) counters = list(self.pollster.get_counters(self.manager, {})) - self.assertEqual(len(counters), 6) + self.assertEqual(len(counters), 2) def test_get_counter_names(self): - self.stubs.Set(swift.SwiftPollster, '_iter_accounts', + self.stubs.Set(self.factory, '_iter_accounts', self.fake_iter_accounts) counters = list(self.pollster.get_counters(self.manager, {})) self.assertEqual(set([c.name for c in counters]),