From 7b9cc2a5ff76041442301c8b6c057f5d26428012 Mon Sep 17 00:00:00 2001 From: Vladislav Kuzmin Date: Tue, 12 Nov 2013 06:26:33 -0600 Subject: [PATCH] Add new meters for swift This patch adds new meters: -storage.containers.objects -storage.containers.objects.size. These meters are implemented through a polling Swift blueprint: https://wiki.openstack.org/wiki/Ceilometer/blueprints/pollster-swift Change-Id: Ic1f3aaf351db0b99392eb13d4f31f2732f87f816 --- ceilometer/objectstore/swift.py | 66 +++++++++++++++++++--- ceilometer/tests/objectstore/test_swift.py | 57 ++++++++++++++----- doc/source/measurements.rst | 22 ++++---- setup.cfg | 2 + 4 files changed, 115 insertions(+), 32 deletions(-) diff --git a/ceilometer/objectstore/swift.py b/ceilometer/objectstore/swift.py index f96a7497c..b72ba9275 100644 --- a/ceilometer/objectstore/swift.py +++ b/ceilometer/objectstore/swift.py @@ -47,15 +47,19 @@ cfg.CONF.register_opts(OPTS) class _Base(plugin.PollsterBase): CACHE_KEY_TENANT = 'tenants' - CACHE_KEY_HEAD = 'swift.head_account' + METHOD = 'head' + + @property + def CACHE_KEY_METHOD(self): + return 'swift.%s_account' % self.METHOD def _iter_accounts(self, ksclient, cache): if self.CACHE_KEY_TENANT not in cache: cache[self.CACHE_KEY_TENANT] = ksclient.tenants.list() - if self.CACHE_KEY_HEAD not in cache: - cache[self.CACHE_KEY_HEAD] = list(self._get_account_info(ksclient, - cache)) - return iter(cache[self.CACHE_KEY_HEAD]) + if self.CACHE_KEY_METHOD not in cache: + cache[self.CACHE_KEY_METHOD] = list(self._get_account_info( + ksclient, cache)) + return iter(cache[self.CACHE_KEY_METHOD]) def _get_account_info(self, ksclient, cache): try: @@ -67,8 +71,10 @@ class _Base(plugin.PollsterBase): raise StopIteration() for t in cache[self.CACHE_KEY_TENANT]: - yield (t.id, swift.head_account(self._neaten_url(endpoint, t.id), - ksclient.auth_token)) + api_method = '%s_account' % self.METHOD + yield (t.id, getattr(swift, api_method) + (self._neaten_url(endpoint, t.id), + ksclient.auth_token)) @staticmethod def _neaten_url(endpoint, tenant_id): @@ -133,3 +139,49 @@ class ObjectsContainersPollster(_Base): timestamp=timeutils.isotime(), resource_metadata=None, ) + + +class ContainersObjectsPollster(_Base): + """Get info about containers using Swift API + """ + + METHOD = 'get' + + def get_samples(self, manager, cache): + for project, account in self._iter_accounts(manager.keystone, cache): + containers_info = account[1] + for container in containers_info: + yield sample.Sample( + name='storage.containers.objects', + type=sample.TYPE_GAUGE, + volume=int(container['count']), + unit='object', + user_id=None, + project_id=project, + resource_id=project + '/' + container['name'], + timestamp=timeutils.isotime(), + resource_metadata=None, + ) + + +class ContainersSizePollster(_Base): + """Get info about containers using Swift API + """ + + METHOD = 'get' + + def get_samples(self, manager, cache): + for project, account in self._iter_accounts(manager.keystone, cache): + containers_info = account[1] + for container in containers_info: + yield sample.Sample( + name='storage.containers.objects.size', + type=sample.TYPE_GAUGE, + volume=int(container['bytes']), + unit='B', + user_id=None, + project_id=project, + resource_id=project + '/' + container['name'], + timestamp=timeutils.isotime(), + resource_metadata=None, + ) diff --git a/ceilometer/tests/objectstore/test_swift.py b/ceilometer/tests/objectstore/test_swift.py index 72bfe0c05..f014e6958 100644 --- a/ceilometer/tests/objectstore/test_swift.py +++ b/ceilometer/tests/objectstore/test_swift.py @@ -31,14 +31,30 @@ from ceilometer.openstack.common import test load_tests = testscenarios.load_tests_apply_scenarios -ACCOUNTS = [('tenant-000', {'x-account-object-count': 12, - 'x-account-bytes-used': 321321321, - 'x-account-container-count': 7, - }), - ('tenant-001', {'x-account-object-count': 34, - 'x-account-bytes-used': 9898989898, - 'x-account-container-count': 17, - })] +HEAD_ACCOUNTS = [('tenant-000', {'x-account-object-count': 12, + 'x-account-bytes-used': 321321321, + 'x-account-container-count': 7, + }), + ('tenant-001', {'x-account-object-count': 34, + 'x-account-bytes-used': 9898989898, + 'x-account-container-count': 17, + })] + +GET_ACCOUNTS = [('tenant-002', ({'x-account-object-count': 10, + 'x-account-bytes-used': 123123, + 'x-account-container-count': 2, + }, + [{'count': 10, + 'bytes': 123123, + 'name': 'my_container'}, + {'count': 0, + 'bytes': 0, + 'name': 'new_container' + }])), + ('tenant-003', ({'x-account-object-count': 0, + 'x-account-bytes-used': 0, + 'x-account-container-count': 0, + }, [])), ] class TestManager(manager.AgentManager): @@ -59,6 +75,10 @@ class TestSwiftPollster(test.BaseTestCase): {'factory': swift.ObjectsSizePollster}), ('storage.objects.containers', {'factory': swift.ObjectsContainersPollster}), + ('storage.containers.objects', + {'factory': swift.ContainersObjectsPollster}), + ('storage.containers.objects.size', + {'factory': swift.ContainersSizePollster}), ] @staticmethod @@ -66,7 +86,7 @@ class TestSwiftPollster(test.BaseTestCase): raise exceptions.EndpointNotFound("Fake keystone exception") def fake_iter_accounts(self, ksclient, cache): - for i in ACCOUNTS: + for i in self.ACCOUNTS: yield i @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock()) @@ -75,6 +95,11 @@ class TestSwiftPollster(test.BaseTestCase): self.pollster = self.factory() self.manager = TestManager() + if self.pollster.CACHE_KEY_METHOD == 'swift.head_account': + self.ACCOUNTS = HEAD_ACCOUNTS + else: + self.ACCOUNTS = GET_ACCOUNTS + def test_iter_accounts_no_cache(self): cache = {} with PatchObject(self.factory, '_get_account_info', @@ -82,7 +107,7 @@ class TestSwiftPollster(test.BaseTestCase): data = list(self.pollster._iter_accounts(mock.Mock(), cache)) self.assertTrue(self.pollster.CACHE_KEY_TENANT in cache) - self.assertTrue(self.pollster.CACHE_KEY_HEAD in cache) + self.assertTrue(self.pollster.CACHE_KEY_METHOD in cache) self.assertEqual(data, []) def test_iter_accounts_tenants_cached(self): @@ -94,16 +119,18 @@ class TestSwiftPollster(test.BaseTestCase): 'should not be called', ) - with PatchObject(swift_client, 'head_account', new=ksclient): + api_method = '%s_account' % self.pollster.METHOD + with PatchObject(swift_client, api_method, new=ksclient): with PatchObject(self.factory, '_neaten_url'): Tenant = collections.namedtuple('Tenant', 'id') cache = { - self.pollster.CACHE_KEY_TENANT: [Tenant(ACCOUNTS[0][0])], + self.pollster.CACHE_KEY_TENANT: [ + Tenant(self.ACCOUNTS[0][0]) + ], } data = list(self.pollster._iter_accounts(mock.Mock(), cache)) - - self.assertTrue(self.pollster.CACHE_KEY_HEAD in cache) - self.assertEqual(data[0][0], ACCOUNTS[0][0]) + self.assertTrue(self.pollster.CACHE_KEY_METHOD in cache) + self.assertEqual(data[0][0], self.ACCOUNTS[0][0]) def test_neaten_url(self): test_endpoint = 'http://127.0.0.1:8080' diff --git a/doc/source/measurements.rst b/doc/source/measurements.rst index 29df3b35d..467f58474 100644 --- a/doc/source/measurements.rst +++ b/doc/source/measurements.rst @@ -144,16 +144,18 @@ Make sure Cinder is properly configured first: see :ref:`installing_manually`. Object Storage (Swift) ====================== -============================== ========== ========== ======== ============ ============================================== -Name Type Volume Resource Origin Note -============================== ========== ========== ======== ============ ============================================== -storage.objects Gauge object store ID pollster Number of objects -storage.objects.size Gauge B store ID pollster Total size of stored objects -storage.objects.containers Gauge container store ID pollster Number of containers -storage.objects.incoming.bytes Delta B store ID notification Number of incoming bytes -storage.objects.outgoing.bytes Delta B store ID notification Number of outgoing bytes -storage.api.request Delta request store ID notification Number of API requests against swift -============================== ========== ========== ======== ============ ============================================== +=============================== ========== ========== =========== ============ ========================================== +Name Type Volume Resource Origin Note +=============================== ========== ========== =========== ============ ========================================== +storage.objects Gauge object store ID pollster Number of objects +storage.objects.size Gauge B store ID pollster Total size of stored objects +storage.objects.containers Gauge container store ID pollster Number of containers +storage.objects.incoming.bytes Delta B store ID notification Number of incoming bytes +storage.objects.outgoing.bytes Delta B store ID notification Number of outgoing bytes +storage.api.request Delta request store ID notification Number of API requests against swift +storage.containers.objects Gauge object str ID/cont pollster Number of objects in container +storage.containers.objects.size Gauge B str ID/cont pollster Total size of stored objects in container +=============================== ========== ========== =========== ============ ========================================== In order to use storage.objects.incoming.bytes and storage.outgoing.bytes, one must configure Swift as described in :ref:`installing_manually`. Note that they may not be diff --git a/setup.cfg b/setup.cfg index 7b029dd47..8ddca0e89 100644 --- a/setup.cfg +++ b/setup.cfg @@ -70,6 +70,8 @@ ceilometer.poll.central = ip.floating = ceilometer.network.floatingip:FloatingIPPollster image = ceilometer.image.glance:ImagePollster image.size = ceilometer.image.glance:ImageSizePollster + storage.containers.objects = ceilometer.objectstore.swift:ContainersObjectsPollster + storage.containers.objects.size = ceilometer.objectstore.swift:ContainersSizePollster storage.objects = ceilometer.objectstore.swift:ObjectsPollster storage.objects.size = ceilometer.objectstore.swift:ObjectsSizePollster storage.objects.containers = ceilometer.objectstore.swift:ObjectsContainersPollster