From a10773e41388140126c38baad7bf88f909736923 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 2 Jul 2013 17:08:49 -0400 Subject: [PATCH] Add pollster data cache Add a dictionary as a data cache for the pollsters, creating it outside of the loop where they are invoked and passing it to them. Future changesets will include changes to the pollsters to put data in the cache. blueprint one-meter-per-plugin Change-Id: Ie65526dfe65a8880ad8683b62fae62f8e7f9e69b Signed-off-by: Doug Hellmann --- ceilometer/central/manager.py | 8 ++++++-- ceilometer/compute/manager.py | 5 ++++- ceilometer/compute/nova_notifier.py | 6 ++++-- ceilometer/compute/plugin.py | 6 +++++- ceilometer/compute/pollsters.py | 8 ++++---- ceilometer/energy/kwapi.py | 2 +- ceilometer/image/glance.py | 2 +- ceilometer/network/floatingip.py | 2 +- ceilometer/objectstore/swift.py | 2 +- ceilometer/plugin.py | 8 +++++++- nova_tests/test_notifier.py | 2 +- tests/agentbase.py | 6 +++--- tests/compute/test_pollsters.py | 8 ++++---- tests/energy/test_kwapi.py | 6 +++--- tests/image/test_glance.py | 4 ++-- tests/network/test_floatingip.py | 4 ++-- tests/objectstore/test_swift.py | 6 +++--- 17 files changed, 52 insertions(+), 33 deletions(-) diff --git a/ceilometer/central/manager.py b/ceilometer/central/manager.py index d2ea27a9c..094fc11e5 100644 --- a/ceilometer/central/manager.py +++ b/ceilometer/central/manager.py @@ -37,11 +37,15 @@ class PollingTask(agent.PollingTask): with self.publish_context as publisher: # TODO(yjiang5) passing counters into get_counters to avoid # polling all counters one by one + cache = {} for pollster in self.pollsters: try: LOG.info("Polling pollster %s", pollster.name) - publisher(list(pollster.obj.get_counters( - self.manager))) + counters = list(pollster.obj.get_counters( + self.manager, + cache, + )) + publisher(counters) except Exception as err: LOG.warning('Continue after error from %s: %s', pollster.name, err) diff --git a/ceilometer/compute/manager.py b/ceilometer/compute/manager.py index fbedeffb0..65b2b3637 100644 --- a/ceilometer/compute/manager.py +++ b/ceilometer/compute/manager.py @@ -37,12 +37,15 @@ class PollingTask(agent.PollingTask): if getattr(instance, 'OS-EXT-STS:vm_state', None) != 'error': # TODO(yjiang5) passing counters to get_counters to avoid # polling all counters one by one + cache = {} for pollster in self.pollsters: try: LOG.info("Polling pollster %s", pollster.name) publisher(list(pollster.obj.get_counters( self.manager, - instance))) + cache, + instance, + ))) except Exception as err: LOG.warning('Continue after error from %s: %s', pollster.name, err) diff --git a/ceilometer/compute/nova_notifier.py b/ceilometer/compute/nova_notifier.py index 4bdc2e28c..887f56c7b 100644 --- a/ceilometer/compute/nova_notifier.py +++ b/ceilometer/compute/nova_notifier.py @@ -55,12 +55,14 @@ class DeletedInstanceStatsGatherer(object): self.mgr = extensions self.inspector = inspector.get_hypervisor_inspector() - def _get_counters_from_plugin(self, ext, instance, *args, **kwds): + def _get_counters_from_plugin(self, ext, cache, instance, *args, **kwds): """Used with the extenaion manager map() method.""" - return ext.obj.get_counters(self, instance) + return ext.obj.get_counters(self, cache, instance) def __call__(self, instance): + cache = {} counters = self.mgr.map(self._get_counters_from_plugin, + cache=cache, instance=instance, ) # counters is a list of lists, so flatten it before returning diff --git a/ceilometer/compute/plugin.py b/ceilometer/compute/plugin.py index 6a7c6c7dd..cadbd40a3 100644 --- a/ceilometer/compute/plugin.py +++ b/ceilometer/compute/plugin.py @@ -30,6 +30,10 @@ class ComputePollster(plugin.PollsterBase): __metaclass__ = abc.ABCMeta @abc.abstractmethod - def get_counters(self, manager, context): + def get_counters(self, manager, cache, instance): """Return a sequence of Counter instances from polling the resources. + + :param manager: The service manager invoking the plugin + :param cache: A dictionary for passing data between plugins + :param instance: The instance to examine """ diff --git a/ceilometer/compute/pollsters.py b/ceilometer/compute/pollsters.py index 45d8c71e8..aa333eabd 100644 --- a/ceilometer/compute/pollsters.py +++ b/ceilometer/compute/pollsters.py @@ -57,7 +57,7 @@ class InstancePollster(plugin.ComputePollster): # variable. We don't need such format in future return ['instance', 'instance:*'] - def get_counters(self, manager, instance): + def get_counters(self, manager, cache, instance): yield make_counter_from_instance(instance, name='instance', type=counter.TYPE_GAUGE, @@ -91,7 +91,7 @@ class DiskIOPollster(plugin.ComputePollster): 'disk.write.requests', 'disk.write.bytes'] - def get_counters(self, manager, instance): + def get_counters(self, manager, cache, instance): instance_name = _instance_name(instance) try: r_bytes = 0 @@ -164,7 +164,7 @@ class CPUPollster(plugin.ComputePollster): def get_counter_names(): return ['cpu', 'cpu_util'] - def get_counters(self, manager, instance): + def get_counters(self, manager, cache, instance): self.LOG.info('checking instance %s', instance.id) instance_name = _instance_name(instance) try: @@ -237,7 +237,7 @@ class NetPollster(plugin.ComputePollster): 'network.outgoing.bytes', 'network.outgoing.packets'] - def get_counters(self, manager, instance): + def get_counters(self, manager, cache, instance): instance_name = _instance_name(instance) self.LOG.info('checking instance %s', instance.id) try: diff --git a/ceilometer/energy/kwapi.py b/ceilometer/energy/kwapi.py index a171d6fb3..131ce03db 100644 --- a/ceilometer/energy/kwapi.py +++ b/ceilometer/energy/kwapi.py @@ -77,7 +77,7 @@ class KwapiPollster(_Base): def get_counter_names(): return ['energy', 'power'] - def get_counters(self, manager): + def get_counters(self, manager, cache): """Returns all counters.""" for probe in self.iter_probes(manager.keystone): yield counter.Counter( diff --git a/ceilometer/image/glance.py b/ceilometer/image/glance.py index 52e699857..e96a9a239 100644 --- a/ceilometer/image/glance.py +++ b/ceilometer/image/glance.py @@ -99,7 +99,7 @@ class ImagePollster(_Base): def get_counter_names(): return ['image', 'image.size'] - def get_counters(self, manager): + def get_counters(self, manager, cache): for image in self.iter_images(manager.keystone): yield counter.Counter( name='image', diff --git a/ceilometer/network/floatingip.py b/ceilometer/network/floatingip.py index 530333ca1..41ca6a200 100644 --- a/ceilometer/network/floatingip.py +++ b/ceilometer/network/floatingip.py @@ -35,7 +35,7 @@ class FloatingIPPollster(plugin.CentralPollster): def get_counter_names(): return ['ip.floating'] - def get_counters(self, manager): + def get_counters(self, manager, cache): nv = nova_client.Client() for ip in nv.floating_ip_get_all(): self.LOG.info("FLOATING IP USAGE: %s" % ip.ip) diff --git a/ceilometer/objectstore/swift.py b/ceilometer/objectstore/swift.py index 0dc6e8fa3..79e58647c 100644 --- a/ceilometer/objectstore/swift.py +++ b/ceilometer/objectstore/swift.py @@ -55,7 +55,7 @@ class _Base(plugin.PollsterBase): def iter_accounts(ksclient): """Iterate over all accounts, yielding (tenant_id, stats) tuples.""" - def get_counters(self, manager): + def get_counters(self, manager, cache): for tenant, account in self.iter_accounts(manager.keystone): yield counter.Counter( name='storage.objects', diff --git a/ceilometer/plugin.py b/ceilometer/plugin.py index 32f0c7339..bcd51d1b3 100644 --- a/ceilometer/plugin.py +++ b/ceilometer/plugin.py @@ -88,7 +88,13 @@ class PollsterBase(PluginBase): """Return a sequence of Counter names supported by the pollster.""" @abc.abstractmethod - def get_counters(self, manager, instance): + def get_counters(self, manager, cache): """Return a sequence of Counter instances from polling the resources. + :param manager: The service manager class invoking the plugin. + :param cache: A dictionary to allow pollsters to pass data + between themselves when recomputing it would be + expensive (e.g., asking another service for a + list of objects). + """ diff --git a/nova_tests/test_notifier.py b/nova_tests/test_notifier.py index 2df68d083..d383b408c 100644 --- a/nova_tests/test_notifier.py +++ b/nova_tests/test_notifier.py @@ -77,7 +77,7 @@ class TestNovaNotifier(base.TestCase): }, ) - def get_counters(self, manager, instance): + def get_counters(self, manager, cache, instance): self.instances.append((manager, instance)) return [self.test_data] diff --git a/tests/agentbase.py b/tests/agentbase.py index 18a1f8c37..cd5aa885a 100644 --- a/tests/agentbase.py +++ b/tests/agentbase.py @@ -52,17 +52,17 @@ class TestPollster: def get_counter_names(self): return [self.test_data.name] - def get_counters(self, manager, instance=None): + def get_counters(self, manager, cache, instance=None): self.counters.append((manager, instance)) return [self.test_data] class TestPollsterException(TestPollster): - def get_counters(self, manager, instance=None): + def get_counters(self, manager, cache, instance=None): # Put an instance parameter here so that it can be used # by both central manager and compute manager # In future, we possibly don't need such hack if we - # combin the get_counters() function again + # combine the get_counters() function again self.counters.append((manager, instance)) raise Exception() diff --git a/tests/compute/test_pollsters.py b/tests/compute/test_pollsters.py index fa57376d2..4afc1ea1d 100644 --- a/tests/compute/test_pollsters.py +++ b/tests/compute/test_pollsters.py @@ -56,7 +56,7 @@ class TestInstancePollster(TestPollsterBase): mgr = manager.AgentManager() pollster = pollsters.InstancePollster() - counters = list(pollster.get_counters(mgr, self.instance)) + counters = list(pollster.get_counters(mgr, {}, self.instance)) self.assertEquals(len(counters), 2) self.assertEqual(counters[0].name, 'instance') self.assertEqual(counters[1].name, 'instance:m1.small') @@ -80,7 +80,7 @@ class TestDiskIOPollster(TestPollsterBase): mgr = manager.AgentManager() pollster = pollsters.DiskIOPollster() - counters = list(pollster.get_counters(mgr, self.instance)) + counters = list(pollster.get_counters(mgr, {}, self.instance)) assert counters self.assertEqual(set([c.name for c in counters]), @@ -143,7 +143,7 @@ class TestNetPollster(TestPollsterBase): mgr = manager.AgentManager() pollster = pollsters.NetPollster() - counters = list(pollster.get_counters(mgr, self.instance)) + counters = list(pollster.get_counters(mgr, {}, self.instance)) assert counters self.assertEqual(set([c.name for c in counters]), set(pollster.get_counter_names())) @@ -203,7 +203,7 @@ class TestCPUPollster(TestPollsterBase): pollster = pollsters.CPUPollster() def _verify_cpu_metering(zero, expected_time): - counters = list(pollster.get_counters(mgr, self.instance)) + counters = list(pollster.get_counters(mgr, {}, self.instance)) self.assertEquals(len(counters), 2) self.assertEqual(set([c.name for c in counters]), set(pollster.get_counter_names())) diff --git a/tests/energy/test_kwapi.py b/tests/energy/test_kwapi.py index d58fb9793..3a1d4dc2a 100644 --- a/tests/energy/test_kwapi.py +++ b/tests/energy/test_kwapi.py @@ -76,13 +76,13 @@ class TestKwapiPollster(base.TestCase): self.stubs.Set(kwapi._Base, 'get_kwapi_client', self.fake_kwapi_get_kwapi_client) - counters = list(kwapi.KwapiPollster().get_counters(self.manager)) + counters = list(kwapi.KwapiPollster().get_counters(self.manager, {})) self.assertEqual(len(counters), 0) def test_kwapi_counter(self): self.stubs.Set(kwapi._Base, 'iter_probes', self.fake_kwapi_iter_probes) - counters = list(kwapi.KwapiPollster().get_counters(self.manager)) + counters = list(kwapi.KwapiPollster().get_counters(self.manager, {})) self.assertEqual(len(counters), 6) energy_counters = [counter for counter in counters if counter.name == "energy"] @@ -104,6 +104,6 @@ class TestKwapiPollster(base.TestCase): def test_kwapi_counter_list(self): self.stubs.Set(kwapi._Base, 'iter_probes', self.fake_kwapi_iter_probes) - counters = list(kwapi.KwapiPollster().get_counters(self.manager)) + counters = list(kwapi.KwapiPollster().get_counters(self.manager, {})) self.assertEqual(set([c.name for c in counters]), set(kwapi.KwapiPollster().get_counter_names())) diff --git a/tests/image/test_glance.py b/tests/image/test_glance.py index 593646d4a..f1618b8b6 100644 --- a/tests/image/test_glance.py +++ b/tests/image/test_glance.py @@ -140,7 +140,7 @@ class TestImagePollster(base.TestCase): self.assertEqual(len(images), len(set(image.id for image in images))) def test_glance_image_counter(self): - counters = list(glance.ImagePollster().get_counters(self.manager)) + counters = list(glance.ImagePollster().get_counters(self.manager, {})) self.assertEqual(len(counters), 6) for counter in [c for c in counters if c.name == 'image']: self.assertEqual(counter.volume, 1) @@ -150,6 +150,6 @@ class TestImagePollster(base.TestCase): counters))) def test_get_counter_names(self): - counters = list(glance.ImagePollster().get_counters(self.manager)) + counters = list(glance.ImagePollster().get_counters(self.manager, {})) self.assertEqual(set([c.name for c in counters]), set(glance.ImagePollster().get_counter_names())) diff --git a/tests/network/test_floatingip.py b/tests/network/test_floatingip.py index bcabb02a6..e54a49437 100644 --- a/tests/network/test_floatingip.py +++ b/tests/network/test_floatingip.py @@ -65,7 +65,7 @@ class TestFloatingIPPollster(base.TestCase): # assert False, 'Should have seen an error' def test_get_counters_not_empty(self): - counters = list(self.pollster.get_counters(self.manager)) + counters = list(self.pollster.get_counters(self.manager, {})) self.assertEqual(len(counters), 3) # It's necessary to verify all the attributes extracted by Nova # API /os-floating-ips to make sure they're available and correct. @@ -82,6 +82,6 @@ class TestFloatingIPPollster(base.TestCase): self.assertEqual(counters[2].resource_metadata["pool"], "public") def test_get_counter_names(self): - counters = list(self.pollster.get_counters(self.manager)) + counters = list(self.pollster.get_counters(self.manager, {})) self.assertEqual(set([c.name for c in counters]), set(self.pollster.get_counter_names())) diff --git a/tests/objectstore/test_swift.py b/tests/objectstore/test_swift.py index 7fc2b950d..ab5b133d5 100644 --- a/tests/objectstore/test_swift.py +++ b/tests/objectstore/test_swift.py @@ -77,18 +77,18 @@ class TestSwiftPollster(base.TestCase): def test_objectstore_metering(self): self.stubs.Set(swift.SwiftPollster, 'iter_accounts', self.fake_iter_accounts) - counters = list(self.pollster.get_counters(self.manager)) + counters = list(self.pollster.get_counters(self.manager, {})) self.assertEqual(len(counters), 6) def test_objectstore_get_counter_names(self): self.stubs.Set(swift.SwiftPollster, 'iter_accounts', self.fake_iter_accounts) - counters = list(self.pollster.get_counters(self.manager)) + counters = list(self.pollster.get_counters(self.manager, {})) self.assertEqual(set([c.name for c in counters]), set(self.pollster.get_counter_names())) def test_objectstore_endpoint_notfound(self): self.stubs.Set(self.manager.keystone.service_catalog, 'url_for', self.fake_ks_service_catalog_url_for) - counters = list(self.pollster.get_counters(self.manager)) + counters = list(self.pollster.get_counters(self.manager, {})) self.assertEqual(len(counters), 0)