diff --git a/ceilometer/central/manager.py b/ceilometer/central/manager.py index d2ea27a9c..094fc11e5 100644 --- a/ceilometer/central/manager.py +++ b/ceilometer/central/manager.py @@ -37,11 +37,15 @@ class PollingTask(agent.PollingTask): with self.publish_context as publisher: # TODO(yjiang5) passing counters into get_counters to avoid # polling all counters one by one + cache = {} for pollster in self.pollsters: try: LOG.info("Polling pollster %s", pollster.name) - publisher(list(pollster.obj.get_counters( - self.manager))) + counters = list(pollster.obj.get_counters( + self.manager, + cache, + )) + publisher(counters) except Exception as err: LOG.warning('Continue after error from %s: %s', pollster.name, err) diff --git a/ceilometer/compute/manager.py b/ceilometer/compute/manager.py index fbedeffb0..65b2b3637 100644 --- a/ceilometer/compute/manager.py +++ b/ceilometer/compute/manager.py @@ -37,12 +37,15 @@ class PollingTask(agent.PollingTask): if getattr(instance, 'OS-EXT-STS:vm_state', None) != 'error': # TODO(yjiang5) passing counters to get_counters to avoid # polling all counters one by one + cache = {} for pollster in self.pollsters: try: LOG.info("Polling pollster %s", pollster.name) publisher(list(pollster.obj.get_counters( self.manager, - instance))) + cache, + instance, + ))) except Exception as err: LOG.warning('Continue after error from %s: %s', pollster.name, err) diff --git a/ceilometer/compute/nova_notifier.py b/ceilometer/compute/nova_notifier.py index 4bdc2e28c..887f56c7b 100644 --- a/ceilometer/compute/nova_notifier.py +++ b/ceilometer/compute/nova_notifier.py @@ -55,12 +55,14 @@ class DeletedInstanceStatsGatherer(object): self.mgr = extensions self.inspector = inspector.get_hypervisor_inspector() - def _get_counters_from_plugin(self, ext, instance, *args, **kwds): + def _get_counters_from_plugin(self, ext, cache, instance, *args, **kwds): """Used with the extenaion manager map() method.""" - return ext.obj.get_counters(self, instance) + return ext.obj.get_counters(self, cache, instance) def __call__(self, instance): + cache = {} counters = self.mgr.map(self._get_counters_from_plugin, + cache=cache, instance=instance, ) # counters is a list of lists, so flatten it before returning diff --git a/ceilometer/compute/plugin.py b/ceilometer/compute/plugin.py index 6a7c6c7dd..cadbd40a3 100644 --- a/ceilometer/compute/plugin.py +++ b/ceilometer/compute/plugin.py @@ -30,6 +30,10 @@ class ComputePollster(plugin.PollsterBase): __metaclass__ = abc.ABCMeta @abc.abstractmethod - def get_counters(self, manager, context): + def get_counters(self, manager, cache, instance): """Return a sequence of Counter instances from polling the resources. + + :param manager: The service manager invoking the plugin + :param cache: A dictionary for passing data between plugins + :param instance: The instance to examine """ diff --git a/ceilometer/compute/pollsters.py b/ceilometer/compute/pollsters.py index 45d8c71e8..aa333eabd 100644 --- a/ceilometer/compute/pollsters.py +++ b/ceilometer/compute/pollsters.py @@ -57,7 +57,7 @@ class InstancePollster(plugin.ComputePollster): # variable. We don't need such format in future return ['instance', 'instance:*'] - def get_counters(self, manager, instance): + def get_counters(self, manager, cache, instance): yield make_counter_from_instance(instance, name='instance', type=counter.TYPE_GAUGE, @@ -91,7 +91,7 @@ class DiskIOPollster(plugin.ComputePollster): 'disk.write.requests', 'disk.write.bytes'] - def get_counters(self, manager, instance): + def get_counters(self, manager, cache, instance): instance_name = _instance_name(instance) try: r_bytes = 0 @@ -164,7 +164,7 @@ class CPUPollster(plugin.ComputePollster): def get_counter_names(): return ['cpu', 'cpu_util'] - def get_counters(self, manager, instance): + def get_counters(self, manager, cache, instance): self.LOG.info('checking instance %s', instance.id) instance_name = _instance_name(instance) try: @@ -237,7 +237,7 @@ class NetPollster(plugin.ComputePollster): 'network.outgoing.bytes', 'network.outgoing.packets'] - def get_counters(self, manager, instance): + def get_counters(self, manager, cache, instance): instance_name = _instance_name(instance) self.LOG.info('checking instance %s', instance.id) try: diff --git a/ceilometer/energy/kwapi.py b/ceilometer/energy/kwapi.py index a171d6fb3..131ce03db 100644 --- a/ceilometer/energy/kwapi.py +++ b/ceilometer/energy/kwapi.py @@ -77,7 +77,7 @@ class KwapiPollster(_Base): def get_counter_names(): return ['energy', 'power'] - def get_counters(self, manager): + def get_counters(self, manager, cache): """Returns all counters.""" for probe in self.iter_probes(manager.keystone): yield counter.Counter( diff --git a/ceilometer/image/glance.py b/ceilometer/image/glance.py index 52e699857..e96a9a239 100644 --- a/ceilometer/image/glance.py +++ b/ceilometer/image/glance.py @@ -99,7 +99,7 @@ class ImagePollster(_Base): def get_counter_names(): return ['image', 'image.size'] - def get_counters(self, manager): + def get_counters(self, manager, cache): for image in self.iter_images(manager.keystone): yield counter.Counter( name='image', diff --git a/ceilometer/network/floatingip.py b/ceilometer/network/floatingip.py index 530333ca1..41ca6a200 100644 --- a/ceilometer/network/floatingip.py +++ b/ceilometer/network/floatingip.py @@ -35,7 +35,7 @@ class FloatingIPPollster(plugin.CentralPollster): def get_counter_names(): return ['ip.floating'] - def get_counters(self, manager): + def get_counters(self, manager, cache): nv = nova_client.Client() for ip in nv.floating_ip_get_all(): self.LOG.info("FLOATING IP USAGE: %s" % ip.ip) diff --git a/ceilometer/objectstore/swift.py b/ceilometer/objectstore/swift.py index 0dc6e8fa3..79e58647c 100644 --- a/ceilometer/objectstore/swift.py +++ b/ceilometer/objectstore/swift.py @@ -55,7 +55,7 @@ class _Base(plugin.PollsterBase): def iter_accounts(ksclient): """Iterate over all accounts, yielding (tenant_id, stats) tuples.""" - def get_counters(self, manager): + def get_counters(self, manager, cache): for tenant, account in self.iter_accounts(manager.keystone): yield counter.Counter( name='storage.objects', diff --git a/ceilometer/plugin.py b/ceilometer/plugin.py index 32f0c7339..bcd51d1b3 100644 --- a/ceilometer/plugin.py +++ b/ceilometer/plugin.py @@ -88,7 +88,13 @@ class PollsterBase(PluginBase): """Return a sequence of Counter names supported by the pollster.""" @abc.abstractmethod - def get_counters(self, manager, instance): + def get_counters(self, manager, cache): """Return a sequence of Counter instances from polling the resources. + :param manager: The service manager class invoking the plugin. + :param cache: A dictionary to allow pollsters to pass data + between themselves when recomputing it would be + expensive (e.g., asking another service for a + list of objects). + """ diff --git a/nova_tests/test_notifier.py b/nova_tests/test_notifier.py index 2df68d083..d383b408c 100644 --- a/nova_tests/test_notifier.py +++ b/nova_tests/test_notifier.py @@ -77,7 +77,7 @@ class TestNovaNotifier(base.TestCase): }, ) - def get_counters(self, manager, instance): + def get_counters(self, manager, cache, instance): self.instances.append((manager, instance)) return [self.test_data] diff --git a/tests/agentbase.py b/tests/agentbase.py index 18a1f8c37..cd5aa885a 100644 --- a/tests/agentbase.py +++ b/tests/agentbase.py @@ -52,17 +52,17 @@ class TestPollster: def get_counter_names(self): return [self.test_data.name] - def get_counters(self, manager, instance=None): + def get_counters(self, manager, cache, instance=None): self.counters.append((manager, instance)) return [self.test_data] class TestPollsterException(TestPollster): - def get_counters(self, manager, instance=None): + def get_counters(self, manager, cache, instance=None): # Put an instance parameter here so that it can be used # by both central manager and compute manager # In future, we possibly don't need such hack if we - # combin the get_counters() function again + # combine the get_counters() function again self.counters.append((manager, instance)) raise Exception() diff --git a/tests/compute/test_pollsters.py b/tests/compute/test_pollsters.py index fa57376d2..4afc1ea1d 100644 --- a/tests/compute/test_pollsters.py +++ b/tests/compute/test_pollsters.py @@ -56,7 +56,7 @@ class TestInstancePollster(TestPollsterBase): mgr = manager.AgentManager() pollster = pollsters.InstancePollster() - counters = list(pollster.get_counters(mgr, self.instance)) + counters = list(pollster.get_counters(mgr, {}, self.instance)) self.assertEquals(len(counters), 2) self.assertEqual(counters[0].name, 'instance') self.assertEqual(counters[1].name, 'instance:m1.small') @@ -80,7 +80,7 @@ class TestDiskIOPollster(TestPollsterBase): mgr = manager.AgentManager() pollster = pollsters.DiskIOPollster() - counters = list(pollster.get_counters(mgr, self.instance)) + counters = list(pollster.get_counters(mgr, {}, self.instance)) assert counters self.assertEqual(set([c.name for c in counters]), @@ -143,7 +143,7 @@ class TestNetPollster(TestPollsterBase): mgr = manager.AgentManager() pollster = pollsters.NetPollster() - counters = list(pollster.get_counters(mgr, self.instance)) + counters = list(pollster.get_counters(mgr, {}, self.instance)) assert counters self.assertEqual(set([c.name for c in counters]), set(pollster.get_counter_names())) @@ -203,7 +203,7 @@ class TestCPUPollster(TestPollsterBase): pollster = pollsters.CPUPollster() def _verify_cpu_metering(zero, expected_time): - counters = list(pollster.get_counters(mgr, self.instance)) + counters = list(pollster.get_counters(mgr, {}, self.instance)) self.assertEquals(len(counters), 2) self.assertEqual(set([c.name for c in counters]), set(pollster.get_counter_names())) diff --git a/tests/energy/test_kwapi.py b/tests/energy/test_kwapi.py index d58fb9793..3a1d4dc2a 100644 --- a/tests/energy/test_kwapi.py +++ b/tests/energy/test_kwapi.py @@ -76,13 +76,13 @@ class TestKwapiPollster(base.TestCase): self.stubs.Set(kwapi._Base, 'get_kwapi_client', self.fake_kwapi_get_kwapi_client) - counters = list(kwapi.KwapiPollster().get_counters(self.manager)) + counters = list(kwapi.KwapiPollster().get_counters(self.manager, {})) self.assertEqual(len(counters), 0) def test_kwapi_counter(self): self.stubs.Set(kwapi._Base, 'iter_probes', self.fake_kwapi_iter_probes) - counters = list(kwapi.KwapiPollster().get_counters(self.manager)) + counters = list(kwapi.KwapiPollster().get_counters(self.manager, {})) self.assertEqual(len(counters), 6) energy_counters = [counter for counter in counters if counter.name == "energy"] @@ -104,6 +104,6 @@ class TestKwapiPollster(base.TestCase): def test_kwapi_counter_list(self): self.stubs.Set(kwapi._Base, 'iter_probes', self.fake_kwapi_iter_probes) - counters = list(kwapi.KwapiPollster().get_counters(self.manager)) + counters = list(kwapi.KwapiPollster().get_counters(self.manager, {})) self.assertEqual(set([c.name for c in counters]), set(kwapi.KwapiPollster().get_counter_names())) diff --git a/tests/image/test_glance.py b/tests/image/test_glance.py index 593646d4a..f1618b8b6 100644 --- a/tests/image/test_glance.py +++ b/tests/image/test_glance.py @@ -140,7 +140,7 @@ class TestImagePollster(base.TestCase): self.assertEqual(len(images), len(set(image.id for image in images))) def test_glance_image_counter(self): - counters = list(glance.ImagePollster().get_counters(self.manager)) + counters = list(glance.ImagePollster().get_counters(self.manager, {})) self.assertEqual(len(counters), 6) for counter in [c for c in counters if c.name == 'image']: self.assertEqual(counter.volume, 1) @@ -150,6 +150,6 @@ class TestImagePollster(base.TestCase): counters))) def test_get_counter_names(self): - counters = list(glance.ImagePollster().get_counters(self.manager)) + counters = list(glance.ImagePollster().get_counters(self.manager, {})) self.assertEqual(set([c.name for c in counters]), set(glance.ImagePollster().get_counter_names())) diff --git a/tests/network/test_floatingip.py b/tests/network/test_floatingip.py index bcabb02a6..e54a49437 100644 --- a/tests/network/test_floatingip.py +++ b/tests/network/test_floatingip.py @@ -65,7 +65,7 @@ class TestFloatingIPPollster(base.TestCase): # assert False, 'Should have seen an error' def test_get_counters_not_empty(self): - counters = list(self.pollster.get_counters(self.manager)) + counters = list(self.pollster.get_counters(self.manager, {})) self.assertEqual(len(counters), 3) # It's necessary to verify all the attributes extracted by Nova # API /os-floating-ips to make sure they're available and correct. @@ -82,6 +82,6 @@ class TestFloatingIPPollster(base.TestCase): self.assertEqual(counters[2].resource_metadata["pool"], "public") def test_get_counter_names(self): - counters = list(self.pollster.get_counters(self.manager)) + counters = list(self.pollster.get_counters(self.manager, {})) self.assertEqual(set([c.name for c in counters]), set(self.pollster.get_counter_names())) diff --git a/tests/objectstore/test_swift.py b/tests/objectstore/test_swift.py index 7fc2b950d..ab5b133d5 100644 --- a/tests/objectstore/test_swift.py +++ b/tests/objectstore/test_swift.py @@ -77,18 +77,18 @@ class TestSwiftPollster(base.TestCase): def test_objectstore_metering(self): self.stubs.Set(swift.SwiftPollster, 'iter_accounts', self.fake_iter_accounts) - counters = list(self.pollster.get_counters(self.manager)) + counters = list(self.pollster.get_counters(self.manager, {})) self.assertEqual(len(counters), 6) def test_objectstore_get_counter_names(self): self.stubs.Set(swift.SwiftPollster, 'iter_accounts', self.fake_iter_accounts) - counters = list(self.pollster.get_counters(self.manager)) + counters = list(self.pollster.get_counters(self.manager, {})) self.assertEqual(set([c.name for c in counters]), set(self.pollster.get_counter_names())) def test_objectstore_endpoint_notfound(self): self.stubs.Set(self.manager.keystone.service_catalog, 'url_for', self.fake_ks_service_catalog_url_for) - counters = list(self.pollster.get_counters(self.manager)) + counters = list(self.pollster.get_counters(self.manager, {})) self.assertEqual(len(counters), 0)