Add pollster data cache

Add a dictionary as a data cache for the pollsters,
creating it outside of the loop where they are
invoked and passing it to them. Future changesets will
include changes to the pollsters to put data in the
cache.

blueprint one-meter-per-plugin

Change-Id: Ie65526dfe65a8880ad8683b62fae62f8e7f9e69b
Signed-off-by: Doug Hellmann <doug.hellmann@dreamhost.com>

ceilometer/central/manager.py

@@ -37,11 +37,15 @@ class PollingTask(agent.PollingTask):
         with self.publish_context as publisher:
             # TODO(yjiang5) passing counters into get_counters to avoid
             # polling all counters one by one
+            cache = {}
             for pollster in self.pollsters:
                 try:
                     LOG.info("Polling pollster %s", pollster.name)
-                    publisher(list(pollster.obj.get_counters(
+                    counters = list(pollster.obj.get_counters(
-                        self.manager)))
+                        self.manager,
+                        cache,
+                    ))
+                    publisher(counters)
                 except Exception as err:
                     LOG.warning('Continue after error from %s: %s',
                                 pollster.name, err)

ceilometer/compute/manager.py

@@ -37,12 +37,15 @@ class PollingTask(agent.PollingTask):
                 if getattr(instance, 'OS-EXT-STS:vm_state', None) != 'error':
                     # TODO(yjiang5) passing counters to get_counters to avoid
                     # polling all counters one by one
+                    cache = {}
                     for pollster in self.pollsters:
                         try:
                             LOG.info("Polling pollster %s", pollster.name)
                             publisher(list(pollster.obj.get_counters(
                                 self.manager,
-                                instance)))
+                                cache,
+                                instance,
+                            )))
                         except Exception as err:
                             LOG.warning('Continue after error from %s: %s',
                                         pollster.name, err)

ceilometer/compute/nova_notifier.py

@@ -55,12 +55,14 @@ class DeletedInstanceStatsGatherer(object):
         self.mgr = extensions
         self.inspector = inspector.get_hypervisor_inspector()
 
-    def _get_counters_from_plugin(self, ext, instance, *args, **kwds):
+    def _get_counters_from_plugin(self, ext, cache, instance, *args, **kwds):
         """Used with the extenaion manager map() method."""
-        return ext.obj.get_counters(self, instance)
+        return ext.obj.get_counters(self, cache, instance)
 
     def __call__(self, instance):
+        cache = {}
         counters = self.mgr.map(self._get_counters_from_plugin,
+                                cache=cache,
                                 instance=instance,
                                 )
         # counters is a list of lists, so flatten it before returning

ceilometer/compute/plugin.py

@@ -30,6 +30,10 @@ class ComputePollster(plugin.PollsterBase):
     __metaclass__ = abc.ABCMeta
 
     @abc.abstractmethod
-    def get_counters(self, manager, context):
+    def get_counters(self, manager, cache, instance):
         """Return a sequence of Counter instances from polling the resources.
+
+        :param manager: The service manager invoking the plugin
+        :param cache: A dictionary for passing data between plugins
+        :param instance: The instance to examine
         """

ceilometer/compute/pollsters.py

@@ -57,7 +57,7 @@ class InstancePollster(plugin.ComputePollster):
         # variable. We don't need such format in future
         return ['instance', 'instance:*']
 
-    def get_counters(self, manager, instance):
+    def get_counters(self, manager, cache, instance):
         yield make_counter_from_instance(instance,
                                          name='instance',
                                          type=counter.TYPE_GAUGE,
@@ -91,7 +91,7 @@ class DiskIOPollster(plugin.ComputePollster):
                 'disk.write.requests',
                 'disk.write.bytes']
 
-    def get_counters(self, manager, instance):
+    def get_counters(self, manager, cache, instance):
         instance_name = _instance_name(instance)
         try:
             r_bytes = 0
@@ -164,7 +164,7 @@ class CPUPollster(plugin.ComputePollster):
     def get_counter_names():
         return ['cpu', 'cpu_util']
 
-    def get_counters(self, manager, instance):
+    def get_counters(self, manager, cache, instance):
         self.LOG.info('checking instance %s', instance.id)
         instance_name = _instance_name(instance)
         try:
@@ -237,7 +237,7 @@ class NetPollster(plugin.ComputePollster):
                 'network.outgoing.bytes',
                 'network.outgoing.packets']
 
-    def get_counters(self, manager, instance):
+    def get_counters(self, manager, cache, instance):
         instance_name = _instance_name(instance)
         self.LOG.info('checking instance %s', instance.id)
         try:

ceilometer/energy/kwapi.py

@@ -77,7 +77,7 @@ class KwapiPollster(_Base):
     def get_counter_names():
         return ['energy', 'power']
 
-    def get_counters(self, manager):
+    def get_counters(self, manager, cache):
         """Returns all counters."""
         for probe in self.iter_probes(manager.keystone):
             yield counter.Counter(

ceilometer/image/glance.py

@@ -99,7 +99,7 @@ class ImagePollster(_Base):
     def get_counter_names():
         return ['image', 'image.size']
 
-    def get_counters(self, manager):
+    def get_counters(self, manager, cache):
         for image in self.iter_images(manager.keystone):
             yield counter.Counter(
                 name='image',

ceilometer/network/floatingip.py

@@ -35,7 +35,7 @@ class FloatingIPPollster(plugin.CentralPollster):
     def get_counter_names():
         return ['ip.floating']
 
-    def get_counters(self, manager):
+    def get_counters(self, manager, cache):
         nv = nova_client.Client()
         for ip in nv.floating_ip_get_all():
             self.LOG.info("FLOATING IP USAGE: %s" % ip.ip)

ceilometer/objectstore/swift.py

@@ -55,7 +55,7 @@ class _Base(plugin.PollsterBase):
     def iter_accounts(ksclient):
         """Iterate over all accounts, yielding (tenant_id, stats) tuples."""
 
-    def get_counters(self, manager):
+    def get_counters(self, manager, cache):
         for tenant, account in self.iter_accounts(manager.keystone):
             yield counter.Counter(
                 name='storage.objects',

ceilometer/plugin.py

@@ -88,7 +88,13 @@ class PollsterBase(PluginBase):
         """Return a sequence of Counter names supported by the pollster."""
 
     @abc.abstractmethod
-    def get_counters(self, manager, instance):
+    def get_counters(self, manager, cache):
         """Return a sequence of Counter instances from polling the resources.
 
+        :param manager: The service manager class invoking the plugin.
+        :param cache: A dictionary to allow pollsters to pass data
+                      between themselves when recomputing it would be
+                      expensive (e.g., asking another service for a
+                      list of objects).
+
         """

nova_tests/test_notifier.py

@@ -77,7 +77,7 @@ class TestNovaNotifier(base.TestCase):
                                },
         )
 
-        def get_counters(self, manager, instance):
+        def get_counters(self, manager, cache, instance):
             self.instances.append((manager, instance))
             return [self.test_data]
 

tests/agentbase.py

@@ -52,17 +52,17 @@ class TestPollster:
     def get_counter_names(self):
         return [self.test_data.name]
 
-    def get_counters(self, manager, instance=None):
+    def get_counters(self, manager, cache, instance=None):
         self.counters.append((manager, instance))
         return [self.test_data]
 
 
 class TestPollsterException(TestPollster):
-    def get_counters(self, manager, instance=None):
+    def get_counters(self, manager, cache, instance=None):
         # Put an instance parameter here so that it can be used
         # by both central manager and compute manager
         # In future, we possibly don't need such hack if we
-        # combin the get_counters() function again
+        # combine the get_counters() function again
         self.counters.append((manager, instance))
         raise Exception()
 

tests/compute/test_pollsters.py

@@ -56,7 +56,7 @@ class TestInstancePollster(TestPollsterBase):
 
         mgr = manager.AgentManager()
         pollster = pollsters.InstancePollster()
-        counters = list(pollster.get_counters(mgr, self.instance))
+        counters = list(pollster.get_counters(mgr, {}, self.instance))
         self.assertEquals(len(counters), 2)
         self.assertEqual(counters[0].name, 'instance')
         self.assertEqual(counters[1].name, 'instance:m1.small')
@@ -80,7 +80,7 @@ class TestDiskIOPollster(TestPollsterBase):
 
         mgr = manager.AgentManager()
         pollster = pollsters.DiskIOPollster()
-        counters = list(pollster.get_counters(mgr, self.instance))
+        counters = list(pollster.get_counters(mgr, {}, self.instance))
         assert counters
 
         self.assertEqual(set([c.name for c in counters]),
@@ -143,7 +143,7 @@ class TestNetPollster(TestPollsterBase):
 
         mgr = manager.AgentManager()
         pollster = pollsters.NetPollster()
-        counters = list(pollster.get_counters(mgr, self.instance))
+        counters = list(pollster.get_counters(mgr, {}, self.instance))
         assert counters
         self.assertEqual(set([c.name for c in counters]),
                          set(pollster.get_counter_names()))
@@ -203,7 +203,7 @@ class TestCPUPollster(TestPollsterBase):
         pollster = pollsters.CPUPollster()
 
         def _verify_cpu_metering(zero, expected_time):
-            counters = list(pollster.get_counters(mgr, self.instance))
+            counters = list(pollster.get_counters(mgr, {}, self.instance))
             self.assertEquals(len(counters), 2)
             self.assertEqual(set([c.name for c in counters]),
                              set(pollster.get_counter_names()))

tests/energy/test_kwapi.py

@@ -76,13 +76,13 @@ class TestKwapiPollster(base.TestCase):
         self.stubs.Set(kwapi._Base, 'get_kwapi_client',
                        self.fake_kwapi_get_kwapi_client)
 
-        counters = list(kwapi.KwapiPollster().get_counters(self.manager))
+        counters = list(kwapi.KwapiPollster().get_counters(self.manager, {}))
         self.assertEqual(len(counters), 0)
 
     def test_kwapi_counter(self):
         self.stubs.Set(kwapi._Base, 'iter_probes', self.fake_kwapi_iter_probes)
 
-        counters = list(kwapi.KwapiPollster().get_counters(self.manager))
+        counters = list(kwapi.KwapiPollster().get_counters(self.manager, {}))
         self.assertEqual(len(counters), 6)
         energy_counters = [counter for counter in counters
                            if counter.name == "energy"]
@@ -104,6 +104,6 @@ class TestKwapiPollster(base.TestCase):
     def test_kwapi_counter_list(self):
         self.stubs.Set(kwapi._Base, 'iter_probes', self.fake_kwapi_iter_probes)
 
-        counters = list(kwapi.KwapiPollster().get_counters(self.manager))
+        counters = list(kwapi.KwapiPollster().get_counters(self.manager, {}))
         self.assertEqual(set([c.name for c in counters]),
                          set(kwapi.KwapiPollster().get_counter_names()))

tests/image/test_glance.py

@@ -140,7 +140,7 @@ class TestImagePollster(base.TestCase):
         self.assertEqual(len(images), len(set(image.id for image in images)))
 
     def test_glance_image_counter(self):
-        counters = list(glance.ImagePollster().get_counters(self.manager))
+        counters = list(glance.ImagePollster().get_counters(self.manager, {}))
         self.assertEqual(len(counters), 6)
         for counter in [c for c in counters if c.name == 'image']:
             self.assertEqual(counter.volume, 1)
@@ -150,6 +150,6 @@ class TestImagePollster(base.TestCase):
                         counters)))
 
     def test_get_counter_names(self):
-        counters = list(glance.ImagePollster().get_counters(self.manager))
+        counters = list(glance.ImagePollster().get_counters(self.manager, {}))
         self.assertEqual(set([c.name for c in counters]),
                          set(glance.ImagePollster().get_counter_names()))

tests/network/test_floatingip.py

@@ -65,7 +65,7 @@ class TestFloatingIPPollster(base.TestCase):
     #         assert False, 'Should have seen an error'
 
     def test_get_counters_not_empty(self):
-        counters = list(self.pollster.get_counters(self.manager))
+        counters = list(self.pollster.get_counters(self.manager, {}))
         self.assertEqual(len(counters), 3)
         # It's necessary to verify all the attributes extracted by Nova
         # API /os-floating-ips to make sure they're available and correct.
@@ -82,6 +82,6 @@ class TestFloatingIPPollster(base.TestCase):
         self.assertEqual(counters[2].resource_metadata["pool"], "public")
 
     def test_get_counter_names(self):
-        counters = list(self.pollster.get_counters(self.manager))
+        counters = list(self.pollster.get_counters(self.manager, {}))
         self.assertEqual(set([c.name for c in counters]),
                          set(self.pollster.get_counter_names()))

tests/objectstore/test_swift.py

@@ -77,18 +77,18 @@ class TestSwiftPollster(base.TestCase):
     def test_objectstore_metering(self):
         self.stubs.Set(swift.SwiftPollster, 'iter_accounts',
                        self.fake_iter_accounts)
-        counters = list(self.pollster.get_counters(self.manager))
+        counters = list(self.pollster.get_counters(self.manager, {}))
         self.assertEqual(len(counters), 6)
 
     def test_objectstore_get_counter_names(self):
         self.stubs.Set(swift.SwiftPollster, 'iter_accounts',
                        self.fake_iter_accounts)
-        counters = list(self.pollster.get_counters(self.manager))
+        counters = list(self.pollster.get_counters(self.manager, {}))
         self.assertEqual(set([c.name for c in counters]),
                          set(self.pollster.get_counter_names()))
 
     def test_objectstore_endpoint_notfound(self):
         self.stubs.Set(self.manager.keystone.service_catalog, 'url_for',
                        self.fake_ks_service_catalog_url_for)
-        counters = list(self.pollster.get_counters(self.manager))
+        counters = list(self.pollster.get_counters(self.manager, {}))
         self.assertEqual(len(counters), 0)