Merge "Add pollster data cache"

ceilometer/central/manager.py

@@ -37,11 +37,15 @@ class PollingTask(agent.PollingTask):
         with self.publish_context as publisher:
             # TODO(yjiang5) passing counters into get_counters to avoid
             # polling all counters one by one
+            cache = {}
             for pollster in self.pollsters:
                 try:
                     LOG.info("Polling pollster %s", pollster.name)
-                    publisher(list(pollster.obj.get_counters(
+                    counters = list(pollster.obj.get_counters(
-                        self.manager)))
+                        self.manager,
+                        cache,
+                    ))
+                    publisher(counters)
                 except Exception as err:
                     LOG.warning('Continue after error from %s: %s',
                                 pollster.name, err)

ceilometer/compute/manager.py

@@ -37,12 +37,15 @@ class PollingTask(agent.PollingTask):
                 if getattr(instance, 'OS-EXT-STS:vm_state', None) != 'error':
                     # TODO(yjiang5) passing counters to get_counters to avoid
                     # polling all counters one by one
+                    cache = {}
                     for pollster in self.pollsters:
                         try:
                             LOG.info("Polling pollster %s", pollster.name)
                             publisher(list(pollster.obj.get_counters(
                                 self.manager,
-                                instance)))
+                                cache,
+                                instance,
+                            )))
                         except Exception as err:
                             LOG.warning('Continue after error from %s: %s',
                                         pollster.name, err)

ceilometer/compute/nova_notifier.py

@@ -55,12 +55,14 @@ class DeletedInstanceStatsGatherer(object):
         self.mgr = extensions
         self.inspector = inspector.get_hypervisor_inspector()
 
-    def _get_counters_from_plugin(self, ext, instance, *args, **kwds):
+    def _get_counters_from_plugin(self, ext, cache, instance, *args, **kwds):
         """Used with the extenaion manager map() method."""
-        return ext.obj.get_counters(self, instance)
+        return ext.obj.get_counters(self, cache, instance)
 
     def __call__(self, instance):
+        cache = {}
         counters = self.mgr.map(self._get_counters_from_plugin,
+                                cache=cache,
                                 instance=instance,
                                 )
         # counters is a list of lists, so flatten it before returning

ceilometer/compute/plugin.py

@@ -30,6 +30,10 @@ class ComputePollster(plugin.PollsterBase):
     __metaclass__ = abc.ABCMeta
 
     @abc.abstractmethod
-    def get_counters(self, manager, context):
+    def get_counters(self, manager, cache, instance):
         """Return a sequence of Counter instances from polling the resources.
+
+        :param manager: The service manager invoking the plugin
+        :param cache: A dictionary for passing data between plugins
+        :param instance: The instance to examine
         """

ceilometer/compute/pollsters.py

@@ -57,7 +57,7 @@ class InstancePollster(plugin.ComputePollster):
         # variable. We don't need such format in future
         return ['instance', 'instance:*']
 
-    def get_counters(self, manager, instance):
+    def get_counters(self, manager, cache, instance):
         yield make_counter_from_instance(instance,
                                          name='instance',
                                          type=counter.TYPE_GAUGE,
@@ -91,7 +91,7 @@ class DiskIOPollster(plugin.ComputePollster):
                 'disk.write.requests',
                 'disk.write.bytes']
 
-    def get_counters(self, manager, instance):
+    def get_counters(self, manager, cache, instance):
         instance_name = _instance_name(instance)
         try:
             r_bytes = 0
@@ -164,7 +164,7 @@ class CPUPollster(plugin.ComputePollster):
     def get_counter_names():
         return ['cpu', 'cpu_util']
 
-    def get_counters(self, manager, instance):
+    def get_counters(self, manager, cache, instance):
         self.LOG.info('checking instance %s', instance.id)
         instance_name = _instance_name(instance)
         try:
@@ -237,7 +237,7 @@ class NetPollster(plugin.ComputePollster):
                 'network.outgoing.bytes',
                 'network.outgoing.packets']
 
-    def get_counters(self, manager, instance):
+    def get_counters(self, manager, cache, instance):
         instance_name = _instance_name(instance)
         self.LOG.info('checking instance %s', instance.id)
         try:

ceilometer/energy/kwapi.py

@@ -77,7 +77,7 @@ class KwapiPollster(_Base):
     def get_counter_names():
         return ['energy', 'power']
 
-    def get_counters(self, manager):
+    def get_counters(self, manager, cache):
         """Returns all counters."""
         for probe in self.iter_probes(manager.keystone):
             yield counter.Counter(

ceilometer/image/glance.py

@@ -99,7 +99,7 @@ class ImagePollster(_Base):
     def get_counter_names():
         return ['image', 'image.size']
 
-    def get_counters(self, manager):
+    def get_counters(self, manager, cache):
         for image in self.iter_images(manager.keystone):
             yield counter.Counter(
                 name='image',

ceilometer/network/floatingip.py

@@ -35,7 +35,7 @@ class FloatingIPPollster(plugin.CentralPollster):
     def get_counter_names():
         return ['ip.floating']
 
-    def get_counters(self, manager):
+    def get_counters(self, manager, cache):
         nv = nova_client.Client()
         for ip in nv.floating_ip_get_all():
             self.LOG.info("FLOATING IP USAGE: %s" % ip.ip)

ceilometer/objectstore/swift.py

@@ -55,7 +55,7 @@ class _Base(plugin.PollsterBase):
     def iter_accounts(ksclient):
         """Iterate over all accounts, yielding (tenant_id, stats) tuples."""
 
-    def get_counters(self, manager):
+    def get_counters(self, manager, cache):
         for tenant, account in self.iter_accounts(manager.keystone):
             yield counter.Counter(
                 name='storage.objects',

ceilometer/plugin.py

@@ -88,7 +88,13 @@ class PollsterBase(PluginBase):
         """Return a sequence of Counter names supported by the pollster."""
 
     @abc.abstractmethod
-    def get_counters(self, manager, instance):
+    def get_counters(self, manager, cache):
         """Return a sequence of Counter instances from polling the resources.
 
+        :param manager: The service manager class invoking the plugin.
+        :param cache: A dictionary to allow pollsters to pass data
+                      between themselves when recomputing it would be
+                      expensive (e.g., asking another service for a
+                      list of objects).
+
         """

nova_tests/test_notifier.py

@@ -77,7 +77,7 @@ class TestNovaNotifier(base.TestCase):
                                },
         )
 
-        def get_counters(self, manager, instance):
+        def get_counters(self, manager, cache, instance):
             self.instances.append((manager, instance))
             return [self.test_data]
 

tests/agentbase.py

@@ -52,17 +52,17 @@ class TestPollster:
     def get_counter_names(self):
         return [self.test_data.name]
 
-    def get_counters(self, manager, instance=None):
+    def get_counters(self, manager, cache, instance=None):
         self.counters.append((manager, instance))
         return [self.test_data]
 
 
 class TestPollsterException(TestPollster):
-    def get_counters(self, manager, instance=None):
+    def get_counters(self, manager, cache, instance=None):
         # Put an instance parameter here so that it can be used
         # by both central manager and compute manager
         # In future, we possibly don't need such hack if we
-        # combin the get_counters() function again
+        # combine the get_counters() function again
         self.counters.append((manager, instance))
         raise Exception()
 

tests/compute/test_pollsters.py

@@ -56,7 +56,7 @@ class TestInstancePollster(TestPollsterBase):
 
         mgr = manager.AgentManager()
         pollster = pollsters.InstancePollster()
-        counters = list(pollster.get_counters(mgr, self.instance))
+        counters = list(pollster.get_counters(mgr, {}, self.instance))
         self.assertEquals(len(counters), 2)
         self.assertEqual(counters[0].name, 'instance')
         self.assertEqual(counters[1].name, 'instance:m1.small')
@@ -80,7 +80,7 @@ class TestDiskIOPollster(TestPollsterBase):
 
         mgr = manager.AgentManager()
         pollster = pollsters.DiskIOPollster()
-        counters = list(pollster.get_counters(mgr, self.instance))
+        counters = list(pollster.get_counters(mgr, {}, self.instance))
         assert counters
 
         self.assertEqual(set([c.name for c in counters]),
@@ -143,7 +143,7 @@ class TestNetPollster(TestPollsterBase):
 
         mgr = manager.AgentManager()
         pollster = pollsters.NetPollster()
-        counters = list(pollster.get_counters(mgr, self.instance))
+        counters = list(pollster.get_counters(mgr, {}, self.instance))
         assert counters
         self.assertEqual(set([c.name for c in counters]),
                          set(pollster.get_counter_names()))
@@ -203,7 +203,7 @@ class TestCPUPollster(TestPollsterBase):
         pollster = pollsters.CPUPollster()
 
         def _verify_cpu_metering(zero, expected_time):
-            counters = list(pollster.get_counters(mgr, self.instance))
+            counters = list(pollster.get_counters(mgr, {}, self.instance))
             self.assertEquals(len(counters), 2)
             self.assertEqual(set([c.name for c in counters]),
                              set(pollster.get_counter_names()))

tests/energy/test_kwapi.py

@@ -76,13 +76,13 @@ class TestKwapiPollster(base.TestCase):
         self.stubs.Set(kwapi._Base, 'get_kwapi_client',
                        self.fake_kwapi_get_kwapi_client)
 
-        counters = list(kwapi.KwapiPollster().get_counters(self.manager))
+        counters = list(kwapi.KwapiPollster().get_counters(self.manager, {}))
         self.assertEqual(len(counters), 0)
 
     def test_kwapi_counter(self):
         self.stubs.Set(kwapi._Base, 'iter_probes', self.fake_kwapi_iter_probes)
 
-        counters = list(kwapi.KwapiPollster().get_counters(self.manager))
+        counters = list(kwapi.KwapiPollster().get_counters(self.manager, {}))
         self.assertEqual(len(counters), 6)
         energy_counters = [counter for counter in counters
                            if counter.name == "energy"]
@@ -104,6 +104,6 @@ class TestKwapiPollster(base.TestCase):
     def test_kwapi_counter_list(self):
         self.stubs.Set(kwapi._Base, 'iter_probes', self.fake_kwapi_iter_probes)
 
-        counters = list(kwapi.KwapiPollster().get_counters(self.manager))
+        counters = list(kwapi.KwapiPollster().get_counters(self.manager, {}))
         self.assertEqual(set([c.name for c in counters]),
                          set(kwapi.KwapiPollster().get_counter_names()))

tests/image/test_glance.py

@@ -140,7 +140,7 @@ class TestImagePollster(base.TestCase):
         self.assertEqual(len(images), len(set(image.id for image in images)))
 
     def test_glance_image_counter(self):
-        counters = list(glance.ImagePollster().get_counters(self.manager))
+        counters = list(glance.ImagePollster().get_counters(self.manager, {}))
         self.assertEqual(len(counters), 6)
         for counter in [c for c in counters if c.name == 'image']:
             self.assertEqual(counter.volume, 1)
@@ -150,6 +150,6 @@ class TestImagePollster(base.TestCase):
                         counters)))
 
     def test_get_counter_names(self):
-        counters = list(glance.ImagePollster().get_counters(self.manager))
+        counters = list(glance.ImagePollster().get_counters(self.manager, {}))
         self.assertEqual(set([c.name for c in counters]),
                          set(glance.ImagePollster().get_counter_names()))

tests/network/test_floatingip.py

@@ -65,7 +65,7 @@ class TestFloatingIPPollster(base.TestCase):
     #         assert False, 'Should have seen an error'
 
     def test_get_counters_not_empty(self):
-        counters = list(self.pollster.get_counters(self.manager))
+        counters = list(self.pollster.get_counters(self.manager, {}))
         self.assertEqual(len(counters), 3)
         # It's necessary to verify all the attributes extracted by Nova
         # API /os-floating-ips to make sure they're available and correct.
@@ -82,6 +82,6 @@ class TestFloatingIPPollster(base.TestCase):
         self.assertEqual(counters[2].resource_metadata["pool"], "public")
 
     def test_get_counter_names(self):
-        counters = list(self.pollster.get_counters(self.manager))
+        counters = list(self.pollster.get_counters(self.manager, {}))
         self.assertEqual(set([c.name for c in counters]),
                          set(self.pollster.get_counter_names()))

tests/objectstore/test_swift.py

@@ -77,18 +77,18 @@ class TestSwiftPollster(base.TestCase):
     def test_objectstore_metering(self):
         self.stubs.Set(swift.SwiftPollster, 'iter_accounts',
                        self.fake_iter_accounts)
-        counters = list(self.pollster.get_counters(self.manager))
+        counters = list(self.pollster.get_counters(self.manager, {}))
         self.assertEqual(len(counters), 6)
 
     def test_objectstore_get_counter_names(self):
         self.stubs.Set(swift.SwiftPollster, 'iter_accounts',
                        self.fake_iter_accounts)
-        counters = list(self.pollster.get_counters(self.manager))
+        counters = list(self.pollster.get_counters(self.manager, {}))
         self.assertEqual(set([c.name for c in counters]),
                          set(self.pollster.get_counter_names()))
 
     def test_objectstore_endpoint_notfound(self):
         self.stubs.Set(self.manager.keystone.service_catalog, 'url_for',
                        self.fake_ks_service_catalog_url_for)
-        counters = list(self.pollster.get_counters(self.manager))
+        counters = list(self.pollster.get_counters(self.manager, {}))
         self.assertEqual(len(counters), 0)