Update swift pollster to use cache

Use the pollster cache to store the list of tenants
and the information swift has about them.

blueprint one-meter-per-plugin

Change-Id: I2c1fbbb99b2ae4796720ec6b7ac94ac27ab9ba5c
Signed-off-by: Doug Hellmann <doug.hellmann@dreamhost.com>

ceilometer/objectstore/swift.py

@@ -27,6 +27,7 @@ from swiftclient import client as swift
 from keystoneclient import exceptions
 
 from ceilometer import counter
+from ceilometer.openstack.common.gettextutils import _
 from ceilometer.openstack.common import log
 from ceilometer.openstack.common import timeutils
 from ceilometer import plugin
@@ -50,13 +51,12 @@ class _Base(plugin.PollsterBase):
 
     __metaclass__ = abc.ABCMeta
 
-    @staticmethod
     @abc.abstractmethod
-    def iter_accounts(ksclient):
+    def _iter_accounts(ksclient, cache):
         """Iterate over all accounts, yielding (tenant_id, stats) tuples."""
 
     def get_counters(self, manager, cache):
-        for tenant, account in self.iter_accounts(manager.keystone):
+        for tenant, account in self._iter_accounts(manager.keystone, cache):
             yield counter.Counter(
                 name='storage.objects',
                 type=counter.TYPE_GAUGE,
@@ -102,17 +102,27 @@ class SwiftPollster(_Base):
                 'storage.objects.size',
                 'storage.objects.containers']
 
-    @staticmethod
+    CACHE_KEY_TENANT = 'tenants'
-    def iter_accounts(ksclient):
+    CACHE_KEY_HEAD = 'swift.head_account'
+
+    def _iter_accounts(self, ksclient, cache):
+        if self.CACHE_KEY_TENANT not in cache:
+            cache[self.CACHE_KEY_TENANT] = ksclient.tenants.list()
+        if self.CACHE_KEY_HEAD not in cache:
+            cache[self.CACHE_KEY_HEAD] = list(self._get_account_info(ksclient,
+                                                                     cache))
+        return iter(cache['swift.head_account'])
+
+    def _get_account_info(self, ksclient, cache):
         try:
             endpoint = ksclient.service_catalog.url_for(
                 service_type='object-store',
                 endpoint_type='adminURL')
         except exceptions.EndpointNotFound:
             LOG.debug(_("Swift endpoint not found"))
-            return
+            raise StopIteration()
 
-        for t in ksclient.tenants.list():
+        for t in cache['tenants']:
             yield (t.id, swift.head_account(SwiftPollster.
                                             _neaten_url(endpoint, t.id),
                                             ksclient.auth_token))

tests/objectstore/test_swift.py

@@ -17,6 +17,8 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import collections
+
 import mock
 
 from ceilometer.central import manager
@@ -24,6 +26,8 @@ from ceilometer.objectstore import swift
 from ceilometer.tests import base
 
 from keystoneclient import exceptions
+from swiftclient import client as swift_client
+
 
 ACCOUNTS = [('tenant-000', {'x-account-object-count': 12,
                             'x-account-bytes-used': 321321321,
@@ -48,8 +52,7 @@ class TestSwiftPollster(base.TestCase):
     def fake_ks_service_catalog_url_for(*args, **kwargs):
         raise exceptions.EndpointNotFound("Fake keystone exception")
 
-    @staticmethod
+    def fake_iter_accounts(self, ksclient, cache):
-    def fake_iter_accounts(self, ksclient):
         for i in ACCOUNTS:
             yield i
 
@@ -59,7 +62,38 @@ class TestSwiftPollster(base.TestCase):
         self.pollster = swift.SwiftPollster()
         self.manager = TestManager()
 
-    def test_objectstore_neaten_url(self):
+    def test_iter_accounts_no_cache(self):
+        def empty_account_info(obj, ksclient, cache):
+            return []
+        self.stubs.Set(swift.SwiftPollster, '_get_account_info',
+                       empty_account_info)
+        cache = {}
+        data = list(self.pollster._iter_accounts(mock.Mock(), cache))
+        self.assertTrue(self.pollster.CACHE_KEY_TENANT in cache)
+        self.assertTrue(self.pollster.CACHE_KEY_HEAD in cache)
+        self.assertEqual(data, [])
+
+    def test_iter_accounts_tenants_cached(self):
+        # Verify that if there are tenants pre-cached then the account
+        # info loop iterates over those instead of asking for the list
+        # again.
+        ksclient = mock.Mock()
+        ksclient.tenants.list.side_effect = AssertionError(
+            'should not be called',
+        )
+        self.stubs.Set(swift_client, 'head_account',
+                       ksclient)
+        self.stubs.Set(swift.SwiftPollster, '_neaten_url',
+                       mock.Mock())
+        Tenant = collections.namedtuple('Tenant', 'id')
+        cache = {
+            self.pollster.CACHE_KEY_TENANT: [Tenant(ACCOUNTS[0][0])],
+        }
+        data = list(self.pollster._iter_accounts(mock.Mock(), cache))
+        self.assertTrue(self.pollster.CACHE_KEY_HEAD in cache)
+        self.assertEqual(data[0][0], ACCOUNTS[0][0])
+
+    def test_neaten_url(self):
         test_endpoint = 'http://127.0.0.1:8080'
         test_tenant_id = 'a7fd1695fa154486a647e44aa99a1b9b'
         standard_url = test_endpoint + '/v1/' + 'AUTH_' + test_tenant_id
@@ -74,20 +108,20 @@ class TestSwiftPollster(base.TestCase):
                          self.pollster._neaten_url(test_endpoint + '/v1',
                                                    test_tenant_id))
 
-    def test_objectstore_metering(self):
+    def test_metering(self):
-        self.stubs.Set(swift.SwiftPollster, 'iter_accounts',
+        self.stubs.Set(swift.SwiftPollster, '_iter_accounts',
                        self.fake_iter_accounts)
         counters = list(self.pollster.get_counters(self.manager, {}))
         self.assertEqual(len(counters), 6)
 
-    def test_objectstore_get_counter_names(self):
+    def test_get_counter_names(self):
-        self.stubs.Set(swift.SwiftPollster, 'iter_accounts',
+        self.stubs.Set(swift.SwiftPollster, '_iter_accounts',
                        self.fake_iter_accounts)
         counters = list(self.pollster.get_counters(self.manager, {}))
         self.assertEqual(set([c.name for c in counters]),
                          set(self.pollster.get_counter_names()))
 
-    def test_objectstore_endpoint_notfound(self):
+    def test_endpoint_notfound(self):
         self.stubs.Set(self.manager.keystone.service_catalog, 'url_for',
                        self.fake_ks_service_catalog_url_for)
         counters = list(self.pollster.get_counters(self.manager, {}))