Set page size when Glance API request is called

The glance related pollsters in central agent uses glanceclient to get
image-list from glance. Since glanceclient has pagination feature,
it calls glance API for every page sizes. When glanceclient is used by
Central Agent, default value of page size, that is 20, is used, since
Central Agent doesn't specify page size. However, default value of page
size is too small when there is a large number of images.

Therefore we need to specify page size from Central Agent.

Change-Id: I5f6457234a2963bbe567d078f2c8bb098c8a311c
Closes-bug: #1340577

ceilometer/image/glance.py

@@ -28,6 +28,20 @@ from ceilometer.openstack.common import timeutils
 from ceilometer import sample
 
 
+OPTS = [
+    cfg.IntOpt('glance_page_size',
+               default=0,
+               help="Number of items to request in "
+                    "each paginated Glance API request "
+                    "(parameter used by glancecelient). "
+                    "If this is less than or equal to 0, "
+                    "page size is not specified "
+                    "(default value in glanceclient is used)."),
+]
+
+cfg.CONF.register_opts(OPTS)
+
+
 class _Base(plugin.CentralPollster):
 
     @staticmethod
@@ -45,13 +59,15 @@ class _Base(plugin.CentralPollster):
 
     def _get_images(self, ksclient):
         client = self.get_glance_client(ksclient)
-        # TODO(eglynn): use pagination to protect against unbounded
+        page_size = cfg.CONF.glance_page_size
-        #              memory usage
+        kwargs = {}
+        if page_size > 0:
+            kwargs['page_size'] = page_size
         rawImageList = list(itertools.chain(
-            client.images.list(filters={"is_public": True}),
+            client.images.list(filters={"is_public": True}, **kwargs),
             # TODO(eglynn): extend glance API with all_tenants logic to
             #              avoid second call to retrieve private images
-            client.images.list(filters={"is_public": False})))
+            client.images.list(filters={"is_public": False}, **kwargs)))
 
         # When retrieving images from glance, glance will check
         # whether the user is of 'admin_role' which is

ceilometer/tests/image/test_glance.py

@@ -20,6 +20,7 @@ import mock
 from ceilometer.central import manager
 from ceilometer.image import glance
 from ceilometer.openstack.common import context
+from ceilometer.openstack.common.fixture import config
 from ceilometer.openstack.common.fixture import mockpatch
 from ceilometer.openstack.common import test
 
@@ -108,6 +109,11 @@ class _BaseObject(object):
     pass
 
 
+class FakeGlanceClient(object):
+    class images(object):
+        pass
+
+
 class TestManager(manager.AgentManager):
 
     def __init__(self):
@@ -115,6 +121,44 @@ class TestManager(manager.AgentManager):
         self.keystone = mock.Mock()
 
 
+class TestImagePollsterPageSize(test.BaseTestCase):
+
+    def fake_get_glance_client(self, ksclient):
+        glanceclient = FakeGlanceClient()
+        glanceclient.images.list = mock.MagicMock(return_value=IMAGE_LIST)
+        return glanceclient
+
+    @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
+    def setUp(self):
+        super(TestImagePollsterPageSize, self).setUp()
+        self.context = context.get_admin_context()
+        self.manager = TestManager()
+        self.useFixture(mockpatch.PatchObject(
+            glance._Base, 'get_glance_client',
+            side_effect=self.fake_get_glance_client))
+        self.CONF = self.useFixture(config.Config()).conf
+
+    def _do_test_iter_images(self, page_size=0):
+        self.CONF.set_override("glance_page_size", page_size)
+        images = list(glance.ImagePollster().
+                      _iter_images(self.manager.keystone, {}))
+        kwargs = {}
+        if page_size > 0:
+            kwargs['page_size'] = page_size
+        FakeGlanceClient.images.list.assert_called_with(
+            filters={'is_public': False}, **kwargs)
+        self.assertEqual(len(set(image.id for image in images)), len(images))
+
+    def test_page_size(self):
+        self._do_test_iter_images(100)
+
+    def test_page_size_default(self):
+        self._do_test_iter_images()
+
+    def test_page_size_negative_number(self):
+        self._do_test_iter_images(-1)
+
+
 class TestImagePollster(test.BaseTestCase):
 
     def fake_get_glance_client(self, ksclient):

doc/source/configuration.rst

@@ -43,6 +43,12 @@ database_connection              mongodb://localhost:27017/ceilometer  Database
 metering_api_port                8777                                  The port for the ceilometer API server
 reseller_prefix                  AUTH\_                                Prefix used by swift for reseller token
 nova_http_log_debug              False                                 Log request/response parameters between nova and ceilometer
+glance_page_size                 0                                     Number of items to request in each paginated Glance API
+                                                                       request (parameter used by glancecelient). If this is less
+                                                                       than or equal to 0, page size is not specified (default value
+                                                                       in glanceclient is used). It is better to check and set
+                                                                       appropriate value in line with each environment when calling
+                                                                       glanceclient, than to define higher default value.
 ===============================  ====================================  ==============================================================
 
 Service polling authentication