Adds pagination to Glance API and tables.

Fixes bug 981252.

Change-Id: Ib1fa6136947a23521dcdbf6d0d7ae783a6e0fae7

horizon/api/glance.py

@@ -23,6 +23,8 @@ from __future__ import absolute_import
 import logging
 import urlparse
 
+from django.conf import settings
+
 from glanceclient.v1 import client as glance_client
 
 from horizon.api.base import url_for
@@ -51,16 +53,30 @@ def image_get(request, image_id):
     return glanceclient(request).images.get(image_id)
 
 
-def image_list_detailed(request, filters=None):
+def image_list_detailed(request, marker=None, filters=None):
     filters = filters or {}
-    return glanceclient(request).images.list(filters=filters)
+    limit = getattr(settings, 'API_RESULT_LIMIT', 1000)
+    images = glanceclient(request).images.list(limit=limit + 1,
+                                               marker=marker,
+                                               filters=filters)
+    if(len(images) > limit):
+        return (images[0:-1], True)
+    else:
+        return (images, False)
 
 
 def image_update(request, image_id, **kwargs):
     return glanceclient(request).images.update(image_id, **kwargs)
 
 
-def snapshot_list_detailed(request, extra_filters=None):
+def snapshot_list_detailed(request, marker=None, extra_filters=None):
     filters = {'property-image_type': 'snapshot'}
     filters.update(extra_filters or {})
-    return glanceclient(request).images.list(filters=filters)
+    limit = getattr(settings, 'API_RESULT_LIMIT', 1000)
+    images = glanceclient(request).images.list(limit=limit + 1,
+                                               marker=marker,
+                                               filters=filters)
+    if(len(images) > limit):
+        return (images[0:-1], True)
+    else:
+        return (images, False)

horizon/dashboards/nova/containers/tables.py

@@ -17,7 +17,6 @@
 import logging
 
 from cloudfiles.errors import ContainerNotEmpty
-from django import shortcuts
 from django.contrib import messages
 from django.core.urlresolvers import reverse
 from django.template.defaultfilters import filesizeformat

horizon/dashboards/nova/images_and_snapshots/images/tables.py

@@ -102,3 +102,4 @@ class ImagesTable(tables.DataTable):
         verbose_name = _("Images")
         table_actions = (DeleteImage,)
         row_actions = (LaunchImage, EditImage, DeleteImage)
+        pagination_param = "image_marker"

horizon/dashboards/nova/images_and_snapshots/snapshots/tables.py

@@ -54,3 +54,4 @@ class SnapshotsTable(ImagesTable):
         verbose_name = _("Instance Snapshots")
         table_actions = (DeleteSnapshot,)
         row_actions = (LaunchSnapshot, EditImage, DeleteSnapshot)
+        pagination_param = "snapshot_marker"

horizon/dashboards/nova/images_and_snapshots/tests.py

@@ -40,8 +40,10 @@ class ImagesAndSnapshotsTests(test.TestCase):
         self.mox.StubOutWithMock(api, 'volume_snapshot_list')
         api.volume_snapshot_list(IsA(http.HttpRequest)) \
                                 .AndReturn(self.volumes.list())
-        api.image_list_detailed(IsA(http.HttpRequest)).AndReturn(images)
+        api.image_list_detailed(IsA(http.HttpRequest),
-        api.snapshot_list_detailed(IsA(http.HttpRequest)).AndReturn(snapshots)
+                                marker=None).AndReturn([images, False])
+        api.snapshot_list_detailed(IsA(http.HttpRequest),
+                                marker=None).AndReturn([snapshots, False])
         self.mox.ReplayAll()
 
         res = self.client.get(INDEX_URL)
@@ -58,9 +60,10 @@ class ImagesAndSnapshotsTests(test.TestCase):
         self.mox.StubOutWithMock(api, 'volume_snapshot_list')
         api.volume_snapshot_list(IsA(http.HttpRequest)) \
                                 .AndReturn(self.volumes.list())
-        api.image_list_detailed(IsA(http.HttpRequest)).AndReturn([])
+        api.image_list_detailed(IsA(http.HttpRequest),
-        api.snapshot_list_detailed(IsA(http.HttpRequest)) \
+                                marker=None).AndReturn([(), False])
-                                   .AndReturn(self.snapshots.list())
+        api.snapshot_list_detailed(IsA(http.HttpRequest), marker=None) \
+                                .AndReturn([self.snapshots.list(), False])
         self.mox.ReplayAll()
 
         res = self.client.get(INDEX_URL)
@@ -72,10 +75,10 @@ class ImagesAndSnapshotsTests(test.TestCase):
         self.mox.StubOutWithMock(api, 'volume_snapshot_list')
         api.volume_snapshot_list(IsA(http.HttpRequest)) \
                                 .AndReturn(self.volumes.list())
-        api.image_list_detailed(IsA(http.HttpRequest)) \
+        api.image_list_detailed(IsA(http.HttpRequest),
-                                .AndRaise(self.exceptions.glance)
+                                marker=None).AndRaise(self.exceptions.glance)
-        api.snapshot_list_detailed(IsA(http.HttpRequest)) \
+        api.snapshot_list_detailed(IsA(http.HttpRequest), marker=None) \
-                                   .AndReturn(self.snapshots.list())
+                                .AndReturn([self.snapshots.list(), False])
         self.mox.ReplayAll()
 
         res = self.client.get(INDEX_URL)
@@ -102,9 +105,10 @@ class ImagesAndSnapshotsTests(test.TestCase):
         self.mox.StubOutWithMock(api, 'volume_snapshot_list')
         api.volume_snapshot_list(IsA(http.HttpRequest)) \
                                 .AndReturn(self.volumes.list())
-        api.image_list_detailed(IsA(http.HttpRequest)).AndReturn(images)
+        api.image_list_detailed(IsA(http.HttpRequest),
-        api.snapshot_list_detailed(IsA(http.HttpRequest)).\
+                                marker=None).AndReturn([images, False])
-                AndReturn(new_snapshots)
+        api.snapshot_list_detailed(IsA(http.HttpRequest), marker=None) \
+                                .AndReturn([new_snapshots, False])
         self.mox.ReplayAll()
 
         res = self.client.get(INDEX_URL)

horizon/dashboards/nova/images_and_snapshots/views.py

@@ -42,9 +42,16 @@ class IndexView(tables.MultiTableView):
     table_classes = (ImagesTable, SnapshotsTable, VolumeSnapshotsTable)
     template_name = 'nova/images_and_snapshots/index.html'
 
+    def has_more_data(self, table):
+        return getattr(self, "_more_%s" % table.name, False)
+
     def get_images_data(self):
+        marker = self.request.GET.get(ImagesTable._meta.pagination_param, None)
         try:
-            all_images = api.image_list_detailed(self.request)
+            # FIXME(gabriel): The paging is going to be strange here due to
+            # our filtering after the fact.
+            all_images, _more_images = api.image_list_detailed(self.request,
+                                                                marker=marker)
             images = [im for im in all_images
                       if im.container_format not in ['aki', 'ari'] and
                       im.properties.get("image_type", '') != "snapshot"]
@@ -54,12 +61,15 @@ class IndexView(tables.MultiTableView):
         return images
 
     def get_snapshots_data(self):
+        req = self.request
+        marker = req.GET.get(SnapshotsTable._meta.pagination_param, None)
         try:
-            snapshots = api.snapshot_list_detailed(self.request)
+            snaps, self._more_snapshots = api.snapshot_list_detailed(req,
+                                                                marker=marker)
         except:
-            snapshots = []
+            snaps = []
-            exceptions.handle(self.request, _("Unable to retrieve snapshots."))
+            exceptions.handle(req, _("Unable to retrieve snapshots."))
-        return snapshots
+        return snaps
 
     def get_volume_snapshots_data(self):
         try:

horizon/dashboards/nova/instances_and_volumes/instances/tests.py

@@ -329,8 +329,10 @@ class InstanceViewTests(test.TestCase):
                             server.id,
                             "snapshot1")
         api.server_get(IsA(http.HttpRequest), server.id).AndReturn(server)
-        api.snapshot_list_detailed(IsA(http.HttpRequest)).AndReturn([])
+        api.snapshot_list_detailed(IsA(http.HttpRequest),
-        api.image_list_detailed(IsA(http.HttpRequest)).AndReturn([])
+                                marker=None).AndReturn([[], False])
+        api.image_list_detailed(IsA(http.HttpRequest),
+                                marker=None).AndReturn([[], False])
         api.volume_snapshot_list(IsA(http.HttpRequest)).AndReturn([])
 
         api.volume_list(IsA(http.HttpRequest)).AndReturn(self.volumes.list())
@@ -414,7 +416,6 @@ class InstanceViewTests(test.TestCase):
         self.assertRedirectsNoFollow(res, INDEX_URL)
 
     def test_launch_get(self):
-        image = self.images.first()
         quota_usages = self.quota_usages.first()
 
         self.mox.StubOutWithMock(api.glance, 'image_list_detailed')
@@ -432,10 +433,10 @@ class InstanceViewTests(test.TestCase):
                                 .AndReturn(self.volumes.list())
         api.glance.image_list_detailed(IsA(http.HttpRequest),
                                        filters={'is_public': True}) \
-                  .AndReturn(self.images.list())
+                  .AndReturn([self.images.list(), False])
         api.glance.image_list_detailed(IsA(http.HttpRequest),
                             filters={'property-owner_id': self.tenant.id}) \
-                  .AndReturn([])
+                  .AndReturn([[], False])
         api.nova.tenant_quota_usages(IsA(http.HttpRequest)) \
                 .AndReturn(quota_usages)
         api.nova.flavor_list(IsA(http.HttpRequest)) \
@@ -489,10 +490,10 @@ class InstanceViewTests(test.TestCase):
                 .AndReturn(self.security_groups.list())
         api.glance.image_list_detailed(IsA(http.HttpRequest),
                                        filters={'is_public': True}) \
-                  .AndReturn(self.images.list())
+                  .AndReturn([self.images.list(), False])
         api.glance.image_list_detailed(IsA(http.HttpRequest),
                             filters={'property-owner_id': self.tenant.id}) \
-                  .AndReturn([])
+                  .AndReturn([[], False])
         api.nova.volume_list(IsA(http.HttpRequest)) \
                 .AndReturn(self.volumes.list())
         api.nova.volume_snapshot_list(IsA(http.HttpRequest)).AndReturn([])
@@ -541,10 +542,10 @@ class InstanceViewTests(test.TestCase):
                 .AndReturn(self.volumes.list())
         api.glance.image_list_detailed(IsA(http.HttpRequest),
                                        filters={'is_public': True}) \
-                  .AndReturn(self.images.list())
+                  .AndReturn([self.images.list(), False])
         api.glance.image_list_detailed(IsA(http.HttpRequest),
                             filters={'property-owner_id': self.tenant.id}) \
-                  .AndReturn([])
+                  .AndReturn([[], False])
         api.nova.tenant_quota_usages(IsA(http.HttpRequest)) \
                 .AndReturn(self.quota_usages.first())
         api.nova.flavor_list(IsA(http.HttpRequest)) \
@@ -586,10 +587,10 @@ class InstanceViewTests(test.TestCase):
                                 .AndReturn(self.security_groups.list())
         api.glance.image_list_detailed(IsA(http.HttpRequest),
                                        filters={'is_public': True}) \
-                  .AndReturn(self.images.list())
+                  .AndReturn([self.images.list(), False])
         api.glance.image_list_detailed(IsA(http.HttpRequest),
                             filters={'property-owner_id': self.tenant.id}) \
-                  .AndReturn([])
+                  .AndReturn([[], False])
         api.nova.volume_list(IgnoreArg()).AndReturn(self.volumes.list())
         api.nova.server_create(IsA(http.HttpRequest),
                                server.name,
@@ -637,8 +638,6 @@ class InstanceViewTests(test.TestCase):
         self.mox.StubOutWithMock(api.nova, 'volume_snapshot_list')
         self.mox.StubOutWithMock(api.nova, 'tenant_quota_usages')
 
-        api.nova.flavor_list(IsA(http.HttpRequest)) \
-                .AndReturn(self.flavors.list())
         api.nova.flavor_list(IsA(http.HttpRequest)) \
                 .AndReturn(self.flavors.list())
         api.nova.keypair_list(IsA(http.HttpRequest)) \
@@ -647,13 +646,16 @@ class InstanceViewTests(test.TestCase):
                 .AndReturn(self.security_groups.list())
         api.glance.image_list_detailed(IsA(http.HttpRequest),
                                        filters={'is_public': True}) \
-                  .AndReturn(self.images.list())
+                  .AndReturn([self.images.list(), False])
         api.glance.image_list_detailed(IsA(http.HttpRequest),
                             filters={'property-owner_id': self.tenant.id}) \
-                  .AndReturn([])
+                  .AndReturn([[], False])
         api.nova.volume_list(IsA(http.HttpRequest)) \
                 .AndReturn(self.volumes.list())
         api.nova.volume_snapshot_list(IsA(http.HttpRequest)).AndReturn([])
+
+        api.nova.flavor_list(IsA(http.HttpRequest)) \
+                .AndReturn(self.flavors.list())
         api.nova.tenant_quota_usages(IsA(http.HttpRequest)) \
                 .AndReturn(self.quota_usages.first())
         self.mox.ReplayAll()

horizon/dashboards/nova/instances_and_volumes/instances/workflows.py

@@ -204,7 +204,7 @@ class SetInstanceDetailsAction(workflows.Action):
         project_id = context.get('project_id', None)
         if not hasattr(self, "_public_images"):
             public = {"is_public": True}
-            public_images = api.glance.image_list_detailed(request,
+            public_images, _more = api.glance.image_list_detailed(request,
                                                            filters=public)
             self._public_images = public_images
 
@@ -214,7 +214,7 @@ class SetInstanceDetailsAction(workflows.Action):
 
         if not hasattr(self, "_images_for_%s" % project_id):
             owner = {"property-owner_id": project_id}
-            owned_images = api.glance.image_list_detailed(request,
+            owned_images, _more = api.glance.image_list_detailed(request,
                                                           filters=owner)
             setattr(self, "_images_for_%s" % project_id, owned_images)
 
@@ -223,12 +223,12 @@ class SetInstanceDetailsAction(workflows.Action):
 
         # Remove duplicate images.
         image_ids = []
+        final_images = []
         for image in images:
             if image.id not in image_ids:
                 image_ids.append(image.id)
-            else:
+                final_images.append(image)
-                images.remove(image)
+        return [image for image in final_images
-        return [image for image in images
                 if image.container_format not in ('aki', 'ari')]
 
     def populate_image_id_choices(self, request, context):
@@ -250,7 +250,7 @@ class SetInstanceDetailsAction(workflows.Action):
         if choices:
             choices.insert(0, ("", _("Select Instance Snapshot")))
         else:
-            choices.insert(0, ("", _("No images available.")))
+            choices.insert(0, ("", _("No snapshots available.")))
         return choices
 
     def populate_flavor_choices(self, request, context):

horizon/dashboards/syspanel/images/views.py

@@ -37,11 +37,18 @@ class IndexView(tables.DataTableView):
     table_class = AdminImagesTable
     template_name = 'syspanel/images/index.html'
 
+    def has_more_data(self, table):
+        return self._more
+
     def get_data(self):
         images = []
+        marker = self.request.GET.get(AdminImagesTable._meta.pagination_param,
+                                      None)
         try:
-            images = api.image_list_detailed(self.request)
+            images, self._more = api.image_list_detailed(self.request,
+                                                         marker=marker)
         except:
+            self._more = False
             msg = _('Unable to retrieve image list.')
             exceptions.handle(self.request, msg)
         return images

horizon/tables/base.py

@@ -563,6 +563,13 @@ class DataTableOptions(object):
         The name of the context variable which will contain the table when
         it is rendered. Defaults to ``"table"``.
 
+    .. attribute:: pagination_param
+
+        The name of the query string parameter which will be used when
+        paginating this table. When using multiple tables in a single
+        view this will need to be changed to differentiate between the
+        tables. Default: ``"marker"``.
+
     .. attribute:: status_columns
 
         A list or tuple of column names which represents the "state"
@@ -597,6 +604,7 @@ class DataTableOptions(object):
         self.row_actions = getattr(options, 'row_actions', [])
         self.row_class = getattr(options, 'row_class', Row)
         self.column_class = getattr(options, 'column_class', Column)
+        self.pagination_param = getattr(options, 'pagination_param', 'marker')
 
         # Set self.filter if we have any FilterActions
         filter_actions = [action for action in self.table_actions if
@@ -1043,6 +1051,10 @@ class DataTable(object):
         """
         return http.urlquote_plus(self.get_object_id(self.data[-1]))
 
+    def get_pagination_string(self):
+        """ Returns the query parameter string to paginate this table. """
+        return "=".join([self._meta.pagination_param, self.get_marker()])
+
     def calculate_row_status(self, statuses):
         """
         Returns a boolean value determining the overall row status

horizon/templates/horizon/common/_data_table.html

@@ -31,7 +31,7 @@
           <span>{% blocktrans count counter=rows|length %}Displaying {{ counter }} item{% plural %}Displaying {{ counter }} items{% endblocktrans %}</span>
           {% if table.has_more_data %}
           <span class="spacer">|</span>
-          <a href="?marker={{ table.get_marker }}">More&nbsp;&raquo;</a>
+          <a href="?{{ table.get_pagination_string }}">More&nbsp;&raquo;</a>
           {% endif %}
         </td>
       </td>

horizon/tests/api_tests/glance_tests.py

@@ -18,6 +18,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from django.conf import settings
+
 from horizon import api
 from horizon import test
 
@@ -26,10 +28,13 @@ class GlanceApiTests(test.APITestCase):
     def test_snapshot_list_detailed(self):
         images = self.images.list()
         filters = {'property-image_type': 'snapshot'}
+        limit = getattr(settings, 'API_RESULT_LIMIT', 1000)
 
         glanceclient = self.stub_glanceclient()
         glanceclient.images = self.mox.CreateMockAnything()
-        glanceclient.images.list(filters=filters).AndReturn(images)
+        glanceclient.images.list(filters=filters,
+                                 limit=limit + 1,
+                                 marker=None).AndReturn(images)
         self.mox.ReplayAll()
 
         # No assertions are necessary. Verification is handled by mox.