Merge "Paginate over usage list to return all usages"

openstackclient/compute/v2/usage.py

@@ -15,8 +15,10 @@
 
 """Usage action implementations"""
 
+import collections
 import datetime
 
+from novaclient import api_versions
 from osc_lib.command import command
 from osc_lib import utils
 import six
@@ -24,6 +26,36 @@ import six
 from openstackclient.i18n import _
 
 
+def _get_usage_marker(usage):
+    marker = None
+    if hasattr(usage, 'server_usages') and usage.server_usages:
+        marker = usage.server_usages[-1]['instance_id']
+    return marker
+
+
+def _get_usage_list_marker(usage_list):
+    marker = None
+    if usage_list:
+        marker = _get_usage_marker(usage_list[-1])
+    return marker
+
+
+def _merge_usage(usage, next_usage):
+    usage.server_usages.extend(next_usage.server_usages)
+    usage.total_hours += next_usage.total_hours
+    usage.total_memory_mb_usage += next_usage.total_memory_mb_usage
+    usage.total_vcpus_usage += next_usage.total_vcpus_usage
+    usage.total_local_gb_usage += next_usage.total_local_gb_usage
+
+
+def _merge_usage_list(usages, next_usage_list):
+    for next_usage in next_usage_list:
+        if next_usage.tenant_id in usages:
+            _merge_usage(usages[next_usage.tenant_id], next_usage)
+        else:
+            usages[next_usage.tenant_id] = next_usage
+
+
 class ListUsage(command.Lister):
     _description = _("List resource usage per project")
 
@@ -83,7 +115,23 @@ class ListUsage(command.Lister):
         else:
             end = now + datetime.timedelta(days=1)
 
-        usage_list = compute_client.usage.list(start, end, detailed=True)
+        if compute_client.api_version < api_versions.APIVersion("2.40"):
+            usage_list = compute_client.usage.list(start, end, detailed=True)
+        else:
+            # If the number of instances used to calculate the usage is greater
+            # than CONF.api.max_limit, the usage will be split across multiple
+            # requests and the responses will need to be merged back together.
+            usages = collections.OrderedDict()
+            usage_list = compute_client.usage.list(start, end, detailed=True)
+            _merge_usage_list(usages, usage_list)
+            marker = _get_usage_list_marker(usage_list)
+            while marker:
+                next_usage_list = compute_client.usage.list(
+                    start, end, detailed=True, marker=marker)
+                marker = _get_usage_list_marker(next_usage_list)
+                if marker:
+                    _merge_usage_list(usages, next_usage_list)
+            usage_list = list(usages.values())
 
         # Cache the project list
         project_cache = {}

openstackclient/tests/unit/compute/v2/fakes.py

@@ -1304,6 +1304,7 @@ class FakeUsage(object):
                     'local_gb': 1,
                     'memory_mb': 512,
                     'name': 'usage-name-' + uuid.uuid4().hex,
+                    'instance_id': uuid.uuid4().hex,
                     'state': 'active',
                     'uptime': 3600,
                     'vcpus': 1

openstackclient/tests/unit/compute/v2/test_usage.py

@@ -14,6 +14,7 @@
 import datetime
 
 import mock
+from novaclient import api_versions
 
 from openstackclient.compute.v2 import usage
 from openstackclient.tests.unit.compute.v2 import fakes as compute_fakes
@@ -104,6 +105,31 @@ class TestUsageList(TestUsage):
         self.assertEqual(self.columns, columns)
         self.assertEqual(tuple(self.data), tuple(data))
 
+    def test_usage_list_with_pagination(self):
+        arglist = []
+        verifylist = [
+            ('start', None),
+            ('end', None),
+        ]
+
+        self.app.client_manager.compute.api_version = api_versions.APIVersion(
+            '2.40')
+        self.usage_mock.list.reset_mock()
+        self.usage_mock.list.side_effect = [self.usages, []]
+
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+        columns, data = self.cmd.take_action(parsed_args)
+
+        self.projects_mock.list.assert_called_with()
+        self.usage_mock.list.assert_has_calls([
+            mock.call(mock.ANY, mock.ANY, detailed=True),
+            mock.call(mock.ANY, mock.ANY, detailed=True,
+                      marker=self.usages[0]['server_usages'][0]['instance_id'])
+        ])
+        self.assertEqual(self.columns, columns)
+        self.assertEqual(tuple(self.data), tuple(data))
+
 
 class TestUsageShow(TestUsage):