From a696c1e89ef1a1c7630c14bbdda3f6c8039bb05e Mon Sep 17 00:00:00 2001
From: Christian Schwede <cschwede@redhat.com>
Date: Mon, 14 Dec 2015 16:03:43 +0100
Subject: [PATCH] Fix full_listing in internal_client

The internal_client is used in swift-dispersion-report, and in case one has more
than 10000 containers or objects these are not queried.

This patch adds support to the internal_client to iterate over all
containers/objects if the listing exceeds the default of 10000 entries and the
argument full_listing=True is used.

Closes-Bug: 1314817
Closes-Bug: 1525995
Change-Id: I6892390d72f70f1bc519b482d4f72603e1570163
---
 swift/common/internal_client.py          | 19 ++++++++++++++++++-
 test/unit/common/test_internal_client.py | 23 +++++++++++++++++++++++
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/swift/common/internal_client.py b/swift/common/internal_client.py
index d9911326f1..ecffa999e4 100644
--- a/swift/common/internal_client.py
+++ b/swift/common/internal_client.py
@@ -741,11 +741,25 @@ class SimpleClient(object):
     def base_request(self, method, container=None, name=None, prefix=None,
                      headers=None, proxy=None, contents=None,
                      full_listing=None, logger=None, additional_info=None,
-                     timeout=None):
+                     timeout=None, marker=None):
         # Common request method
         trans_start = time()
         url = self.url
 
+        if full_listing:
+            body_data = self.base_request(method, container, name, prefix,
+                                          headers, proxy, timeout=timeout,
+                                          marker=marker)
+            listing = body_data[1]
+            while listing:
+                marker = listing[-1]['name']
+                listing = self.base_request(method, container, name, prefix,
+                                            headers, proxy, timeout=timeout,
+                                            marker=marker)[1]
+                if listing:
+                    body_data[1].extend(listing)
+            return body_data
+
         if headers is None:
             headers = {}
 
@@ -762,6 +776,9 @@ class SimpleClient(object):
             if prefix:
                 url += '&prefix=%s' % prefix
 
+            if marker:
+                url += '&marker=%s' % quote(marker)
+
         req = urllib2.Request(url, headers=headers, data=contents)
         if proxy:
             proxy = urllib.parse.urlparse(proxy)
diff --git a/test/unit/common/test_internal_client.py b/test/unit/common/test_internal_client.py
index 68dbc3e18d..e4218a9e02 100644
--- a/test/unit/common/test_internal_client.py
+++ b/test/unit/common/test_internal_client.py
@@ -356,6 +356,29 @@ class TestInternalClient(unittest.TestCase):
                 # sanity check
                 self.assertEqual(body, resp_body)
 
+    def test_base_full_listing(self):
+        body1 = [{'name': 'a'}, {'name': "b"}, {'name': "c"}]
+        body2 = [{'name': 'd'}]
+        body3 = []
+
+        class FakeConn(object):
+            def __init__(self, body):
+                self.body = body
+
+            def read(self):
+                return json.dumps(self.body)
+
+            def info(self):
+                return {}
+
+        mocked_func = 'swift.common.internal_client.urllib2.urlopen'
+        with mock.patch(mocked_func) as mock_urlopen:
+            mock_urlopen.side_effect = [
+                FakeConn(body1), FakeConn(body2), FakeConn(body3)]
+            sc = internal_client.SimpleClient('http://0.0.0.0/')
+            _, resp_body = sc.base_request('GET', full_listing=True)
+            self.assertEqual(body1 + body2, resp_body)
+
     def test_make_request_method_path_headers(self):
         class InternalClient(internal_client.InternalClient):
             def __init__(self):