From 1105d2badac8d3a0c06c4e5e35bce5fe789a3177 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Fri, 15 Nov 2024 12:08:38 -0800 Subject: [PATCH] Implement paging in swift object listing Swift by default only returns a maximum of 10k objects per object listing. This is problematic when pruning (and potentially otherwise) as we may not list valid manifests resulting in us never marking the blobs belonging to those manfiests as valid. These blobs would then get erroneously deleted. Address this by paging through object listings using swift's marker parameter. Change-Id: Ida85076b716a7718a8ca5fe50e4fbb90b3a41cbf --- zuul_registry/swift.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/zuul_registry/swift.py b/zuul_registry/swift.py index 6c28282..80ef2b7 100644 --- a/zuul_registry/swift.py +++ b/zuul_registry/swift.py @@ -65,7 +65,19 @@ class SwiftDriver(storageutils.StorageDriver): def list_objects(self, path): self.log.debug("List objects %s", path) - url = self.get_url('') + '?prefix=%s&delimiter=/&format=json' % (path,) + marker = '' + ret = [] + while inner_ret := self._list_objects(path, marker): + # Swift limits the total number of responses per request + # (typically 10k) so we have to paginate and accumulate responses. + ret.extend(inner_ret) + marker = inner_ret[-1].path + return ret + + def _list_objects(self, path, marker): + # TODO should path and marker be url encoded? + url = self.get_url('') + \ + '?prefix=%s&delimiter=/&format=json&marker=%s' % (path, marker) ret = retry_function( lambda: self.conn.session.get(url).content.decode('utf8')) data = json.loads(ret)