From 8595026f464b0dff8a41e8297d27e212db0a5a73 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Wed, 14 Aug 2019 13:51:42 -0700 Subject: [PATCH] Retry more operations This adds a generic retry handler that takes a callable, and uses that not only for the exsting retries around the POST call, but also the actions when creating the container. Change-Id: I910b8e886f107d4fe38a9334ba836f010f92557c --- .../library/zuul_swift_upload.py | 90 ++++++++++--------- 1 file changed, 50 insertions(+), 40 deletions(-) diff --git a/roles/upload-logs-swift/library/zuul_swift_upload.py b/roles/upload-logs-swift/library/zuul_swift_upload.py index 4a3420a7e..345644c97 100755 --- a/roles/upload-logs-swift/library/zuul_swift_upload.py +++ b/roles/upload-logs-swift/library/zuul_swift_upload.py @@ -141,6 +141,18 @@ def get_cloud(cloud): return openstack.connect(cloud=cloud) +def retry_function(func): + for attempt in range(1, POST_ATTEMPTS + 1): + try: + return func() + except Exception: + if attempt >= POST_ATTEMPTS: + raise + else: + logging.exception("Error on attempt %d" % attempt) + time.sleep(attempt * 10) + + def sizeof_fmt(num, suffix='B'): # From http://stackoverflow.com/questions/1094841/ # reusable-library-to-get-human-readable-version-of-file-size @@ -491,30 +503,36 @@ class Uploader(): cdn_url = None if not self.cloud.get_container(self.container): - self.cloud.create_container(name=self.container, public=public) - self.cloud.update_container( - name=self.container, - headers={'X-Container-Meta-Web-Index': 'index.html', - 'X-Container-Meta-Access-Control-Allow-Origin': '*'}) + retry_function( + lambda: self.cloud.create_container( + name=self.container, public=public)) + headers = {'X-Container-Meta-Web-Index': 'index.html', + 'X-Container-Meta-Access-Control-Allow-Origin': '*'} + retry_function( + lambda: self.cloud.update_container( + name=self.container, + headers=headers)) # 'X-Container-Meta-Web-Listings': 'true' # The ceph radosgw swift implementation requires an # index.html at the root in order for any other indexes to # work. index_headers = {'access-control-allow-origin': '*'} - self.cloud.create_object(self.container, - name='index.html', - data='', - content_type='text/html', - **index_headers) + retry_function( + lambda: self.cloud.create_object(self.container, + name='index.html', + data='', + content_type='text/html', + **index_headers)) # Enable the CDN in rax if cdn_url: - self.cloud.session.put(cdn_url) + retry_function(lambda: self.cloud.session.put(cdn_url)) if cdn_url: - endpoint = (self.cloud.session.head(cdn_url) - .headers['X-Cdn-Ssl-Uri']) + endpoint = retry_function( + lambda: self.cloud.session.head( + cdn_url).headers['X-Cdn-Ssl-Uri']) container = endpoint else: endpoint = self.cloud.object_store.get_endpoint() @@ -546,7 +564,7 @@ class Uploader(): logging.debug("%s: processing job %s", threading.current_thread(), file_detail) - self._post_file(file_detail) + retry_function(lambda: self._post_file(file_detail)) except requests.exceptions.RequestException: # Do our best to attempt to upload all the files logging.exception("Error posting file after multiple attempts") @@ -584,32 +602,24 @@ class Uploader(): # This is required for Rackspace CDN headers['access-control-allow-origin'] = '*' - for attempt in range(1, POST_ATTEMPTS + 1): - try: - if not file_detail.folder: - if (file_detail.encoding is None and - self._is_text_type(file_detail.mimetype)): - headers['content-encoding'] = 'deflate' - data = DeflateFilter(open(file_detail.full_path, 'rb')) - else: - if file_detail.encoding: - headers['content-encoding'] = file_detail.encoding - data = open(file_detail.full_path, 'rb') - else: - data = '' - relative_path = relative_path.rstrip('/') - if relative_path == '': - relative_path = '/' - self.cloud.create_object(self.container, - name=relative_path, - data=data, - **headers) - break - except requests.exceptions.RequestException: - logging.exception( - "File posting error on attempt %d" % attempt) - if attempt >= POST_ATTEMPTS: - raise + if not file_detail.folder: + if (file_detail.encoding is None and + self._is_text_type(file_detail.mimetype)): + headers['content-encoding'] = 'deflate' + data = DeflateFilter(open(file_detail.full_path, 'rb')) + else: + if file_detail.encoding: + headers['content-encoding'] = file_detail.encoding + data = open(file_detail.full_path, 'rb') + else: + data = '' + relative_path = relative_path.rstrip('/') + if relative_path == '': + relative_path = '/' + self.cloud.create_object(self.container, + name=relative_path, + data=data, + **headers) def run(cloud, container, files,