openstack/swift
Code Issues Proposed changes

Consolidate retry code in functest client

Browse Source 
We added retries to the functest client ages ago, but object PUTs were
skipped because they used slightly different machinery.

We added retires to PUT later in the related change, but it added it's
own slightly different retry machinery.

Although neither is perfect (e.g. no exponential backoff) - this change
moves them closer together, so that future improvements can help both.

Related-Change-Id: I90a319943e948ac7df86cb29046f711adbb2fe20

Change-Id: If65dbc6e524b6ba83b27f5697cae0927a3891320
This commit is contained in:
Clay Gerrard 2019-02-06 10:03:50 -06:00
parent dd379e7f5e
commit c6d213f826
1 changed files with 38 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
test/functional/swift_test_client.py
View File

@ -48,12 +48,13 @@ class RequestError(Exception):
class ResponseError(Exception): class ResponseError(Exception):
def __init__(self, response, method=None, path=None): def __init__(self, response, method=None, path=None, details=None):
self.status = response.status self.status = response.status
self.reason = response.reason self.reason = response.reason
self.method = method self.method = method
self.path = path self.path = path
self.headers = response.getheaders() self.headers = response.getheaders()
self.details = details
for name, value in self.headers: for name, value in self.headers:
if name.lower() == 'x-trans-id': if name.lower() == 'x-trans-id':
@ -68,8 +69,11 @@ class ResponseError(Exception):
return repr(self) return repr(self)
def __repr__(self): def __repr__(self):
return '%d: %r (%r %r) txid=%s' % ( msg = '%d: %r (%r %r) txid=%s' % (
self.status, self.reason, self.method, self.path, self.txid) self.status, self.reason, self.method, self.path, self.txid)
if self.details:
msg += '\n%s' % self.details
return msg
def listing_empty(method): def listing_empty(method):
@ -299,6 +303,16 @@ class Connection(object):
self.connection.request(method, path, data, headers) self.connection.request(method, path, data, headers)
return self.connection.getresponse() return self.connection.getresponse()
try:
self.response = self.request_with_retry(try_request)
except RequestError as e:
details = "{method} {path} headers: {headers} data: {data}".format(
method=method, path=path, headers=headers, data=data)
raise RequestError('Unable to complete request: %s.\n%s' % (
details, str(e)))
return self.response.status
def request_with_retry(self, try_request):
self.response = None self.response = None
try_count = 0 try_count = 0
fail_messages = [] fail_messages = []
@ -307,6 +321,9 @@ class Connection(object):
try: try:
self.response = try_request() self.response = try_request()
except socket.timeout as e:
fail_messages.append(safe_repr(e))
continue
except http_client.HTTPException as e: except http_client.HTTPException as e:
fail_messages.append(safe_repr(e)) fail_messages.append(safe_repr(e))
continue continue
@ -320,17 +337,13 @@ class Connection(object):
if try_count != 5: if try_count != 5:
time.sleep(5) time.sleep(5)
continue continue
break break
if self.response: if self.response:
return self.response.status return self.response
request = "{method} {path} headers: {headers} data: {data}".format( raise RequestError('Attempts: %s, Failures: %s' % (
method=method, path=path, headers=headers, data=data) len(fail_messages), fail_messages))
raise RequestError('Unable to complete http request: %s. '
'Attempts: %s, Failures: %s' %
(request, len(fail_messages), fail_messages))
def put_start(self, path, hdrs=None, parms=None, cfg=None, chunked=False): def put_start(self, path, hdrs=None, parms=None, cfg=None, chunked=False):
if hdrs is None: if hdrs is None:
@ -1026,26 +1039,27 @@ class File(Base):
headers = self.make_headers(cfg=cfg) headers = self.make_headers(cfg=cfg)
headers.update(hdrs) headers.update(hdrs)
for _attempt in range(3): def try_request():
# rewind to be ready for another attempt
data.seek(0)
self.conn.put_start(self.path, hdrs=headers, parms=parms, cfg=cfg) self.conn.put_start(self.path, hdrs=headers, parms=parms, cfg=cfg)
transferred = 0 transferred = 0
try: for buff in iter(lambda: data.read(block_size), b''):
for buff in iter(lambda: data.read(block_size), b''): self.conn.put_data(buff)
self.conn.put_data(buff) transferred += len(buff)
transferred += len(buff) if callable(callback):
if callable(callback): callback(transferred, self.size)
callback(transferred, self.size)
self.conn.put_end() self.conn.put_end()
except socket.timeout as err: return self.conn.response
raise err
if is_success(self.conn.response.status): try:
break self.response = self.conn.request_with_retry(try_request)
# else, rewind to be ready for another attempt except RequestError as e:
data.seek(0) raise ResponseError(self.conn.response, 'PUT',
else: self.conn.make_path(self.path), details=str(e))
if not is_success(self.response.status):
raise ResponseError(self.conn.response, 'PUT', raise ResponseError(self.conn.response, 'PUT',
self.conn.make_path(self.path)) self.conn.make_path(self.path))