openstack/swift
Code Issues Proposed changes

Add normalize_etag() helper function

Browse Source 
... and drive-by a import rename

Co-Authored-By: Clay Gerrard <clay.gerrard@gmail.com>
Change-Id: I1eaf075ff9855cfa03e7991bdf33375b0e4397e6
This commit is contained in:
Tim Burke 2019-11-19 21:25:45 -08:00
parent b3cd0cd4bb
commit 1f7b97ec0f
13 changed files with 280 additions and 249 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
swift/common/direct_client.py
View File

@ -29,11 +29,11 @@ from six.moves.http_client import HTTPException
from swift.common.bufferedhttp import http_connect
from swift.common.exceptions import ClientException
from swift.common.utils import Timestamp, FileLikeIter
from swift.common.swob import normalize_etag
from swift.common.utils import Timestamp, FileLikeIter, quote
from swift.common.http import HTTP_NO_CONTENT, HTTP_INSUFFICIENT_STORAGE, \
is_success, is_server_error
from swift.common.header_key_dict import HeaderKeyDict
from swift.common.utils import quote
class DirectClientException(ClientException):
@ -485,7 +485,7 @@ def direct_put_object(node, part, account, container, name, contents,
if headers is None:
headers = {}
if etag:
headers['ETag'] = etag.strip('"')
headers['ETag'] = normalize_etag(etag)
if content_type is not None:
headers['Content-Type'] = content_type
else:
@ -498,7 +498,7 @@ def direct_put_object(node, part, account, container, name, contents,
'Object', conn_timeout, response_timeout, contents=contents,
content_length=content_length, chunk_size=chunk_size)
return resp.getheader('etag').strip('"')
return normalize_etag(resp.getheader('etag'))
def direct_post_object(node, part, account, container, name, headers,

4
swift/common/middleware/crypto/encrypter.py
View File

@ -25,7 +25,7 @@ from swift.common.request_helpers import get_object_transient_sysmeta, \
strip_user_meta_prefix, is_user_meta, update_etag_is_at_header, \
get_container_update_override_key
from swift.common.swob import Request, Match, HTTPException, \
HTTPUnprocessableEntity, wsgi_to_bytes, bytes_to_wsgi
HTTPUnprocessableEntity, wsgi_to_bytes, bytes_to_wsgi, normalize_etag
from swift.common.utils import get_logger, config_true_value, \
MD5_OF_EMPTY_STRING
@ -263,7 +263,7 @@ class EncrypterObjContext(CryptoWSGIContext):
ciphertext_etag = enc_input_proxy.ciphertext_md5.hexdigest()
mod_resp_headers = [
(h, v if (h.lower() != 'etag' or
v.strip('"') != ciphertext_etag)
normalize_etag(v) != ciphertext_etag)
else plaintext_etag)
for h, v in mod_resp_headers]

4
swift/common/middleware/dlo.py
View File

@ -128,7 +128,7 @@ from swift.common.exceptions import ListingIterError, SegmentError
from swift.common.http import is_success
from swift.common.swob import Request, Response, \
HTTPRequestedRangeNotSatisfiable, HTTPBadRequest, HTTPConflict, \
str_to_wsgi, wsgi_to_str, wsgi_quote, wsgi_unquote
str_to_wsgi, wsgi_to_str, wsgi_quote, wsgi_unquote, normalize_etag
from swift.common.utils import get_logger, \
RateLimitedIterator, quote, close_if_possible, closing_if_possible
from swift.common.request_helpers import SegmentedIterable
@ -333,7 +333,7 @@ class GetContext(WSGIContext):
if h.lower() != "etag"]
etag = md5()
for seg_dict in segments:
etag.update(seg_dict['hash'].strip('"').encode('utf8'))
etag.update(normalize_etag(seg_dict['hash']).encode('utf8'))
response_headers.append(('Etag', '"%s"' % etag.hexdigest()))
app_iter = None

2
swift/common/middleware/s3api/controllers/bucket.py
View File

@ -232,7 +232,7 @@ class BucketController(Controller):
etag = o['s3_etag']
elif 'slo_etag' in o:
# SLOs may be in something *close* to the MU format
etag = '"%s-N"' % o['slo_etag'].strip('"')
etag = '"%s-N"' % swob.normalize_etag(o['slo_etag'])
else:
etag = o['hash']
if len(etag) < 2 or etag[::len(etag) - 1] != '""':

7
swift/common/middleware/s3api/controllers/multi_upload.py
View File

@ -68,7 +68,7 @@ import time
import six
from swift.common.swob import Range, bytes_to_wsgi
from swift.common.swob import Range, bytes_to_wsgi, normalize_etag
from swift.common.utils import json, public, reiterate
from swift.common.db import utf8encode
from swift.common.request_helpers import get_container_update_override_key
@ -620,10 +620,7 @@ class UploadController(Controller):
raise InvalidPartOrder(upload_id=upload_id)
previous_number = part_number
etag = part_elem.find('./ETag').text
if len(etag) >= 2 and etag[0] == '"' and etag[-1] == '"':
# strip double quotes
etag = etag[1:-1]
etag = normalize_etag(part_elem.find('./ETag').text)
if len(etag) != 32 or any(c not in '0123456789abcdef'
for c in etag):
raise InvalidPart(upload_id=upload_id,

6
swift/common/middleware/s3api/controllers/obj.py
View File

@ -15,7 +15,8 @@
from swift.common.http import HTTP_OK, HTTP_PARTIAL_CONTENT, HTTP_NO_CONTENT
from swift.common.request_helpers import update_etag_is_at_header
from swift.common.swob import Range, content_range_header_value
from swift.common.swob import Range, content_range_header_value, \
normalize_etag
from swift.common.utils import public, list_from_csv
from swift.common.middleware.s3api.utils import S3Timestamp, sysmeta_header
@ -68,8 +69,7 @@ class ObjectController(Controller):
continue
had_match = True
for value in list_from_csv(req.headers[match_header]):
if value.startswith('"') and value.endswith('"'):
value = value[1:-1]
value = normalize_etag(value)
if value.endswith('-N'):
# Deal with fake S3-like etags for SLOs uploaded via Swift
req.headers[match_header] += ', ' + value[:-2]

6
swift/common/middleware/slo.py
View File

@ -331,7 +331,7 @@ from swift.common.swob import Request, HTTPBadRequest, HTTPServerError, \
HTTPMethodNotAllowed, HTTPRequestEntityTooLarge, HTTPLengthRequired, \
HTTPOk, HTTPPreconditionFailed, HTTPException, HTTPNotFound, \
HTTPUnauthorized, HTTPConflict, HTTPUnprocessableEntity, \
HTTPServiceUnavailable, Response, Range, \
HTTPServiceUnavailable, Response, Range, normalize_etag, \
RESPONSE_REASONS, str_to_wsgi, wsgi_to_str, wsgi_quote
from swift.common.utils import get_logger, config_true_value, \
get_valid_utf8_str, override_bytes_from_content_type, split_path, \
@ -1324,8 +1324,8 @@ class StaticLargeObject(object):
slo_etag.update(r.encode('ascii') if six.PY3 else r)
slo_etag = slo_etag.hexdigest()
client_etag = req.headers.get('Etag')
if client_etag and client_etag.strip('"') != slo_etag:
client_etag = normalize_etag(req.headers.get('Etag'))
if client_etag and client_etag != slo_etag:
err = HTTPUnprocessableEntity(request=req)
if heartbeat:
resp_dict = {}

15
swift/common/swob.py
View File

@ -689,6 +689,12 @@ class Range(object):
return all_ranges
def normalize_etag(tag):
if tag and tag.startswith('"') and tag.endswith('"') and tag != '"':
return tag[1:-1]
return tag
class Match(object):
"""
Wraps a Request's If-[None-]Match header as a friendly object.
@ -701,15 +707,10 @@ class Match(object):
tag = tag.strip()
if not tag:
continue
if tag.startswith('"') and tag.endswith('"'):
self.tags.add(tag[1:-1])
else:
self.tags.add(tag)
self.tags.add(normalize_etag(tag))
def __contains__(self, val):
if val and val.startswith('"') and val.endswith('"'):
val = val[1:-1]
return '*' in self.tags or val in self.tags
return '*' in self.tags or normalize_etag(val) in self.tags
def __repr__(self):
return '%s(%r)' % (

3
swift/container/sync.py
View File

@ -36,6 +36,7 @@ from swift.common.internal_client import (
from swift.common.exceptions import ClientException
from swift.common.ring import Ring
from swift.common.ring.utils import is_local_device
from swift.common.swob import normalize_etag
from swift.common.utils import (
clean_content_type, config_true_value,
FileLikeIter, get_logger, hash_path, quote, validate_sync_to,
@ -607,7 +608,7 @@ class ContainerSync(Daemon):
if key in headers:
del headers[key]
if 'etag' in headers:
headers['etag'] = headers['etag'].strip('"')
headers['etag'] = normalize_etag(headers['etag'])
if 'content-type' in headers:
headers['content-type'] = clean_content_type(
headers['content-type'])

6
swift/obj/server.py
View File

@ -58,7 +58,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, \
HTTPInsufficientStorage, HTTPForbidden, HTTPException, HTTPConflict, \
HTTPServerError, wsgi_to_bytes, wsgi_to_str
HTTPServerError, wsgi_to_bytes, wsgi_to_str, normalize_etag
from swift.obj.diskfile import RESERVED_DATAFILE_META, DiskFileRouter
from swift.obj.expirer import build_task_obj
@ -942,8 +942,8 @@ class ObjectController(BaseStorageServer):
if (is_sys_or_user_meta('object', val[0]) or
is_object_transient_sysmeta(val[0])))
# N.B. footers_metadata is a HeaderKeyDict
received_etag = footers_metadata.get('etag', request.headers.get(
'etag', '')).strip('"')
received_etag = normalize_etag(footers_metadata.get(
'etag', request.headers.get('etag', '')))
if received_etag and received_etag != metadata['ETag']:
raise HTTPUnprocessableEntity(request=request)

13
swift/proxy/controllers/base.py
View File

@ -57,7 +57,7 @@ from swift.common.http import is_informational, is_success, is_redirection, \
HTTP_INSUFFICIENT_STORAGE, HTTP_UNAUTHORIZED, HTTP_CONTINUE, HTTP_GONE
from swift.common.swob import Request, Response, Range, \
HTTPException, HTTPRequestedRangeNotSatisfiable, HTTPServiceUnavailable, \
status_map, wsgi_to_str, str_to_wsgi, wsgi_quote
status_map, wsgi_to_str, str_to_wsgi, wsgi_quote, normalize_etag
from swift.common.request_helpers import strip_sys_meta_prefix, \
strip_user_meta_prefix, is_user_meta, is_sys_meta, is_sys_or_user_meta, \
http_response_to_document_iters, is_object_transient_sysmeta, \
@ -1268,9 +1268,9 @@ class ResumingGetter(object):
close_swift_conn(possible_source)
else:
if self.used_source_etag and \
self.used_source_etag != src_headers.get(
self.used_source_etag != normalize_etag(src_headers.get(
'x-object-sysmeta-ec-etag',
src_headers.get('etag', '')).strip('"'):
src_headers.get('etag', ''))):
self.statuses.append(HTTP_NOT_FOUND)
self.reasons.append('')
self.bodies.append('')
@ -1373,9 +1373,8 @@ class ResumingGetter(object):
# from the same object (EC). Otherwise, if the cluster has two
# versions of the same object, we might end up switching between
# old and new mid-stream and giving garbage to the client.
self.used_source_etag = src_headers.get(
'x-object-sysmeta-ec-etag',
src_headers.get('etag', '')).strip('"')
self.used_source_etag = normalize_etag(src_headers.get(
'x-object-sysmeta-ec-etag', src_headers.get('etag', '')))
self.node = node
return source, node
return None, None
@ -1922,7 +1921,7 @@ class Controller(object):
if headers:
update_headers(resp, headers[status_index])
if etag:
resp.headers['etag'] = etag.strip('"')
resp.headers['etag'] = normalize_etag(etag)
return resp
return None

9
swift/proxy/controllers/obj.py
View File

@ -70,7 +70,8 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \
HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPRequestTimeout, \
HTTPServerError, HTTPServiceUnavailable, HTTPClientDisconnect, \
HTTPUnprocessableEntity, Response, HTTPException, \
HTTPRequestedRangeNotSatisfiable, Range, HTTPInternalServerError
HTTPRequestedRangeNotSatisfiable, Range, HTTPInternalServerError, \
normalize_etag
from swift.common.request_helpers import update_etag_is_at_header, \
resolve_etag_is_at_header, validate_internal_obj
@ -478,7 +479,7 @@ class BaseObjectController(Controller):
{'status': response.status,
'body': body[:1024], 'path': req.path})
elif is_success(response.status):
etags.add(response.getheader('etag').strip('"'))
etags.add(normalize_etag(response.getheader('etag')))
for (putter, response) in pile:
if response:
@ -2638,8 +2639,8 @@ class ECObjectController(BaseObjectController):
computed_etag = (etag_hasher.hexdigest()
if etag_hasher else None)
footers = self._get_footers(req)
received_etag = footers.get('etag', req.headers.get(
'etag', '')).strip('"')
received_etag = normalize_etag(footers.get(
'etag', req.headers.get('etag', '')))
if (computed_etag and received_etag and
computed_etag != received_etag):
raise HTTPUnprocessableEntity(request=req)

410
test/unit/common/test_swob.py
View File

File diff suppressed because it is too large Load Diff