Add normalize_etag() helper function

... and drive-by a import rename

Co-Authored-By: Clay Gerrard <clay.gerrard@gmail.com>
Change-Id: I1eaf075ff9855cfa03e7991bdf33375b0e4397e6

swift/common/direct_client.py

@@ -29,11 +29,11 @@ from six.moves.http_client import HTTPException
 
 from swift.common.bufferedhttp import http_connect
 from swift.common.exceptions import ClientException
-from swift.common.utils import Timestamp, FileLikeIter
+from swift.common.swob import normalize_etag
+from swift.common.utils import Timestamp, FileLikeIter, quote
 from swift.common.http import HTTP_NO_CONTENT, HTTP_INSUFFICIENT_STORAGE, \
     is_success, is_server_error
 from swift.common.header_key_dict import HeaderKeyDict
-from swift.common.utils import quote
 
 
 class DirectClientException(ClientException):
@@ -485,7 +485,7 @@ def direct_put_object(node, part, account, container, name, contents,
     if headers is None:
         headers = {}
     if etag:
-        headers['ETag'] = etag.strip('"')
+        headers['ETag'] = normalize_etag(etag)
     if content_type is not None:
         headers['Content-Type'] = content_type
     else:
@@ -498,7 +498,7 @@ def direct_put_object(node, part, account, container, name, contents,
         'Object', conn_timeout, response_timeout, contents=contents,
         content_length=content_length, chunk_size=chunk_size)
 
-    return resp.getheader('etag').strip('"')
+    return normalize_etag(resp.getheader('etag'))
 
 
 def direct_post_object(node, part, account, container, name, headers,

swift/common/middleware/crypto/encrypter.py

@@ -25,7 +25,7 @@ from swift.common.request_helpers import get_object_transient_sysmeta, \
     strip_user_meta_prefix, is_user_meta, update_etag_is_at_header, \
     get_container_update_override_key
 from swift.common.swob import Request, Match, HTTPException, \
-    HTTPUnprocessableEntity, wsgi_to_bytes, bytes_to_wsgi
+    HTTPUnprocessableEntity, wsgi_to_bytes, bytes_to_wsgi, normalize_etag
 from swift.common.utils import get_logger, config_true_value, \
     MD5_OF_EMPTY_STRING
 
@@ -263,7 +263,7 @@ class EncrypterObjContext(CryptoWSGIContext):
             ciphertext_etag = enc_input_proxy.ciphertext_md5.hexdigest()
             mod_resp_headers = [
                 (h, v if (h.lower() != 'etag' or
-                          v.strip('"') != ciphertext_etag)
+                          normalize_etag(v) != ciphertext_etag)
                     else plaintext_etag)
                 for h, v in mod_resp_headers]
 

swift/common/middleware/dlo.py

@@ -128,7 +128,7 @@ from swift.common.exceptions import ListingIterError, SegmentError
 from swift.common.http import is_success
 from swift.common.swob import Request, Response, \
     HTTPRequestedRangeNotSatisfiable, HTTPBadRequest, HTTPConflict, \
-    str_to_wsgi, wsgi_to_str, wsgi_quote, wsgi_unquote
+    str_to_wsgi, wsgi_to_str, wsgi_quote, wsgi_unquote, normalize_etag
 from swift.common.utils import get_logger, \
     RateLimitedIterator, quote, close_if_possible, closing_if_possible
 from swift.common.request_helpers import SegmentedIterable
@@ -333,7 +333,7 @@ class GetContext(WSGIContext):
                                 if h.lower() != "etag"]
             etag = md5()
             for seg_dict in segments:
-                etag.update(seg_dict['hash'].strip('"').encode('utf8'))
+                etag.update(normalize_etag(seg_dict['hash']).encode('utf8'))
             response_headers.append(('Etag', '"%s"' % etag.hexdigest()))
 
         app_iter = None

swift/common/middleware/s3api/controllers/bucket.py

@@ -232,7 +232,7 @@ class BucketController(Controller):
                     etag = o['s3_etag']
                 elif 'slo_etag' in o:
                     # SLOs may be in something *close* to the MU format
-                    etag = '"%s-N"' % o['slo_etag'].strip('"')
+                    etag = '"%s-N"' % swob.normalize_etag(o['slo_etag'])
                 else:
                     etag = o['hash']
                     if len(etag) < 2 or etag[::len(etag) - 1] != '""':

swift/common/middleware/s3api/controllers/multi_upload.py

@@ -68,7 +68,7 @@ import time
 
 import six
 
-from swift.common.swob import Range, bytes_to_wsgi
+from swift.common.swob import Range, bytes_to_wsgi, normalize_etag
 from swift.common.utils import json, public, reiterate
 from swift.common.db import utf8encode
 from swift.common.request_helpers import get_container_update_override_key
@@ -620,10 +620,7 @@ class UploadController(Controller):
                     raise InvalidPartOrder(upload_id=upload_id)
                 previous_number = part_number
 
-                etag = part_elem.find('./ETag').text
+                etag = normalize_etag(part_elem.find('./ETag').text)
-                if len(etag) >= 2 and etag[0] == '"' and etag[-1] == '"':
-                    # strip double quotes
-                    etag = etag[1:-1]
                 if len(etag) != 32 or any(c not in '0123456789abcdef'
                                           for c in etag):
                     raise InvalidPart(upload_id=upload_id,

swift/common/middleware/s3api/controllers/obj.py

@@ -15,7 +15,8 @@
 
 from swift.common.http import HTTP_OK, HTTP_PARTIAL_CONTENT, HTTP_NO_CONTENT
 from swift.common.request_helpers import update_etag_is_at_header
-from swift.common.swob import Range, content_range_header_value
+from swift.common.swob import Range, content_range_header_value, \
+    normalize_etag
 from swift.common.utils import public, list_from_csv
 
 from swift.common.middleware.s3api.utils import S3Timestamp, sysmeta_header
@@ -68,8 +69,7 @@ class ObjectController(Controller):
                 continue
             had_match = True
             for value in list_from_csv(req.headers[match_header]):
-                if value.startswith('"') and value.endswith('"'):
+                value = normalize_etag(value)
-                    value = value[1:-1]
                 if value.endswith('-N'):
                     # Deal with fake S3-like etags for SLOs uploaded via Swift
                     req.headers[match_header] += ', ' + value[:-2]

swift/common/middleware/slo.py

@@ -331,7 +331,7 @@ from swift.common.swob import Request, HTTPBadRequest, HTTPServerError, \
     HTTPMethodNotAllowed, HTTPRequestEntityTooLarge, HTTPLengthRequired, \
     HTTPOk, HTTPPreconditionFailed, HTTPException, HTTPNotFound, \
     HTTPUnauthorized, HTTPConflict, HTTPUnprocessableEntity, \
-    HTTPServiceUnavailable, Response, Range, \
+    HTTPServiceUnavailable, Response, Range, normalize_etag, \
     RESPONSE_REASONS, str_to_wsgi, wsgi_to_str, wsgi_quote
 from swift.common.utils import get_logger, config_true_value, \
     get_valid_utf8_str, override_bytes_from_content_type, split_path, \
@@ -1324,8 +1324,8 @@ class StaticLargeObject(object):
                 slo_etag.update(r.encode('ascii') if six.PY3 else r)
 
             slo_etag = slo_etag.hexdigest()
-            client_etag = req.headers.get('Etag')
+            client_etag = normalize_etag(req.headers.get('Etag'))
-            if client_etag and client_etag.strip('"') != slo_etag:
+            if client_etag and client_etag != slo_etag:
                 err = HTTPUnprocessableEntity(request=req)
                 if heartbeat:
                     resp_dict = {}

swift/common/swob.py

@@ -689,6 +689,12 @@ class Range(object):
         return all_ranges
 
 
+def normalize_etag(tag):
+    if tag and tag.startswith('"') and tag.endswith('"') and tag != '"':
+        return tag[1:-1]
+    return tag
+
+
 class Match(object):
     """
     Wraps a Request's If-[None-]Match header as a friendly object.
@@ -701,15 +707,10 @@ class Match(object):
             tag = tag.strip()
             if not tag:
                 continue
-            if tag.startswith('"') and tag.endswith('"'):
+            self.tags.add(normalize_etag(tag))
-                self.tags.add(tag[1:-1])
-            else:
-                self.tags.add(tag)
 
     def __contains__(self, val):
-        if val and val.startswith('"') and val.endswith('"'):
+        return '*' in self.tags or normalize_etag(val) in self.tags
-            val = val[1:-1]
-        return '*' in self.tags or val in self.tags
 
     def __repr__(self):
         return '%s(%r)' % (

swift/container/sync.py

@@ -36,6 +36,7 @@ from swift.common.internal_client import (
 from swift.common.exceptions import ClientException
 from swift.common.ring import Ring
 from swift.common.ring.utils import is_local_device
+from swift.common.swob import normalize_etag
 from swift.common.utils import (
     clean_content_type, config_true_value,
     FileLikeIter, get_logger, hash_path, quote, validate_sync_to,
@@ -607,7 +608,7 @@ class ContainerSync(Daemon):
                     if key in headers:
                         del headers[key]
                 if 'etag' in headers:
-                    headers['etag'] = headers['etag'].strip('"')
+                    headers['etag'] = normalize_etag(headers['etag'])
                 if 'content-type' in headers:
                     headers['content-type'] = clean_content_type(
                         headers['content-type'])

swift/obj/server.py

@@ -58,7 +58,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
     HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
     HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, \
     HTTPInsufficientStorage, HTTPForbidden, HTTPException, HTTPConflict, \
-    HTTPServerError, wsgi_to_bytes, wsgi_to_str
+    HTTPServerError, wsgi_to_bytes, wsgi_to_str, normalize_etag
 from swift.obj.diskfile import RESERVED_DATAFILE_META, DiskFileRouter
 from swift.obj.expirer import build_task_obj
 
@@ -942,8 +942,8 @@ class ObjectController(BaseStorageServer):
                         if (is_sys_or_user_meta('object', val[0]) or
                             is_object_transient_sysmeta(val[0])))
         # N.B. footers_metadata is a HeaderKeyDict
-        received_etag = footers_metadata.get('etag', request.headers.get(
+        received_etag = normalize_etag(footers_metadata.get(
-            'etag', '')).strip('"')
+            'etag', request.headers.get('etag', '')))
         if received_etag and received_etag != metadata['ETag']:
             raise HTTPUnprocessableEntity(request=request)
 

swift/proxy/controllers/base.py

@@ -57,7 +57,7 @@ from swift.common.http import is_informational, is_success, is_redirection, \
     HTTP_INSUFFICIENT_STORAGE, HTTP_UNAUTHORIZED, HTTP_CONTINUE, HTTP_GONE
 from swift.common.swob import Request, Response, Range, \
     HTTPException, HTTPRequestedRangeNotSatisfiable, HTTPServiceUnavailable, \
-    status_map, wsgi_to_str, str_to_wsgi, wsgi_quote
+    status_map, wsgi_to_str, str_to_wsgi, wsgi_quote, normalize_etag
 from swift.common.request_helpers import strip_sys_meta_prefix, \
     strip_user_meta_prefix, is_user_meta, is_sys_meta, is_sys_or_user_meta, \
     http_response_to_document_iters, is_object_transient_sysmeta, \
@@ -1268,9 +1268,9 @@ class ResumingGetter(object):
                 close_swift_conn(possible_source)
             else:
                 if self.used_source_etag and \
-                    self.used_source_etag != src_headers.get(
+                    self.used_source_etag != normalize_etag(src_headers.get(
                         'x-object-sysmeta-ec-etag',
-                        src_headers.get('etag', '')).strip('"'):
+                        src_headers.get('etag', ''))):
                     self.statuses.append(HTTP_NOT_FOUND)
                     self.reasons.append('')
                     self.bodies.append('')
@@ -1373,9 +1373,8 @@ class ResumingGetter(object):
             # from the same object (EC). Otherwise, if the cluster has two
             # versions of the same object, we might end up switching between
             # old and new mid-stream and giving garbage to the client.
-            self.used_source_etag = src_headers.get(
+            self.used_source_etag = normalize_etag(src_headers.get(
-                'x-object-sysmeta-ec-etag',
+                'x-object-sysmeta-ec-etag', src_headers.get('etag', '')))
-                src_headers.get('etag', '')).strip('"')
             self.node = node
             return source, node
         return None, None
@@ -1922,7 +1921,7 @@ class Controller(object):
                 if headers:
                     update_headers(resp, headers[status_index])
                 if etag:
-                    resp.headers['etag'] = etag.strip('"')
+                    resp.headers['etag'] = normalize_etag(etag)
                 return resp
         return None
 

swift/proxy/controllers/obj.py

@@ -70,7 +70,8 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \
     HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPRequestTimeout, \
     HTTPServerError, HTTPServiceUnavailable, HTTPClientDisconnect, \
     HTTPUnprocessableEntity, Response, HTTPException, \
-    HTTPRequestedRangeNotSatisfiable, Range, HTTPInternalServerError
+    HTTPRequestedRangeNotSatisfiable, Range, HTTPInternalServerError, \
+    normalize_etag
 from swift.common.request_helpers import update_etag_is_at_header, \
     resolve_etag_is_at_header, validate_internal_obj
 
@@ -478,7 +479,7 @@ class BaseObjectController(Controller):
                     {'status': response.status,
                      'body': body[:1024], 'path': req.path})
             elif is_success(response.status):
-                etags.add(response.getheader('etag').strip('"'))
+                etags.add(normalize_etag(response.getheader('etag')))
 
         for (putter, response) in pile:
             if response:
@@ -2638,8 +2639,8 @@ class ECObjectController(BaseObjectController):
             computed_etag = (etag_hasher.hexdigest()
                              if etag_hasher else None)
             footers = self._get_footers(req)
-            received_etag = footers.get('etag', req.headers.get(
+            received_etag = normalize_etag(footers.get(
-                'etag', '')).strip('"')
+                'etag', req.headers.get('etag', '')))
             if (computed_etag and received_etag and
                     computed_etag != received_etag):
                 raise HTTPUnprocessableEntity(request=req)